drivers/net/e1000/igb_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <stdarg.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_interrupts.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_log.h>
  44 #include <rte_debug.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_memory.h>
  49 #include <rte_memzone.h>
  50 #include <rte_eal.h>
  51 #include <rte_atomic.h>
  52 #include <rte_malloc.h>
  53 #include <rte_dev.h>
  54
  55 #include "e1000_logs.h"
  56 #include "base/e1000_api.h"
  57 #include "e1000_ethdev.h"
  58 #include "igb_regs.h"
  59
  60 /*
  61  * Default values for port configuration
  62  */
  63 #define IGB_DEFAULT_RX_FREE_THRESH  32
  64 #define IGB_DEFAULT_RX_PTHRESH      8
  65 #define IGB_DEFAULT_RX_HTHRESH      8
  66 #define IGB_DEFAULT_RX_WTHRESH      0
  67
  68 #define IGB_DEFAULT_TX_PTHRESH      32
  69 #define IGB_DEFAULT_TX_HTHRESH      0
  70 #define IGB_DEFAULT_TX_WTHRESH      0
  71
  72 #define IGB_HKEY_MAX_INDEX 10
  73
  74 /* Bit shift and mask */
  75 #define IGB_4_BIT_WIDTH  (CHAR_BIT / 2)
  76 #define IGB_4_BIT_MASK   RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t)
  77 #define IGB_8_BIT_WIDTH  CHAR_BIT
  78 #define IGB_8_BIT_MASK   UINT8_MAX
  79
  80 /* Additional timesync values. */
  81 #define E1000_CYCLECOUNTER_MASK      0xffffffffffffffff
  82 #define E1000_ETQF_FILTER_1588       3
  83 #define IGB_82576_TSYNC_SHIFT        16
  84 #define E1000_INCPERIOD_82576        (1 << E1000_TIMINCA_16NS_SHIFT)
  85 #define E1000_INCVALUE_82576         (16 << IGB_82576_TSYNC_SHIFT)
  86 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000
  87
  88 static int  eth_igb_configure(struct rte_eth_dev *dev);
  89 static int  eth_igb_start(struct rte_eth_dev *dev);
  90 static void eth_igb_stop(struct rte_eth_dev *dev);
  91 static void eth_igb_close(struct rte_eth_dev *dev);
  92 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev);
  93 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev);
  94 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev);
  95 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev);
  96 static int  eth_igb_link_update(struct rte_eth_dev *dev,
  97                                 int wait_to_complete);
  98 static void eth_igb_stats_get(struct rte_eth_dev *dev,
  99                                 struct rte_eth_stats *rte_stats);
 100 static int eth_igb_xstats_get(struct rte_eth_dev *dev,
 101                               struct rte_eth_xstats *xstats, unsigned n);
 102 static void eth_igb_stats_reset(struct rte_eth_dev *dev);
 103 static void eth_igb_xstats_reset(struct rte_eth_dev *dev);
 104 static void eth_igb_infos_get(struct rte_eth_dev *dev,
 105                               struct rte_eth_dev_info *dev_info);
 106 static void eth_igbvf_infos_get(struct rte_eth_dev *dev,
 107                                 struct rte_eth_dev_info *dev_info);
 108 static int  eth_igb_flow_ctrl_get(struct rte_eth_dev *dev,
 109                                 struct rte_eth_fc_conf *fc_conf);
 110 static int  eth_igb_flow_ctrl_set(struct rte_eth_dev *dev,
 111                                 struct rte_eth_fc_conf *fc_conf);
 112 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev);
 113 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev);
 114 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev);
 115 static int eth_igb_interrupt_action(struct rte_eth_dev *dev);
 116 static void eth_igb_interrupt_handler(struct rte_intr_handle *handle,
 117                                                         void *param);
 118 static int  igb_hardware_init(struct e1000_hw *hw);
 119 static void igb_hw_control_acquire(struct e1000_hw *hw);
 120 static void igb_hw_control_release(struct e1000_hw *hw);
 121 static void igb_init_manageability(struct e1000_hw *hw);
 122 static void igb_release_manageability(struct e1000_hw *hw);
 123
 124 static int  eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 125
 126 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev,
 127                 uint16_t vlan_id, int on);
 128 static void eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 129 static void eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 130
 131 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 132 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 133 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev);
 134 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev);
 135 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 136 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 137
 138 static int eth_igb_led_on(struct rte_eth_dev *dev);
 139 static int eth_igb_led_off(struct rte_eth_dev *dev);
 140
 141 static void igb_intr_disable(struct e1000_hw *hw);
 142 static int  igb_get_rx_buffer_size(struct e1000_hw *hw);
 143 static void eth_igb_rar_set(struct rte_eth_dev *dev,
 144                 struct ether_addr *mac_addr,
 145                 uint32_t index, uint32_t pool);
 146 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 147 static void eth_igb_default_mac_addr_set(struct rte_eth_dev *dev,
 148                 struct ether_addr *addr);
 149
 150 static void igbvf_intr_disable(struct e1000_hw *hw);
 151 static int igbvf_dev_configure(struct rte_eth_dev *dev);
 152 static int igbvf_dev_start(struct rte_eth_dev *dev);
 153 static void igbvf_dev_stop(struct rte_eth_dev *dev);
 154 static void igbvf_dev_close(struct rte_eth_dev *dev);
 155 static int eth_igbvf_link_update(struct e1000_hw *hw);
 156 static void eth_igbvf_stats_get(struct rte_eth_dev *dev,
 157                                 struct rte_eth_stats *rte_stats);
 158 static int eth_igbvf_xstats_get(struct rte_eth_dev *dev,
 159                                 struct rte_eth_xstats *xstats, unsigned n);
 160 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev);
 161 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev,
 162                 uint16_t vlan_id, int on);
 163 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on);
 164 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 165 static void igbvf_default_mac_addr_set(struct rte_eth_dev *dev,
 166                 struct ether_addr *addr);
 167 static int igbvf_get_reg_length(struct rte_eth_dev *dev);
 168 static int igbvf_get_regs(struct rte_eth_dev *dev,
 169                 struct rte_dev_reg_info *regs);
 170
 171 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev,
 172                                    struct rte_eth_rss_reta_entry64 *reta_conf,
 173                                    uint16_t reta_size);
 174 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev,
 175                                   struct rte_eth_rss_reta_entry64 *reta_conf,
 176                                   uint16_t reta_size);
 177
 178 static int eth_igb_syn_filter_set(struct rte_eth_dev *dev,
 179                         struct rte_eth_syn_filter *filter,
 180                         bool add);
 181 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev,
 182                         struct rte_eth_syn_filter *filter);
 183 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
 184                         enum rte_filter_op filter_op,
 185                         void *arg);
 186 static int igb_add_2tuple_filter(struct rte_eth_dev *dev,
 187                         struct rte_eth_ntuple_filter *ntuple_filter);
 188 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev,
 189                         struct rte_eth_ntuple_filter *ntuple_filter);
 190 static int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
 191                         struct rte_eth_flex_filter *filter,
 192                         bool add);
 193 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev,
 194                         struct rte_eth_flex_filter *filter);
 195 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
 196                         enum rte_filter_op filter_op,
 197                         void *arg);
 198 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
 199                         struct rte_eth_ntuple_filter *ntuple_filter);
 200 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
 201                         struct rte_eth_ntuple_filter *ntuple_filter);
 202 static int igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
 203                         struct rte_eth_ntuple_filter *filter,
 204                         bool add);
 205 static int igb_get_ntuple_filter(struct rte_eth_dev *dev,
 206                         struct rte_eth_ntuple_filter *filter);
 207 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev,
 208                                 enum rte_filter_op filter_op,
 209                                 void *arg);
 210 static int igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
 211                         struct rte_eth_ethertype_filter *filter,
 212                         bool add);
 213 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev,
 214                                 enum rte_filter_op filter_op,
 215                                 void *arg);
 216 static int igb_get_ethertype_filter(struct rte_eth_dev *dev,
 217                         struct rte_eth_ethertype_filter *filter);
 218 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev,
 219                      enum rte_filter_type filter_type,
 220                      enum rte_filter_op filter_op,
 221                      void *arg);
 222 static int eth_igb_get_reg_length(struct rte_eth_dev *dev);
 223 static int eth_igb_get_regs(struct rte_eth_dev *dev,
 224                 struct rte_dev_reg_info *regs);
 225 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev);
 226 static int eth_igb_get_eeprom(struct rte_eth_dev *dev,
 227                 struct rte_dev_eeprom_info *eeprom);
 228 static int eth_igb_set_eeprom(struct rte_eth_dev *dev,
 229                 struct rte_dev_eeprom_info *eeprom);
 230 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
 231                                     struct ether_addr *mc_addr_set,
 232                                     uint32_t nb_mc_addr);
 233 static int igb_timesync_enable(struct rte_eth_dev *dev);
 234 static int igb_timesync_disable(struct rte_eth_dev *dev);
 235 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 236                                           struct timespec *timestamp,
 237                                           uint32_t flags);
 238 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 239                                           struct timespec *timestamp);
 240 static int igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta);
 241 static int igb_timesync_read_time(struct rte_eth_dev *dev,
 242                                   struct timespec *timestamp);
 243 static int igb_timesync_write_time(struct rte_eth_dev *dev,
 244                                    const struct timespec *timestamp);
 245 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev,
 246                                         uint16_t queue_id);
 247 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev,
 248                                          uint16_t queue_id);
 249 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction,
 250                                        uint8_t queue, uint8_t msix_vector);
 251 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector,
 252                                uint8_t index, uint8_t offset);
 253 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev);
 254
 255 /*
 256  * Define VF Stats MACRO for Non "cleared on read" register
 257  */
 258 #define UPDATE_VF_STAT(reg, last, cur)            \
 259 {                                                 \
 260         u32 latest = E1000_READ_REG(hw, reg);     \
 261         cur += (latest - last) & UINT_MAX;        \
 262         last = latest;                            \
 263 }
 264
 265 #define IGB_FC_PAUSE_TIME 0x0680
 266 #define IGB_LINK_UPDATE_CHECK_TIMEOUT  90  /* 9s */
 267 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */
 268
 269 #define IGBVF_PMD_NAME "rte_igbvf_pmd"     /* PMD name */
 270
 271 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full;
 272
 273 /*
 274  * The set of PCI devices this driver supports
 275  */
 276 static const struct rte_pci_id pci_id_igb_map[] = {
 277
 278 #define RTE_PCI_DEV_ID_DECL_IGB(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 279 #include "rte_pci_dev_ids.h"
 280
 281 {0},
 282 };
 283
 284 /*
 285  * The set of PCI devices this driver supports (for 82576&I350 VF)
 286  */
 287 static const struct rte_pci_id pci_id_igbvf_map[] = {
 288
 289 #define RTE_PCI_DEV_ID_DECL_IGBVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 290 #include "rte_pci_dev_ids.h"
 291
 292 {0},
 293 };
 294
 295 static const struct rte_eth_desc_lim rx_desc_lim = {
 296         .nb_max = E1000_MAX_RING_DESC,
 297         .nb_min = E1000_MIN_RING_DESC,
 298         .nb_align = IGB_RXD_ALIGN,
 299 };
 300
 301 static const struct rte_eth_desc_lim tx_desc_lim = {
 302         .nb_max = E1000_MAX_RING_DESC,
 303         .nb_min = E1000_MIN_RING_DESC,
 304         .nb_align = IGB_RXD_ALIGN,
 305 };
 306
 307 static const struct eth_dev_ops eth_igb_ops = {
 308         .dev_configure        = eth_igb_configure,
 309         .dev_start            = eth_igb_start,
 310         .dev_stop             = eth_igb_stop,
 311         .dev_close            = eth_igb_close,
 312         .promiscuous_enable   = eth_igb_promiscuous_enable,
 313         .promiscuous_disable  = eth_igb_promiscuous_disable,
 314         .allmulticast_enable  = eth_igb_allmulticast_enable,
 315         .allmulticast_disable = eth_igb_allmulticast_disable,
 316         .link_update          = eth_igb_link_update,
 317         .stats_get            = eth_igb_stats_get,
 318         .xstats_get           = eth_igb_xstats_get,
 319         .stats_reset          = eth_igb_stats_reset,
 320         .xstats_reset         = eth_igb_xstats_reset,
 321         .dev_infos_get        = eth_igb_infos_get,
 322         .mtu_set              = eth_igb_mtu_set,
 323         .vlan_filter_set      = eth_igb_vlan_filter_set,
 324         .vlan_tpid_set        = eth_igb_vlan_tpid_set,
 325         .vlan_offload_set     = eth_igb_vlan_offload_set,
 326         .rx_queue_setup       = eth_igb_rx_queue_setup,
 327         .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable,
 328         .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable,
 329         .rx_queue_release     = eth_igb_rx_queue_release,
 330         .rx_queue_count       = eth_igb_rx_queue_count,
 331         .rx_descriptor_done   = eth_igb_rx_descriptor_done,
 332         .tx_queue_setup       = eth_igb_tx_queue_setup,
 333         .tx_queue_release     = eth_igb_tx_queue_release,
 334         .dev_led_on           = eth_igb_led_on,
 335         .dev_led_off          = eth_igb_led_off,
 336         .flow_ctrl_get        = eth_igb_flow_ctrl_get,
 337         .flow_ctrl_set        = eth_igb_flow_ctrl_set,
 338         .mac_addr_add         = eth_igb_rar_set,
 339         .mac_addr_remove      = eth_igb_rar_clear,
 340         .mac_addr_set         = eth_igb_default_mac_addr_set,
 341         .reta_update          = eth_igb_rss_reta_update,
 342         .reta_query           = eth_igb_rss_reta_query,
 343         .rss_hash_update      = eth_igb_rss_hash_update,
 344         .rss_hash_conf_get    = eth_igb_rss_hash_conf_get,
 345         .filter_ctrl          = eth_igb_filter_ctrl,
 346         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 347         .rxq_info_get         = igb_rxq_info_get,
 348         .txq_info_get         = igb_txq_info_get,
 349         .timesync_enable      = igb_timesync_enable,
 350         .timesync_disable     = igb_timesync_disable,
 351         .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp,
 352         .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp,
 353         .get_reg_length       = eth_igb_get_reg_length,
 354         .get_reg              = eth_igb_get_regs,
 355         .get_eeprom_length    = eth_igb_get_eeprom_length,
 356         .get_eeprom           = eth_igb_get_eeprom,
 357         .set_eeprom           = eth_igb_set_eeprom,
 358         .timesync_adjust_time = igb_timesync_adjust_time,
 359         .timesync_read_time   = igb_timesync_read_time,
 360         .timesync_write_time  = igb_timesync_write_time,
 361 };
 362
 363 /*
 364  * dev_ops for virtual function, bare necessities for basic vf
 365  * operation have been implemented
 366  */
 367 static const struct eth_dev_ops igbvf_eth_dev_ops = {
 368         .dev_configure        = igbvf_dev_configure,
 369         .dev_start            = igbvf_dev_start,
 370         .dev_stop             = igbvf_dev_stop,
 371         .dev_close            = igbvf_dev_close,
 372         .link_update          = eth_igb_link_update,
 373         .stats_get            = eth_igbvf_stats_get,
 374         .xstats_get           = eth_igbvf_xstats_get,
 375         .stats_reset          = eth_igbvf_stats_reset,
 376         .xstats_reset         = eth_igbvf_stats_reset,
 377         .vlan_filter_set      = igbvf_vlan_filter_set,
 378         .dev_infos_get        = eth_igbvf_infos_get,
 379         .rx_queue_setup       = eth_igb_rx_queue_setup,
 380         .rx_queue_release     = eth_igb_rx_queue_release,
 381         .tx_queue_setup       = eth_igb_tx_queue_setup,
 382         .tx_queue_release     = eth_igb_tx_queue_release,
 383         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 384         .rxq_info_get         = igb_rxq_info_get,
 385         .txq_info_get         = igb_txq_info_get,
 386         .mac_addr_set         = igbvf_default_mac_addr_set,
 387         .get_reg_length       = igbvf_get_reg_length,
 388         .get_reg              = igbvf_get_regs,
 389 };
 390
 391 /* store statistics names and its offset in stats structure */
 392 struct rte_igb_xstats_name_off {
 393         char name[RTE_ETH_XSTATS_NAME_SIZE];
 394         unsigned offset;
 395 };
 396
 397 static const struct rte_igb_xstats_name_off rte_igb_stats_strings[] = {
 398         {"rx_crc_errors", offsetof(struct e1000_hw_stats, crcerrs)},
 399         {"rx_align_errors", offsetof(struct e1000_hw_stats, algnerrc)},
 400         {"rx_symbol_errors", offsetof(struct e1000_hw_stats, symerrs)},
 401         {"rx_missed_packets", offsetof(struct e1000_hw_stats, mpc)},
 402         {"tx_single_collision_packets", offsetof(struct e1000_hw_stats, scc)},
 403         {"tx_multiple_collision_packets", offsetof(struct e1000_hw_stats, mcc)},
 404         {"tx_excessive_collision_packets", offsetof(struct e1000_hw_stats,
 405                 ecol)},
 406         {"tx_late_collisions", offsetof(struct e1000_hw_stats, latecol)},
 407         {"tx_total_collisions", offsetof(struct e1000_hw_stats, colc)},
 408         {"tx_deferred_packets", offsetof(struct e1000_hw_stats, dc)},
 409         {"tx_no_carrier_sense_packets", offsetof(struct e1000_hw_stats, tncrs)},
 410         {"rx_carrier_ext_errors", offsetof(struct e1000_hw_stats, cexterr)},
 411         {"rx_length_errors", offsetof(struct e1000_hw_stats, rlec)},
 412         {"rx_xon_packets", offsetof(struct e1000_hw_stats, xonrxc)},
 413         {"tx_xon_packets", offsetof(struct e1000_hw_stats, xontxc)},
 414         {"rx_xoff_packets", offsetof(struct e1000_hw_stats, xoffrxc)},
 415         {"tx_xoff_packets", offsetof(struct e1000_hw_stats, xofftxc)},
 416         {"rx_flow_control_unsupported_packets", offsetof(struct e1000_hw_stats,
 417                 fcruc)},
 418         {"rx_size_64_packets", offsetof(struct e1000_hw_stats, prc64)},
 419         {"rx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, prc127)},
 420         {"rx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, prc255)},
 421         {"rx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, prc511)},
 422         {"rx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats,
 423                 prc1023)},
 424         {"rx_size_1024_to_max_packets", offsetof(struct e1000_hw_stats,
 425                 prc1522)},
 426         {"rx_broadcast_packets", offsetof(struct e1000_hw_stats, bprc)},
 427         {"rx_multicast_packets", offsetof(struct e1000_hw_stats, mprc)},
 428         {"rx_undersize_errors", offsetof(struct e1000_hw_stats, ruc)},
 429         {"rx_fragment_errors", offsetof(struct e1000_hw_stats, rfc)},
 430         {"rx_oversize_errors", offsetof(struct e1000_hw_stats, roc)},
 431         {"rx_jabber_errors", offsetof(struct e1000_hw_stats, rjc)},
 432         {"rx_management_packets", offsetof(struct e1000_hw_stats, mgprc)},
 433         {"rx_management_dropped", offsetof(struct e1000_hw_stats, mgpdc)},
 434         {"tx_management_packets", offsetof(struct e1000_hw_stats, mgptc)},
 435         {"rx_total_packets", offsetof(struct e1000_hw_stats, tpr)},
 436         {"tx_total_packets", offsetof(struct e1000_hw_stats, tpt)},
 437         {"rx_total_bytes", offsetof(struct e1000_hw_stats, tor)},
 438         {"tx_total_bytes", offsetof(struct e1000_hw_stats, tot)},
 439         {"tx_size_64_packets", offsetof(struct e1000_hw_stats, ptc64)},
 440         {"tx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, ptc127)},
 441         {"tx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, ptc255)},
 442         {"tx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, ptc511)},
 443         {"tx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats,
 444                 ptc1023)},
 445         {"tx_size_1023_to_max_packets", offsetof(struct e1000_hw_stats,
 446                 ptc1522)},
 447         {"tx_multicast_packets", offsetof(struct e1000_hw_stats, mptc)},
 448         {"tx_broadcast_packets", offsetof(struct e1000_hw_stats, bptc)},
 449         {"tx_tso_packets", offsetof(struct e1000_hw_stats, tsctc)},
 450         {"tx_tso_errors", offsetof(struct e1000_hw_stats, tsctfc)},
 451         {"rx_sent_to_host_packets", offsetof(struct e1000_hw_stats, rpthc)},
 452         {"tx_sent_by_host_packets", offsetof(struct e1000_hw_stats, hgptc)},
 453         {"rx_code_violation_packets", offsetof(struct e1000_hw_stats, scvpc)},
 454
 455         {"interrupt_assert_count", offsetof(struct e1000_hw_stats, iac)},
 456 };
 457
 458 #define IGB_NB_XSTATS (sizeof(rte_igb_stats_strings) / \
 459                 sizeof(rte_igb_stats_strings[0]))
 460
 461 static const struct rte_igb_xstats_name_off rte_igbvf_stats_strings[] = {
 462         {"rx_multicast_packets", offsetof(struct e1000_vf_stats, mprc)},
 463         {"rx_good_loopback_packets", offsetof(struct e1000_vf_stats, gprlbc)},
 464         {"tx_good_loopback_packets", offsetof(struct e1000_vf_stats, gptlbc)},
 465         {"rx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gorlbc)},
 466         {"tx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gotlbc)},
 467 };
 468
 469 #define IGBVF_NB_XSTATS (sizeof(rte_igbvf_stats_strings) / \
 470                 sizeof(rte_igbvf_stats_strings[0]))
 471
 472 /**
 473  * Atomically reads the link status information from global
 474  * structure rte_eth_dev.
 475  *
 476  * @param dev
 477  *   - Pointer to the structure rte_eth_dev to read from.
 478  *   - Pointer to the buffer to be saved with the link status.
 479  *
 480  * @return
 481  *   - On success, zero.
 482  *   - On failure, negative value.
 483  */
 484 static inline int
 485 rte_igb_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 486                                 struct rte_eth_link *link)
 487 {
 488         struct rte_eth_link *dst = link;
 489         struct rte_eth_link *src = &(dev->data->dev_link);
 490
 491         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 492                                         *(uint64_t *)src) == 0)
 493                 return -1;
 494
 495         return 0;
 496 }
 497
 498 /**
 499  * Atomically writes the link status information into global
 500  * structure rte_eth_dev.
 501  *
 502  * @param dev
 503  *   - Pointer to the structure rte_eth_dev to read from.
 504  *   - Pointer to the buffer to be saved with the link status.
 505  *
 506  * @return
 507  *   - On success, zero.
 508  *   - On failure, negative value.
 509  */
 510 static inline int
 511 rte_igb_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 512                                 struct rte_eth_link *link)
 513 {
 514         struct rte_eth_link *dst = &(dev->data->dev_link);
 515         struct rte_eth_link *src = link;
 516
 517         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 518                                         *(uint64_t *)src) == 0)
 519                 return -1;
 520
 521         return 0;
 522 }
 523
 524 static inline void
 525 igb_intr_enable(struct rte_eth_dev *dev)
 526 {
 527         struct e1000_interrupt *intr =
 528                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 529         struct e1000_hw *hw =
 530                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 531
 532         E1000_WRITE_REG(hw, E1000_IMS, intr->mask);
 533         E1000_WRITE_FLUSH(hw);
 534 }
 535
 536 static void
 537 igb_intr_disable(struct e1000_hw *hw)
 538 {
 539         E1000_WRITE_REG(hw, E1000_IMC, ~0);
 540         E1000_WRITE_FLUSH(hw);
 541 }
 542
 543 static inline int32_t
 544 igb_pf_reset_hw(struct e1000_hw *hw)
 545 {
 546         uint32_t ctrl_ext;
 547         int32_t status;
 548
 549         status = e1000_reset_hw(hw);
 550
 551         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 552         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 553         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 554         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 555         E1000_WRITE_FLUSH(hw);
 556
 557         return status;
 558 }
 559
 560 static void
 561 igb_identify_hardware(struct rte_eth_dev *dev)
 562 {
 563         struct e1000_hw *hw =
 564                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 565
 566         hw->vendor_id = dev->pci_dev->id.vendor_id;
 567         hw->device_id = dev->pci_dev->id.device_id;
 568         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
 569         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
 570
 571         e1000_set_mac_type(hw);
 572
 573         /* need to check if it is a vf device below */
 574 }
 575
 576 static int
 577 igb_reset_swfw_lock(struct e1000_hw *hw)
 578 {
 579         int ret_val;
 580
 581         /*
 582          * Do mac ops initialization manually here, since we will need
 583          * some function pointers set by this call.
 584          */
 585         ret_val = e1000_init_mac_params(hw);
 586         if (ret_val)
 587                 return ret_val;
 588
 589         /*
 590          * SMBI lock should not fail in this early stage. If this is the case,
 591          * it is due to an improper exit of the application.
 592          * So force the release of the faulty lock.
 593          */
 594         if (e1000_get_hw_semaphore_generic(hw) < 0) {
 595                 PMD_DRV_LOG(DEBUG, "SMBI lock released");
 596         }
 597         e1000_put_hw_semaphore_generic(hw);
 598
 599         if (hw->mac.ops.acquire_swfw_sync != NULL) {
 600                 uint16_t mask;
 601
 602                 /*
 603                  * Phy lock should not fail in this early stage. If this is the case,
 604                  * it is due to an improper exit of the application.
 605                  * So force the release of the faulty lock.
 606                  */
 607                 mask = E1000_SWFW_PHY0_SM << hw->bus.func;
 608                 if (hw->bus.func > E1000_FUNC_1)
 609                         mask <<= 2;
 610                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 611                         PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released",
 612                                     hw->bus.func);
 613                 }
 614                 hw->mac.ops.release_swfw_sync(hw, mask);
 615
 616                 /*
 617                  * This one is more tricky since it is common to all ports; but
 618                  * swfw_sync retries last long enough (1s) to be almost sure that if
 619                  * lock can not be taken it is due to an improper lock of the
 620                  * semaphore.
 621                  */
 622                 mask = E1000_SWFW_EEP_SM;
 623                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 624                         PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 625                 }
 626                 hw->mac.ops.release_swfw_sync(hw, mask);
 627         }
 628
 629         return E1000_SUCCESS;
 630 }
 631
 632 static int
 633 eth_igb_dev_init(struct rte_eth_dev *eth_dev)
 634 {
 635         int error = 0;
 636         struct rte_pci_device *pci_dev;
 637         struct e1000_hw *hw =
 638                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 639         struct e1000_vfta * shadow_vfta =
 640                 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 641         struct e1000_filter_info *filter_info =
 642                 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
 643         struct e1000_adapter *adapter =
 644                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 645
 646         uint32_t ctrl_ext;
 647
 648         pci_dev = eth_dev->pci_dev;
 649
 650         rte_eth_copy_pci_info(eth_dev, pci_dev);
 651
 652         eth_dev->dev_ops = &eth_igb_ops;
 653         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 654         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 655
 656         /* for secondary processes, we don't initialise any further as primary
 657          * has already done this work. Only check we don't need a different
 658          * RX function */
 659         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 660                 if (eth_dev->data->scattered_rx)
 661                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 662                 return 0;
 663         }
 664
 665         hw->hw_addr= (void *)pci_dev->mem_resource[0].addr;
 666
 667         igb_identify_hardware(eth_dev);
 668         if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) {
 669                 error = -EIO;
 670                 goto err_late;
 671         }
 672
 673         e1000_get_bus_info(hw);
 674
 675         /* Reset any pending lock */
 676         if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) {
 677                 error = -EIO;
 678                 goto err_late;
 679         }
 680
 681         /* Finish initialization */
 682         if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) {
 683                 error = -EIO;
 684                 goto err_late;
 685         }
 686
 687         hw->mac.autoneg = 1;
 688         hw->phy.autoneg_wait_to_complete = 0;
 689         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 690
 691         /* Copper options */
 692         if (hw->phy.media_type == e1000_media_type_copper) {
 693                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
 694                 hw->phy.disable_polarity_correction = 0;
 695                 hw->phy.ms_type = e1000_ms_hw_default;
 696         }
 697
 698         /*
 699          * Start from a known state, this is important in reading the nvm
 700          * and mac from that.
 701          */
 702         igb_pf_reset_hw(hw);
 703
 704         /* Make sure we have a good EEPROM before we read from it */
 705         if (e1000_validate_nvm_checksum(hw) < 0) {
 706                 /*
 707                  * Some PCI-E parts fail the first check due to
 708                  * the link being in sleep state, call it again,
 709                  * if it fails a second time its a real issue.
 710                  */
 711                 if (e1000_validate_nvm_checksum(hw) < 0) {
 712                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
 713                         error = -EIO;
 714                         goto err_late;
 715                 }
 716         }
 717
 718         /* Read the permanent MAC address out of the EEPROM */
 719         if (e1000_read_mac_addr(hw) != 0) {
 720                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
 721                 error = -EIO;
 722                 goto err_late;
 723         }
 724
 725         /* Allocate memory for storing MAC addresses */
 726         eth_dev->data->mac_addrs = rte_zmalloc("e1000",
 727                 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
 728         if (eth_dev->data->mac_addrs == NULL) {
 729                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to "
 730                                                 "store MAC addresses",
 731                                 ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 732                 error = -ENOMEM;
 733                 goto err_late;
 734         }
 735
 736         /* Copy the permanent MAC address */
 737         ether_addr_copy((struct ether_addr *)hw->mac.addr, &eth_dev->data->mac_addrs[0]);
 738
 739         /* initialize the vfta */
 740         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 741
 742         /* Now initialize the hardware */
 743         if (igb_hardware_init(hw) != 0) {
 744                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
 745                 rte_free(eth_dev->data->mac_addrs);
 746                 eth_dev->data->mac_addrs = NULL;
 747                 error = -ENODEV;
 748                 goto err_late;
 749         }
 750         hw->mac.get_link_status = 1;
 751         adapter->stopped = 0;
 752
 753         /* Indicate SOL/IDER usage */
 754         if (e1000_check_reset_block(hw) < 0) {
 755                 PMD_INIT_LOG(ERR, "PHY reset is blocked due to"
 756                                         "SOL/IDER session");
 757         }
 758
 759         /* initialize PF if max_vfs not zero */
 760         igb_pf_host_init(eth_dev);
 761
 762         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 763         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 764         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 765         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 766         E1000_WRITE_FLUSH(hw);
 767
 768         PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x",
 769                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 770                      pci_dev->id.device_id);
 771
 772         rte_intr_callback_register(&pci_dev->intr_handle,
 773                                    eth_igb_interrupt_handler,
 774                                    (void *)eth_dev);
 775
 776         /* enable uio/vfio intr/eventfd mapping */
 777         rte_intr_enable(&pci_dev->intr_handle);
 778
 779         /* enable support intr */
 780         igb_intr_enable(eth_dev);
 781
 782         TAILQ_INIT(&filter_info->flex_list);
 783         filter_info->flex_mask = 0;
 784         TAILQ_INIT(&filter_info->twotuple_list);
 785         filter_info->twotuple_mask = 0;
 786         TAILQ_INIT(&filter_info->fivetuple_list);
 787         filter_info->fivetuple_mask = 0;
 788
 789         return 0;
 790
 791 err_late:
 792         igb_hw_control_release(hw);
 793
 794         return (error);
 795 }
 796
 797 static int
 798 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev)
 799 {
 800         struct rte_pci_device *pci_dev;
 801         struct e1000_hw *hw;
 802         struct e1000_adapter *adapter =
 803                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 804
 805         PMD_INIT_FUNC_TRACE();
 806
 807         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 808                 return -EPERM;
 809
 810         hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 811         pci_dev = eth_dev->pci_dev;
 812
 813         if (adapter->stopped == 0)
 814                 eth_igb_close(eth_dev);
 815
 816         eth_dev->dev_ops = NULL;
 817         eth_dev->rx_pkt_burst = NULL;
 818         eth_dev->tx_pkt_burst = NULL;
 819
 820         /* Reset any pending lock */
 821         igb_reset_swfw_lock(hw);
 822
 823         rte_free(eth_dev->data->mac_addrs);
 824         eth_dev->data->mac_addrs = NULL;
 825
 826         /* uninitialize PF if max_vfs not zero */
 827         igb_pf_host_uninit(eth_dev);
 828
 829         /* disable uio intr before callback unregister */
 830         rte_intr_disable(&(pci_dev->intr_handle));
 831         rte_intr_callback_unregister(&(pci_dev->intr_handle),
 832                 eth_igb_interrupt_handler, (void *)eth_dev);
 833
 834         return 0;
 835 }
 836
 837 /*
 838  * Virtual Function device init
 839  */
 840 static int
 841 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev)
 842 {
 843         struct rte_pci_device *pci_dev;
 844         struct e1000_adapter *adapter =
 845                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 846         struct e1000_hw *hw =
 847                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 848         int diag;
 849
 850         PMD_INIT_FUNC_TRACE();
 851
 852         eth_dev->dev_ops = &igbvf_eth_dev_ops;
 853         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 854         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 855
 856         /* for secondary processes, we don't initialise any further as primary
 857          * has already done this work. Only check we don't need a different
 858          * RX function */
 859         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 860                 if (eth_dev->data->scattered_rx)
 861                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 862                 return 0;
 863         }
 864
 865         pci_dev = eth_dev->pci_dev;
 866
 867         rte_eth_copy_pci_info(eth_dev, pci_dev);
 868
 869         hw->device_id = pci_dev->id.device_id;
 870         hw->vendor_id = pci_dev->id.vendor_id;
 871         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 872         adapter->stopped = 0;
 873
 874         /* Initialize the shared code (base driver) */
 875         diag = e1000_setup_init_funcs(hw, TRUE);
 876         if (diag != 0) {
 877                 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d",
 878                         diag);
 879                 return -EIO;
 880         }
 881
 882         /* init_mailbox_params */
 883         hw->mbx.ops.init_params(hw);
 884
 885         /* Disable the interrupts for VF */
 886         igbvf_intr_disable(hw);
 887
 888         diag = hw->mac.ops.reset_hw(hw);
 889
 890         /* Allocate memory for storing MAC addresses */
 891         eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN *
 892                 hw->mac.rar_entry_count, 0);
 893         if (eth_dev->data->mac_addrs == NULL) {
 894                 PMD_INIT_LOG(ERR,
 895                         "Failed to allocate %d bytes needed to store MAC "
 896                         "addresses",
 897                         ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 898                 return -ENOMEM;
 899         }
 900
 901         /* Copy the permanent MAC address */
 902         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 903                         &eth_dev->data->mac_addrs[0]);
 904
 905         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x "
 906                      "mac.type=%s",
 907                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 908                      pci_dev->id.device_id, "igb_mac_82576_vf");
 909
 910         return 0;
 911 }
 912
 913 static int
 914 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev)
 915 {
 916         struct e1000_adapter *adapter =
 917                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 918
 919         PMD_INIT_FUNC_TRACE();
 920
 921         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 922                 return -EPERM;
 923
 924         if (adapter->stopped == 0)
 925                 igbvf_dev_close(eth_dev);
 926
 927         eth_dev->dev_ops = NULL;
 928         eth_dev->rx_pkt_burst = NULL;
 929         eth_dev->tx_pkt_burst = NULL;
 930
 931         rte_free(eth_dev->data->mac_addrs);
 932         eth_dev->data->mac_addrs = NULL;
 933
 934         return 0;
 935 }
 936
 937 static struct eth_driver rte_igb_pmd = {
 938         .pci_drv = {
 939                 .name = "rte_igb_pmd",
 940                 .id_table = pci_id_igb_map,
 941                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
 942                         RTE_PCI_DRV_DETACHABLE,
 943         },
 944         .eth_dev_init = eth_igb_dev_init,
 945         .eth_dev_uninit = eth_igb_dev_uninit,
 946         .dev_private_size = sizeof(struct e1000_adapter),
 947 };
 948
 949 /*
 950  * virtual function driver struct
 951  */
 952 static struct eth_driver rte_igbvf_pmd = {
 953         .pci_drv = {
 954                 .name = "rte_igbvf_pmd",
 955                 .id_table = pci_id_igbvf_map,
 956                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
 957         },
 958         .eth_dev_init = eth_igbvf_dev_init,
 959         .eth_dev_uninit = eth_igbvf_dev_uninit,
 960         .dev_private_size = sizeof(struct e1000_adapter),
 961 };
 962
 963 static int
 964 rte_igb_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 965 {
 966         rte_eth_driver_register(&rte_igb_pmd);
 967         return 0;
 968 }
 969
 970 static void
 971 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 972 {
 973         struct e1000_hw *hw =
 974                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 975         /* RCTL: enable VLAN filter since VMDq always use VLAN filter */
 976         uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL);
 977         rctl |= E1000_RCTL_VFE;
 978         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
 979 }
 980
 981 /*
 982  * VF Driver initialization routine.
 983  * Invoked one at EAL init time.
 984  * Register itself as the [Virtual Poll Mode] Driver of PCI IGB devices.
 985  */
 986 static int
 987 rte_igbvf_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 988 {
 989         PMD_INIT_FUNC_TRACE();
 990
 991         rte_eth_driver_register(&rte_igbvf_pmd);
 992         return (0);
 993 }
 994
 995 static int
 996 igb_check_mq_mode(struct rte_eth_dev *dev)
 997 {
 998         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 999         enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode;
1000         uint16_t nb_rx_q = dev->data->nb_rx_queues;
1001         uint16_t nb_tx_q = dev->data->nb_rx_queues;
1002
1003         if ((rx_mq_mode & ETH_MQ_RX_DCB_FLAG) ||
1004             tx_mq_mode == ETH_MQ_TX_DCB ||
1005             tx_mq_mode == ETH_MQ_TX_VMDQ_DCB) {
1006                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
1007                 return -EINVAL;
1008         }
1009         if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
1010                 /* Check multi-queue mode.
1011                  * To no break software we accept ETH_MQ_RX_NONE as this might
1012                  * be used to turn off VLAN filter.
1013                  */
1014
1015                 if (rx_mq_mode == ETH_MQ_RX_NONE ||
1016                     rx_mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1017                         dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY;
1018                         RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1;
1019                 } else {
1020                         /* Only support one queue on VFs.
1021                          * RSS together with SRIOV is not supported.
1022                          */
1023                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1024                                         " wrong mq_mode rx %d.",
1025                                         rx_mq_mode);
1026                         return -EINVAL;
1027                 }
1028                 /* TX mode is not used here, so mode might be ignored.*/
1029                 if (tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) {
1030                         /* SRIOV only works in VMDq enable mode */
1031                         PMD_INIT_LOG(WARNING, "SRIOV is active,"
1032                                         " TX mode %d is not supported. "
1033                                         " Driver will behave as %d mode.",
1034                                         tx_mq_mode, ETH_MQ_TX_VMDQ_ONLY);
1035                 }
1036
1037                 /* check valid queue number */
1038                 if ((nb_rx_q > 1) || (nb_tx_q > 1)) {
1039                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1040                                         " only support one queue on VFs.");
1041                         return -EINVAL;
1042                 }
1043         } else {
1044                 /* To no break software that set invalid mode, only display
1045                  * warning if invalid mode is used.
1046                  */
1047                 if (rx_mq_mode != ETH_MQ_RX_NONE &&
1048                     rx_mq_mode != ETH_MQ_RX_VMDQ_ONLY &&
1049                     rx_mq_mode != ETH_MQ_RX_RSS) {
1050                         /* RSS together with VMDq not supported*/
1051                         PMD_INIT_LOG(ERR, "RX mode %d is not supported.",
1052                                      rx_mq_mode);
1053                         return -EINVAL;
1054                 }
1055
1056                 if (tx_mq_mode != ETH_MQ_TX_NONE &&
1057                     tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) {
1058                         PMD_INIT_LOG(WARNING, "TX mode %d is not supported."
1059                                         " Due to txmode is meaningless in this"
1060                                         " driver, just ignore.",
1061                                         tx_mq_mode);
1062                 }
1063         }
1064         return 0;
1065 }
1066
1067 static int
1068 eth_igb_configure(struct rte_eth_dev *dev)
1069 {
1070         struct e1000_interrupt *intr =
1071                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1072         int ret;
1073
1074         PMD_INIT_FUNC_TRACE();
1075
1076         /* multipe queue mode checking */
1077         ret  = igb_check_mq_mode(dev);
1078         if (ret != 0) {
1079                 PMD_DRV_LOG(ERR, "igb_check_mq_mode fails with %d.",
1080                             ret);
1081                 return ret;
1082         }
1083
1084         intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
1085         PMD_INIT_FUNC_TRACE();
1086
1087         return 0;
1088 }
1089
1090 static int
1091 eth_igb_start(struct rte_eth_dev *dev)
1092 {
1093         struct e1000_hw *hw =
1094                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1095         struct e1000_adapter *adapter =
1096                 E1000_DEV_PRIVATE(dev->data->dev_private);
1097         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1098         int ret, mask;
1099         uint32_t intr_vector = 0;
1100         uint32_t ctrl_ext;
1101
1102         PMD_INIT_FUNC_TRACE();
1103
1104         /* Power up the phy. Needed to make the link go Up */
1105         e1000_power_up_phy(hw);
1106
1107         /*
1108          * Packet Buffer Allocation (PBA)
1109          * Writing PBA sets the receive portion of the buffer
1110          * the remainder is used for the transmit buffer.
1111          */
1112         if (hw->mac.type == e1000_82575) {
1113                 uint32_t pba;
1114
1115                 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
1116                 E1000_WRITE_REG(hw, E1000_PBA, pba);
1117         }
1118
1119         /* Put the address into the Receive Address Array */
1120         e1000_rar_set(hw, hw->mac.addr, 0);
1121
1122         /* Initialize the hardware */
1123         if (igb_hardware_init(hw)) {
1124                 PMD_INIT_LOG(ERR, "Unable to initialize the hardware");
1125                 return (-EIO);
1126         }
1127         adapter->stopped = 0;
1128
1129         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1130
1131         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1132         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
1133         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
1134         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
1135         E1000_WRITE_FLUSH(hw);
1136
1137         /* configure PF module if SRIOV enabled */
1138         igb_pf_host_configure(dev);
1139
1140         /* check and configure queue intr-vector mapping */
1141         if ((rte_intr_cap_multiple(intr_handle) ||
1142              !RTE_ETH_DEV_SRIOV(dev).active) &&
1143             dev->data->dev_conf.intr_conf.rxq != 0) {
1144                 intr_vector = dev->data->nb_rx_queues;
1145                 if (rte_intr_efd_enable(intr_handle, intr_vector))
1146                         return -1;
1147         }
1148
1149         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
1150                 intr_handle->intr_vec =
1151                         rte_zmalloc("intr_vec",
1152                                     dev->data->nb_rx_queues * sizeof(int), 0);
1153                 if (intr_handle->intr_vec == NULL) {
1154                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
1155                                      " intr_vec\n", dev->data->nb_rx_queues);
1156                         return -ENOMEM;
1157                 }
1158         }
1159
1160         /* confiugre msix for rx interrupt */
1161         eth_igb_configure_msix_intr(dev);
1162
1163         /* Configure for OS presence */
1164         igb_init_manageability(hw);
1165
1166         eth_igb_tx_init(dev);
1167
1168         /* This can fail when allocating mbufs for descriptor rings */
1169         ret = eth_igb_rx_init(dev);
1170         if (ret) {
1171                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
1172                 igb_dev_clear_queues(dev);
1173                 return ret;
1174         }
1175
1176         e1000_clear_hw_cntrs_base_generic(hw);
1177
1178         /*
1179          * VLAN Offload Settings
1180          */
1181         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1182                         ETH_VLAN_EXTEND_MASK;
1183         eth_igb_vlan_offload_set(dev, mask);
1184
1185         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1186                 /* Enable VLAN filter since VMDq always use VLAN filter */
1187                 igb_vmdq_vlan_hw_filter_enable(dev);
1188         }
1189
1190         if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) ||
1191                 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) ||
1192                 (hw->mac.type == e1000_i211)) {
1193                 /* Configure EITR with the maximum possible value (0xFFFF) */
1194                 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF);
1195         }
1196
1197         /* Setup link speed and duplex */
1198         switch (dev->data->dev_conf.link_speed) {
1199         case ETH_LINK_SPEED_AUTONEG:
1200                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
1201                         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
1202                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
1203                         hw->phy.autoneg_advertised = E1000_ALL_HALF_DUPLEX;
1204                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
1205                         hw->phy.autoneg_advertised = E1000_ALL_FULL_DUPLEX;
1206                 else
1207                         goto error_invalid_config;
1208                 break;
1209         case ETH_LINK_SPEED_10:
1210                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
1211                         hw->phy.autoneg_advertised = E1000_ALL_10_SPEED;
1212                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
1213                         hw->phy.autoneg_advertised = ADVERTISE_10_HALF;
1214                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
1215                         hw->phy.autoneg_advertised = ADVERTISE_10_FULL;
1216                 else
1217                         goto error_invalid_config;
1218                 break;
1219         case ETH_LINK_SPEED_100:
1220                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
1221                         hw->phy.autoneg_advertised = E1000_ALL_100_SPEED;
1222                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
1223                         hw->phy.autoneg_advertised = ADVERTISE_100_HALF;
1224                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
1225                         hw->phy.autoneg_advertised = ADVERTISE_100_FULL;
1226                 else
1227                         goto error_invalid_config;
1228                 break;
1229         case ETH_LINK_SPEED_1000:
1230                 if ((dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX) ||
1231                                 (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX))
1232                         hw->phy.autoneg_advertised = ADVERTISE_1000_FULL;
1233                 else
1234                         goto error_invalid_config;
1235                 break;
1236         case ETH_LINK_SPEED_10000:
1237         default:
1238                 goto error_invalid_config;
1239         }
1240         e1000_setup_link(hw);
1241
1242         if (rte_intr_allow_others(intr_handle)) {
1243                 /* check if lsc interrupt is enabled */
1244                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1245                         eth_igb_lsc_interrupt_setup(dev);
1246         } else {
1247                 rte_intr_callback_unregister(intr_handle,
1248                                              eth_igb_interrupt_handler,
1249                                              (void *)dev);
1250                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1251                         PMD_INIT_LOG(INFO, "lsc won't enable because of"
1252                                      " no intr multiplex\n");
1253         }
1254
1255         /* check if rxq interrupt is enabled */
1256         if (dev->data->dev_conf.intr_conf.rxq != 0 &&
1257             rte_intr_dp_is_en(intr_handle))
1258                 eth_igb_rxq_interrupt_setup(dev);
1259
1260         /* enable uio/vfio intr/eventfd mapping */
1261         rte_intr_enable(intr_handle);
1262
1263         /* resume enabled intr since hw reset */
1264         igb_intr_enable(dev);
1265
1266         PMD_INIT_LOG(DEBUG, "<<");
1267
1268         return (0);
1269
1270 error_invalid_config:
1271         PMD_INIT_LOG(ERR, "Invalid link_speed/link_duplex (%u/%u) for port %u",
1272                      dev->data->dev_conf.link_speed,
1273                      dev->data->dev_conf.link_duplex, dev->data->port_id);
1274         igb_dev_clear_queues(dev);
1275         return (-EINVAL);
1276 }
1277
1278 /*********************************************************************
1279  *
1280  *  This routine disables all traffic on the adapter by issuing a
1281  *  global reset on the MAC.
1282  *
1283  **********************************************************************/
1284 static void
1285 eth_igb_stop(struct rte_eth_dev *dev)
1286 {
1287         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1288         struct e1000_filter_info *filter_info =
1289                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
1290         struct rte_eth_link link;
1291         struct e1000_flex_filter *p_flex;
1292         struct e1000_5tuple_filter *p_5tuple, *p_5tuple_next;
1293         struct e1000_2tuple_filter *p_2tuple, *p_2tuple_next;
1294         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1295
1296         igb_intr_disable(hw);
1297
1298         /* disable intr eventfd mapping */
1299         rte_intr_disable(intr_handle);
1300
1301         igb_pf_reset_hw(hw);
1302         E1000_WRITE_REG(hw, E1000_WUC, 0);
1303
1304         /* Set bit for Go Link disconnect */
1305         if (hw->mac.type >= e1000_82580) {
1306                 uint32_t phpm_reg;
1307
1308                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1309                 phpm_reg |= E1000_82580_PM_GO_LINKD;
1310                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1311         }
1312
1313         /* Power down the phy. Needed to make the link go Down */
1314         if (hw->phy.media_type == e1000_media_type_copper)
1315                 e1000_power_down_phy(hw);
1316         else
1317                 e1000_shutdown_fiber_serdes_link(hw);
1318
1319         igb_dev_clear_queues(dev);
1320
1321         /* clear the recorded link status */
1322         memset(&link, 0, sizeof(link));
1323         rte_igb_dev_atomic_write_link_status(dev, &link);
1324
1325         /* Remove all flex filters of the device */
1326         while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) {
1327                 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries);
1328                 rte_free(p_flex);
1329         }
1330         filter_info->flex_mask = 0;
1331
1332         /* Remove all ntuple filters of the device */
1333         for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list);
1334              p_5tuple != NULL; p_5tuple = p_5tuple_next) {
1335                 p_5tuple_next = TAILQ_NEXT(p_5tuple, entries);
1336                 TAILQ_REMOVE(&filter_info->fivetuple_list,
1337                              p_5tuple, entries);
1338                 rte_free(p_5tuple);
1339         }
1340         filter_info->fivetuple_mask = 0;
1341         for (p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list);
1342              p_2tuple != NULL; p_2tuple = p_2tuple_next) {
1343                 p_2tuple_next = TAILQ_NEXT(p_2tuple, entries);
1344                 TAILQ_REMOVE(&filter_info->twotuple_list,
1345                              p_2tuple, entries);
1346                 rte_free(p_2tuple);
1347         }
1348         filter_info->twotuple_mask = 0;
1349
1350         if (!rte_intr_allow_others(intr_handle))
1351                 /* resume to the default handler */
1352                 rte_intr_callback_register(intr_handle,
1353                                            eth_igb_interrupt_handler,
1354                                            (void *)dev);
1355
1356         /* Clean datapath event and queue/vec mapping */
1357         rte_intr_efd_disable(intr_handle);
1358         if (intr_handle->intr_vec != NULL) {
1359                 rte_free(intr_handle->intr_vec);
1360                 intr_handle->intr_vec = NULL;
1361         }
1362 }
1363
1364 static void
1365 eth_igb_close(struct rte_eth_dev *dev)
1366 {
1367         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1368         struct e1000_adapter *adapter =
1369                 E1000_DEV_PRIVATE(dev->data->dev_private);
1370         struct rte_eth_link link;
1371         struct rte_pci_device *pci_dev;
1372
1373         eth_igb_stop(dev);
1374         adapter->stopped = 1;
1375
1376         e1000_phy_hw_reset(hw);
1377         igb_release_manageability(hw);
1378         igb_hw_control_release(hw);
1379
1380         /* Clear bit for Go Link disconnect */
1381         if (hw->mac.type >= e1000_82580) {
1382                 uint32_t phpm_reg;
1383
1384                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1385                 phpm_reg &= ~E1000_82580_PM_GO_LINKD;
1386                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1387         }
1388
1389         igb_dev_free_queues(dev);
1390
1391         pci_dev = dev->pci_dev;
1392         if (pci_dev->intr_handle.intr_vec) {
1393                 rte_free(pci_dev->intr_handle.intr_vec);
1394                 pci_dev->intr_handle.intr_vec = NULL;
1395         }
1396
1397         memset(&link, 0, sizeof(link));
1398         rte_igb_dev_atomic_write_link_status(dev, &link);
1399 }
1400
1401 static int
1402 igb_get_rx_buffer_size(struct e1000_hw *hw)
1403 {
1404         uint32_t rx_buf_size;
1405         if (hw->mac.type == e1000_82576) {
1406                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10;
1407         } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) {
1408                 /* PBS needs to be translated according to a lookup table */
1409                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf);
1410                 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size);
1411                 rx_buf_size = (rx_buf_size << 10);
1412         } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) {
1413                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10;
1414         } else {
1415                 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10;
1416         }
1417
1418         return rx_buf_size;
1419 }
1420
1421 /*********************************************************************
1422  *
1423  *  Initialize the hardware
1424  *
1425  **********************************************************************/
1426 static int
1427 igb_hardware_init(struct e1000_hw *hw)
1428 {
1429         uint32_t rx_buf_size;
1430         int diag;
1431
1432         /* Let the firmware know the OS is in control */
1433         igb_hw_control_acquire(hw);
1434
1435         /*
1436          * These parameters control the automatic generation (Tx) and
1437          * response (Rx) to Ethernet PAUSE frames.
1438          * - High water mark should allow for at least two standard size (1518)
1439          *   frames to be received after sending an XOFF.
1440          * - Low water mark works best when it is very near the high water mark.
1441          *   This allows the receiver to restart by sending XON when it has
1442          *   drained a bit. Here we use an arbitrary value of 1500 which will
1443          *   restart after one full frame is pulled from the buffer. There
1444          *   could be several smaller frames in the buffer and if so they will
1445          *   not trigger the XON until their total number reduces the buffer
1446          *   by 1500.
1447          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
1448          */
1449         rx_buf_size = igb_get_rx_buffer_size(hw);
1450
1451         hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2);
1452         hw->fc.low_water = hw->fc.high_water - 1500;
1453         hw->fc.pause_time = IGB_FC_PAUSE_TIME;
1454         hw->fc.send_xon = 1;
1455
1456         /* Set Flow control, use the tunable location if sane */
1457         if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4))
1458                 hw->fc.requested_mode = igb_fc_setting;
1459         else
1460                 hw->fc.requested_mode = e1000_fc_none;
1461
1462         /* Issue a global reset */
1463         igb_pf_reset_hw(hw);
1464         E1000_WRITE_REG(hw, E1000_WUC, 0);
1465
1466         diag = e1000_init_hw(hw);
1467         if (diag < 0)
1468                 return (diag);
1469
1470         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1471         e1000_get_phy_info(hw);
1472         e1000_check_for_link(hw);
1473
1474         return (0);
1475 }
1476
1477 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */
1478 static void
1479 igb_read_stats_registers(struct e1000_hw *hw, struct e1000_hw_stats *stats)
1480 {
1481         int pause_frames;
1482
1483         if(hw->phy.media_type == e1000_media_type_copper ||
1484             (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
1485                 stats->symerrs +=
1486                     E1000_READ_REG(hw,E1000_SYMERRS);
1487                 stats->sec += E1000_READ_REG(hw, E1000_SEC);
1488         }
1489
1490         stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
1491         stats->mpc += E1000_READ_REG(hw, E1000_MPC);
1492         stats->scc += E1000_READ_REG(hw, E1000_SCC);
1493         stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
1494
1495         stats->mcc += E1000_READ_REG(hw, E1000_MCC);
1496         stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
1497         stats->colc += E1000_READ_REG(hw, E1000_COLC);
1498         stats->dc += E1000_READ_REG(hw, E1000_DC);
1499         stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
1500         stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
1501         stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
1502         /*
1503         ** For watchdog management we need to know if we have been
1504         ** paused during the last interval, so capture that here.
1505         */
1506         pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC);
1507         stats->xoffrxc += pause_frames;
1508         stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
1509         stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
1510         stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
1511         stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
1512         stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
1513         stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
1514         stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
1515         stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
1516         stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
1517         stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
1518         stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
1519         stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
1520
1521         /* For the 64-bit byte counters the low dword must be read first. */
1522         /* Both registers clear on the read of the high dword */
1523
1524         stats->gorc += E1000_READ_REG(hw, E1000_GORCL);
1525         stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32);
1526         stats->gotc += E1000_READ_REG(hw, E1000_GOTCL);
1527         stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32);
1528
1529         stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
1530         stats->ruc += E1000_READ_REG(hw, E1000_RUC);
1531         stats->rfc += E1000_READ_REG(hw, E1000_RFC);
1532         stats->roc += E1000_READ_REG(hw, E1000_ROC);
1533         stats->rjc += E1000_READ_REG(hw, E1000_RJC);
1534
1535         stats->tor += E1000_READ_REG(hw, E1000_TORL);
1536         stats->tor += ((uint64_t)E1000_READ_REG(hw, E1000_TORH) << 32);
1537         stats->tot += E1000_READ_REG(hw, E1000_TOTL);
1538         stats->tot += ((uint64_t)E1000_READ_REG(hw, E1000_TOTH) << 32);
1539
1540         stats->tpr += E1000_READ_REG(hw, E1000_TPR);
1541         stats->tpt += E1000_READ_REG(hw, E1000_TPT);
1542         stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
1543         stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
1544         stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
1545         stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
1546         stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
1547         stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
1548         stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
1549         stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
1550
1551         /* Interrupt Counts */
1552
1553         stats->iac += E1000_READ_REG(hw, E1000_IAC);
1554         stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
1555         stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
1556         stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
1557         stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
1558         stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
1559         stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
1560         stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
1561         stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
1562
1563         /* Host to Card Statistics */
1564
1565         stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
1566         stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
1567         stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
1568         stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
1569         stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
1570         stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
1571         stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
1572         stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL);
1573         stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32);
1574         stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL);
1575         stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32);
1576         stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
1577         stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
1578         stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
1579
1580         stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
1581         stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
1582         stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
1583         stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
1584         stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
1585         stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
1586 }
1587
1588 static void
1589 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1590 {
1591         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1592         struct e1000_hw_stats *stats =
1593                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1594
1595         igb_read_stats_registers(hw, stats);
1596
1597         if (rte_stats == NULL)
1598                 return;
1599
1600         /* Rx Errors */
1601         rte_stats->imissed = stats->mpc;
1602         rte_stats->ierrors = stats->crcerrs +
1603                              stats->rlec + stats->ruc + stats->roc +
1604                              rte_stats->imissed +
1605                              stats->rxerrc + stats->algnerrc + stats->cexterr;
1606
1607         /* Tx Errors */
1608         rte_stats->oerrors = stats->ecol + stats->latecol;
1609
1610         rte_stats->ipackets = stats->gprc;
1611         rte_stats->opackets = stats->gptc;
1612         rte_stats->ibytes   = stats->gorc;
1613         rte_stats->obytes   = stats->gotc;
1614 }
1615
1616 static void
1617 eth_igb_stats_reset(struct rte_eth_dev *dev)
1618 {
1619         struct e1000_hw_stats *hw_stats =
1620                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1621
1622         /* HW registers are cleared on read */
1623         eth_igb_stats_get(dev, NULL);
1624
1625         /* Reset software totals */
1626         memset(hw_stats, 0, sizeof(*hw_stats));
1627 }
1628
1629 static void
1630 eth_igb_xstats_reset(struct rte_eth_dev *dev)
1631 {
1632         struct e1000_hw_stats *stats =
1633                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1634
1635         /* HW registers are cleared on read */
1636         eth_igb_xstats_get(dev, NULL, IGB_NB_XSTATS);
1637
1638         /* Reset software totals */
1639         memset(stats, 0, sizeof(*stats));
1640 }
1641
1642 static int
1643 eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
1644                    unsigned n)
1645 {
1646         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1647         struct e1000_hw_stats *hw_stats =
1648                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1649         unsigned i;
1650
1651         if (n < IGB_NB_XSTATS)
1652                 return IGB_NB_XSTATS;
1653
1654         igb_read_stats_registers(hw, hw_stats);
1655
1656         /* If this is a reset xstats is NULL, and we have cleared the
1657          * registers by reading them.
1658          */
1659         if (!xstats)
1660                 return 0;
1661
1662         /* Extended stats */
1663         for (i = 0; i < IGB_NB_XSTATS; i++) {
1664                 snprintf(xstats[i].name, sizeof(xstats[i].name),
1665                          "%s", rte_igb_stats_strings[i].name);
1666                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
1667                         rte_igb_stats_strings[i].offset);
1668         }
1669
1670         return IGB_NB_XSTATS;
1671 }
1672
1673 static void
1674 igbvf_read_stats_registers(struct e1000_hw *hw, struct e1000_vf_stats *hw_stats)
1675 {
1676         /* Good Rx packets, include VF loopback */
1677         UPDATE_VF_STAT(E1000_VFGPRC,
1678             hw_stats->last_gprc, hw_stats->gprc);
1679
1680         /* Good Rx octets, include VF loopback */
1681         UPDATE_VF_STAT(E1000_VFGORC,
1682             hw_stats->last_gorc, hw_stats->gorc);
1683
1684         /* Good Tx packets, include VF loopback */
1685         UPDATE_VF_STAT(E1000_VFGPTC,
1686             hw_stats->last_gptc, hw_stats->gptc);
1687
1688         /* Good Tx octets, include VF loopback */
1689         UPDATE_VF_STAT(E1000_VFGOTC,
1690             hw_stats->last_gotc, hw_stats->gotc);
1691
1692         /* Rx Multicst packets */
1693         UPDATE_VF_STAT(E1000_VFMPRC,
1694             hw_stats->last_mprc, hw_stats->mprc);
1695
1696         /* Good Rx loopback packets */
1697         UPDATE_VF_STAT(E1000_VFGPRLBC,
1698             hw_stats->last_gprlbc, hw_stats->gprlbc);
1699
1700         /* Good Rx loopback octets */
1701         UPDATE_VF_STAT(E1000_VFGORLBC,
1702             hw_stats->last_gorlbc, hw_stats->gorlbc);
1703
1704         /* Good Tx loopback packets */
1705         UPDATE_VF_STAT(E1000_VFGPTLBC,
1706             hw_stats->last_gptlbc, hw_stats->gptlbc);
1707
1708         /* Good Tx loopback octets */
1709         UPDATE_VF_STAT(E1000_VFGOTLBC,
1710             hw_stats->last_gotlbc, hw_stats->gotlbc);
1711 }
1712
1713 static int
1714 eth_igbvf_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
1715                      unsigned n)
1716 {
1717         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1718         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *)
1719                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1720         unsigned i;
1721
1722         if (n < IGBVF_NB_XSTATS)
1723                 return IGBVF_NB_XSTATS;
1724
1725         igbvf_read_stats_registers(hw, hw_stats);
1726
1727         if (!xstats)
1728                 return 0;
1729
1730         for (i = 0; i < IGBVF_NB_XSTATS; i++) {
1731                 snprintf(xstats[i].name, sizeof(xstats[i].name), "%s",
1732                          rte_igbvf_stats_strings[i].name);
1733                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
1734                         rte_igbvf_stats_strings[i].offset);
1735         }
1736
1737         return IGBVF_NB_XSTATS;
1738 }
1739
1740 static void
1741 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1742 {
1743         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1744         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *)
1745                           E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1746
1747         igbvf_read_stats_registers(hw, hw_stats);
1748
1749         if (rte_stats == NULL)
1750                 return;
1751
1752         rte_stats->ipackets = hw_stats->gprc;
1753         rte_stats->ibytes = hw_stats->gorc;
1754         rte_stats->opackets = hw_stats->gptc;
1755         rte_stats->obytes = hw_stats->gotc;
1756         rte_stats->imcasts = hw_stats->mprc;
1757         rte_stats->ilbpackets = hw_stats->gprlbc;
1758         rte_stats->ilbbytes = hw_stats->gorlbc;
1759         rte_stats->olbpackets = hw_stats->gptlbc;
1760         rte_stats->olbbytes = hw_stats->gotlbc;
1761 }
1762
1763 static void
1764 eth_igbvf_stats_reset(struct rte_eth_dev *dev)
1765 {
1766         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1767                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1768
1769         /* Sync HW register to the last stats */
1770         eth_igbvf_stats_get(dev, NULL);
1771
1772         /* reset HW current stats*/
1773         memset(&hw_stats->gprc, 0, sizeof(*hw_stats) -
1774                offsetof(struct e1000_vf_stats, gprc));
1775 }
1776
1777 static void
1778 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1779 {
1780         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1781
1782         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1783         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1784         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1785         dev_info->rx_offload_capa =
1786                 DEV_RX_OFFLOAD_VLAN_STRIP |
1787                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1788                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1789                 DEV_RX_OFFLOAD_TCP_CKSUM;
1790         dev_info->tx_offload_capa =
1791                 DEV_TX_OFFLOAD_VLAN_INSERT |
1792                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1793                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1794                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1795                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
1796                 DEV_TX_OFFLOAD_TCP_TSO;
1797
1798         switch (hw->mac.type) {
1799         case e1000_82575:
1800                 dev_info->max_rx_queues = 4;
1801                 dev_info->max_tx_queues = 4;
1802                 dev_info->max_vmdq_pools = 0;
1803                 break;
1804
1805         case e1000_82576:
1806                 dev_info->max_rx_queues = 16;
1807                 dev_info->max_tx_queues = 16;
1808                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1809                 dev_info->vmdq_queue_num = 16;
1810                 break;
1811
1812         case e1000_82580:
1813                 dev_info->max_rx_queues = 8;
1814                 dev_info->max_tx_queues = 8;
1815                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1816                 dev_info->vmdq_queue_num = 8;
1817                 break;
1818
1819         case e1000_i350:
1820                 dev_info->max_rx_queues = 8;
1821                 dev_info->max_tx_queues = 8;
1822                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1823                 dev_info->vmdq_queue_num = 8;
1824                 break;
1825
1826         case e1000_i354:
1827                 dev_info->max_rx_queues = 8;
1828                 dev_info->max_tx_queues = 8;
1829                 break;
1830
1831         case e1000_i210:
1832                 dev_info->max_rx_queues = 4;
1833                 dev_info->max_tx_queues = 4;
1834                 dev_info->max_vmdq_pools = 0;
1835                 break;
1836
1837         case e1000_i211:
1838                 dev_info->max_rx_queues = 2;
1839                 dev_info->max_tx_queues = 2;
1840                 dev_info->max_vmdq_pools = 0;
1841                 break;
1842
1843         default:
1844                 /* Should not happen */
1845                 break;
1846         }
1847         dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t);
1848         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
1849         dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL;
1850
1851         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1852                 .rx_thresh = {
1853                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1854                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1855                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
1856                 },
1857                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
1858                 .rx_drop_en = 0,
1859         };
1860
1861         dev_info->default_txconf = (struct rte_eth_txconf) {
1862                 .tx_thresh = {
1863                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
1864                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
1865                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
1866                 },
1867                 .txq_flags = 0,
1868         };
1869
1870         dev_info->rx_desc_lim = rx_desc_lim;
1871         dev_info->tx_desc_lim = tx_desc_lim;
1872 }
1873
1874 static void
1875 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1876 {
1877         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1878
1879         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1880         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1881         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1882         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
1883                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1884                                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1885                                 DEV_RX_OFFLOAD_TCP_CKSUM;
1886         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
1887                                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1888                                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1889                                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1890                                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
1891                                 DEV_TX_OFFLOAD_TCP_TSO;
1892         switch (hw->mac.type) {
1893         case e1000_vfadapt:
1894                 dev_info->max_rx_queues = 2;
1895                 dev_info->max_tx_queues = 2;
1896                 break;
1897         case e1000_vfadapt_i350:
1898                 dev_info->max_rx_queues = 1;
1899                 dev_info->max_tx_queues = 1;
1900                 break;
1901         default:
1902                 /* Should not happen */
1903                 break;
1904         }
1905
1906         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1907                 .rx_thresh = {
1908                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1909                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1910                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
1911                 },
1912                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
1913                 .rx_drop_en = 0,
1914         };
1915
1916         dev_info->default_txconf = (struct rte_eth_txconf) {
1917                 .tx_thresh = {
1918                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
1919                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
1920                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
1921                 },
1922                 .txq_flags = 0,
1923         };
1924
1925         dev_info->rx_desc_lim = rx_desc_lim;
1926         dev_info->tx_desc_lim = tx_desc_lim;
1927 }
1928
1929 /* return 0 means link status changed, -1 means not changed */
1930 static int
1931 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1932 {
1933         struct e1000_hw *hw =
1934                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1935         struct rte_eth_link link, old;
1936         int link_check, count;
1937
1938         link_check = 0;
1939         hw->mac.get_link_status = 1;
1940
1941         /* possible wait-to-complete in up to 9 seconds */
1942         for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) {
1943                 /* Read the real link status */
1944                 switch (hw->phy.media_type) {
1945                 case e1000_media_type_copper:
1946                         /* Do the work to read phy */
1947                         e1000_check_for_link(hw);
1948                         link_check = !hw->mac.get_link_status;
1949                         break;
1950
1951                 case e1000_media_type_fiber:
1952                         e1000_check_for_link(hw);
1953                         link_check = (E1000_READ_REG(hw, E1000_STATUS) &
1954                                       E1000_STATUS_LU);
1955                         break;
1956
1957                 case e1000_media_type_internal_serdes:
1958                         e1000_check_for_link(hw);
1959                         link_check = hw->mac.serdes_has_link;
1960                         break;
1961
1962                 /* VF device is type_unknown */
1963                 case e1000_media_type_unknown:
1964                         eth_igbvf_link_update(hw);
1965                         link_check = !hw->mac.get_link_status;
1966                         break;
1967
1968                 default:
1969                         break;
1970                 }
1971                 if (link_check || wait_to_complete == 0)
1972                         break;
1973                 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL);
1974         }
1975         memset(&link, 0, sizeof(link));
1976         rte_igb_dev_atomic_read_link_status(dev, &link);
1977         old = link;
1978
1979         /* Now we check if a transition has happened */
1980         if (link_check) {
1981                 hw->mac.ops.get_link_up_info(hw, &link.link_speed,
1982                                           &link.link_duplex);
1983                 link.link_status = 1;
1984         } else if (!link_check) {
1985                 link.link_speed = 0;
1986                 link.link_duplex = 0;
1987                 link.link_status = 0;
1988         }
1989         rte_igb_dev_atomic_write_link_status(dev, &link);
1990
1991         /* not changed */
1992         if (old.link_status == link.link_status)
1993                 return -1;
1994
1995         /* changed */
1996         return 0;
1997 }
1998
1999 /*
2000  * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
2001  * For ASF and Pass Through versions of f/w this means
2002  * that the driver is loaded.
2003  */
2004 static void
2005 igb_hw_control_acquire(struct e1000_hw *hw)
2006 {
2007         uint32_t ctrl_ext;
2008
2009         /* Let firmware know the driver has taken over */
2010         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
2011         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
2012 }
2013
2014 /*
2015  * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
2016  * For ASF and Pass Through versions of f/w this means that the
2017  * driver is no longer loaded.
2018  */
2019 static void
2020 igb_hw_control_release(struct e1000_hw *hw)
2021 {
2022         uint32_t ctrl_ext;
2023
2024         /* Let firmware taken over control of h/w */
2025         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
2026         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
2027                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
2028 }
2029
2030 /*
2031  * Bit of a misnomer, what this really means is
2032  * to enable OS management of the system... aka
2033  * to disable special hardware management features.
2034  */
2035 static void
2036 igb_init_manageability(struct e1000_hw *hw)
2037 {
2038         if (e1000_enable_mng_pass_thru(hw)) {
2039                 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H);
2040                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
2041
2042                 /* disable hardware interception of ARP */
2043                 manc &= ~(E1000_MANC_ARP_EN);
2044
2045                 /* enable receiving management packets to the host */
2046                 manc |= E1000_MANC_EN_MNG2HOST;
2047                 manc2h |= 1 << 5;  /* Mng Port 623 */
2048                 manc2h |= 1 << 6;  /* Mng Port 664 */
2049                 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h);
2050                 E1000_WRITE_REG(hw, E1000_MANC, manc);
2051         }
2052 }
2053
2054 static void
2055 igb_release_manageability(struct e1000_hw *hw)
2056 {
2057         if (e1000_enable_mng_pass_thru(hw)) {
2058                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
2059
2060                 manc |= E1000_MANC_ARP_EN;
2061                 manc &= ~E1000_MANC_EN_MNG2HOST;
2062
2063                 E1000_WRITE_REG(hw, E1000_MANC, manc);
2064         }
2065 }
2066
2067 static void
2068 eth_igb_promiscuous_enable(struct rte_eth_dev *dev)
2069 {
2070         struct e1000_hw *hw =
2071                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2072         uint32_t rctl;
2073
2074         rctl = E1000_READ_REG(hw, E1000_RCTL);
2075         rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2076         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2077 }
2078
2079 static void
2080 eth_igb_promiscuous_disable(struct rte_eth_dev *dev)
2081 {
2082         struct e1000_hw *hw =
2083                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2084         uint32_t rctl;
2085
2086         rctl = E1000_READ_REG(hw, E1000_RCTL);
2087         rctl &= (~E1000_RCTL_UPE);
2088         if (dev->data->all_multicast == 1)
2089                 rctl |= E1000_RCTL_MPE;
2090         else
2091                 rctl &= (~E1000_RCTL_MPE);
2092         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2093 }
2094
2095 static void
2096 eth_igb_allmulticast_enable(struct rte_eth_dev *dev)
2097 {
2098         struct e1000_hw *hw =
2099                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2100         uint32_t rctl;
2101
2102         rctl = E1000_READ_REG(hw, E1000_RCTL);
2103         rctl |= E1000_RCTL_MPE;
2104         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2105 }
2106
2107 static void
2108 eth_igb_allmulticast_disable(struct rte_eth_dev *dev)
2109 {
2110         struct e1000_hw *hw =
2111                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2112         uint32_t rctl;
2113
2114         if (dev->data->promiscuous == 1)
2115                 return; /* must remain in all_multicast mode */
2116         rctl = E1000_READ_REG(hw, E1000_RCTL);
2117         rctl &= (~E1000_RCTL_MPE);
2118         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2119 }
2120
2121 static int
2122 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2123 {
2124         struct e1000_hw *hw =
2125                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2126         struct e1000_vfta * shadow_vfta =
2127                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2128         uint32_t vfta;
2129         uint32_t vid_idx;
2130         uint32_t vid_bit;
2131
2132         vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) &
2133                               E1000_VFTA_ENTRY_MASK);
2134         vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK));
2135         vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx);
2136         if (on)
2137                 vfta |= vid_bit;
2138         else
2139                 vfta &= ~vid_bit;
2140         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta);
2141
2142         /* update local VFTA copy */
2143         shadow_vfta->vfta[vid_idx] = vfta;
2144
2145         return 0;
2146 }
2147
2148 static void
2149 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
2150 {
2151         struct e1000_hw *hw =
2152                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2153         uint32_t reg = ETHER_TYPE_VLAN ;
2154
2155         reg |= (tpid << 16);
2156         E1000_WRITE_REG(hw, E1000_VET, reg);
2157 }
2158
2159 static void
2160 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev)
2161 {
2162         struct e1000_hw *hw =
2163                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2164         uint32_t reg;
2165
2166         /* Filter Table Disable */
2167         reg = E1000_READ_REG(hw, E1000_RCTL);
2168         reg &= ~E1000_RCTL_CFIEN;
2169         reg &= ~E1000_RCTL_VFE;
2170         E1000_WRITE_REG(hw, E1000_RCTL, reg);
2171 }
2172
2173 static void
2174 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev)
2175 {
2176         struct e1000_hw *hw =
2177                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2178         struct e1000_vfta * shadow_vfta =
2179                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2180         uint32_t reg;
2181         int i;
2182
2183         /* Filter Table Enable, CFI not used for packet acceptance */
2184         reg = E1000_READ_REG(hw, E1000_RCTL);
2185         reg &= ~E1000_RCTL_CFIEN;
2186         reg |= E1000_RCTL_VFE;
2187         E1000_WRITE_REG(hw, E1000_RCTL, reg);
2188
2189         /* restore VFTA table */
2190         for (i = 0; i < IGB_VFTA_SIZE; i++)
2191                 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]);
2192 }
2193
2194 static void
2195 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev)
2196 {
2197         struct e1000_hw *hw =
2198                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2199         uint32_t reg;
2200
2201         /* VLAN Mode Disable */
2202         reg = E1000_READ_REG(hw, E1000_CTRL);
2203         reg &= ~E1000_CTRL_VME;
2204         E1000_WRITE_REG(hw, E1000_CTRL, reg);
2205 }
2206
2207 static void
2208 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev)
2209 {
2210         struct e1000_hw *hw =
2211                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2212         uint32_t reg;
2213
2214         /* VLAN Mode Enable */
2215         reg = E1000_READ_REG(hw, E1000_CTRL);
2216         reg |= E1000_CTRL_VME;
2217         E1000_WRITE_REG(hw, E1000_CTRL, reg);
2218 }
2219
2220 static void
2221 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev)
2222 {
2223         struct e1000_hw *hw =
2224                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2225         uint32_t reg;
2226
2227         /* CTRL_EXT: Extended VLAN */
2228         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
2229         reg &= ~E1000_CTRL_EXT_EXTEND_VLAN;
2230         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
2231
2232         /* Update maximum packet length */
2233         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
2234                 E1000_WRITE_REG(hw, E1000_RLPML,
2235                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
2236                                                 VLAN_TAG_SIZE);
2237 }
2238
2239 static void
2240 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev)
2241 {
2242         struct e1000_hw *hw =
2243                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2244         uint32_t reg;
2245
2246         /* CTRL_EXT: Extended VLAN */
2247         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
2248         reg |= E1000_CTRL_EXT_EXTEND_VLAN;
2249         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
2250
2251         /* Update maximum packet length */
2252         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
2253                 E1000_WRITE_REG(hw, E1000_RLPML,
2254                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
2255                                                 2 * VLAN_TAG_SIZE);
2256 }
2257
2258 static void
2259 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2260 {
2261         if(mask & ETH_VLAN_STRIP_MASK){
2262                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
2263                         igb_vlan_hw_strip_enable(dev);
2264                 else
2265                         igb_vlan_hw_strip_disable(dev);
2266         }
2267
2268         if(mask & ETH_VLAN_FILTER_MASK){
2269                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
2270                         igb_vlan_hw_filter_enable(dev);
2271                 else
2272                         igb_vlan_hw_filter_disable(dev);
2273         }
2274
2275         if(mask & ETH_VLAN_EXTEND_MASK){
2276                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
2277                         igb_vlan_hw_extend_enable(dev);
2278                 else
2279                         igb_vlan_hw_extend_disable(dev);
2280         }
2281 }
2282
2283
2284 /**
2285  * It enables the interrupt mask and then enable the interrupt.
2286  *
2287  * @param dev
2288  *  Pointer to struct rte_eth_dev.
2289  *
2290  * @return
2291  *  - On success, zero.
2292  *  - On failure, a negative value.
2293  */
2294 static int
2295 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev)
2296 {
2297         struct e1000_interrupt *intr =
2298                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2299
2300         intr->mask |= E1000_ICR_LSC;
2301
2302         return 0;
2303 }
2304
2305 /* It clears the interrupt causes and enables the interrupt.
2306  * It will be called once only during nic initialized.
2307  *
2308  * @param dev
2309  *  Pointer to struct rte_eth_dev.
2310  *
2311  * @return
2312  *  - On success, zero.
2313  *  - On failure, a negative value.
2314  */
2315 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev)
2316 {
2317         uint32_t mask, regval;
2318         struct e1000_hw *hw =
2319                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2320         struct rte_eth_dev_info dev_info;
2321
2322         memset(&dev_info, 0, sizeof(dev_info));
2323         eth_igb_infos_get(dev, &dev_info);
2324
2325         mask = 0xFFFFFFFF >> (32 - dev_info.max_rx_queues);
2326         regval = E1000_READ_REG(hw, E1000_EIMS);
2327         E1000_WRITE_REG(hw, E1000_EIMS, regval | mask);
2328
2329         return 0;
2330 }
2331
2332 /*
2333  * It reads ICR and gets interrupt causes, check it and set a bit flag
2334  * to update link status.
2335  *
2336  * @param dev
2337  *  Pointer to struct rte_eth_dev.
2338  *
2339  * @return
2340  *  - On success, zero.
2341  *  - On failure, a negative value.
2342  */
2343 static int
2344 eth_igb_interrupt_get_status(struct rte_eth_dev *dev)
2345 {
2346         uint32_t icr;
2347         struct e1000_hw *hw =
2348                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2349         struct e1000_interrupt *intr =
2350                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2351
2352         igb_intr_disable(hw);
2353
2354         /* read-on-clear nic registers here */
2355         icr = E1000_READ_REG(hw, E1000_ICR);
2356
2357         intr->flags = 0;
2358         if (icr & E1000_ICR_LSC) {
2359                 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
2360         }
2361
2362         if (icr & E1000_ICR_VMMB)
2363                 intr->flags |= E1000_FLAG_MAILBOX;
2364
2365         return 0;
2366 }
2367
2368 /*
2369  * It executes link_update after knowing an interrupt is prsent.
2370  *
2371  * @param dev
2372  *  Pointer to struct rte_eth_dev.
2373  *
2374  * @return
2375  *  - On success, zero.
2376  *  - On failure, a negative value.
2377  */
2378 static int
2379 eth_igb_interrupt_action(struct rte_eth_dev *dev)
2380 {
2381         struct e1000_hw *hw =
2382                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2383         struct e1000_interrupt *intr =
2384                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2385         uint32_t tctl, rctl;
2386         struct rte_eth_link link;
2387         int ret;
2388
2389         if (intr->flags & E1000_FLAG_MAILBOX) {
2390                 igb_pf_mbx_process(dev);
2391                 intr->flags &= ~E1000_FLAG_MAILBOX;
2392         }
2393
2394         igb_intr_enable(dev);
2395         rte_intr_enable(&(dev->pci_dev->intr_handle));
2396
2397         if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) {
2398                 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE;
2399
2400                 /* set get_link_status to check register later */
2401                 hw->mac.get_link_status = 1;
2402                 ret = eth_igb_link_update(dev, 0);
2403
2404                 /* check if link has changed */
2405                 if (ret < 0)
2406                         return 0;
2407
2408                 memset(&link, 0, sizeof(link));
2409                 rte_igb_dev_atomic_read_link_status(dev, &link);
2410                 if (link.link_status) {
2411                         PMD_INIT_LOG(INFO,
2412                                      " Port %d: Link Up - speed %u Mbps - %s",
2413                                      dev->data->port_id,
2414                                      (unsigned)link.link_speed,
2415                                      link.link_duplex == ETH_LINK_FULL_DUPLEX ?
2416                                      "full-duplex" : "half-duplex");
2417                 } else {
2418                         PMD_INIT_LOG(INFO, " Port %d: Link Down",
2419                                      dev->data->port_id);
2420                 }
2421
2422                 PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d",
2423                              dev->pci_dev->addr.domain,
2424                              dev->pci_dev->addr.bus,
2425                              dev->pci_dev->addr.devid,
2426                              dev->pci_dev->addr.function);
2427                 tctl = E1000_READ_REG(hw, E1000_TCTL);
2428                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2429                 if (link.link_status) {
2430                         /* enable Tx/Rx */
2431                         tctl |= E1000_TCTL_EN;
2432                         rctl |= E1000_RCTL_EN;
2433                 } else {
2434                         /* disable Tx/Rx */
2435                         tctl &= ~E1000_TCTL_EN;
2436                         rctl &= ~E1000_RCTL_EN;
2437                 }
2438                 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
2439                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2440                 E1000_WRITE_FLUSH(hw);
2441                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
2442         }
2443
2444         return 0;
2445 }
2446
2447 /**
2448  * Interrupt handler which shall be registered at first.
2449  *
2450  * @param handle
2451  *  Pointer to interrupt handle.
2452  * @param param
2453  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2454  *
2455  * @return
2456  *  void
2457  */
2458 static void
2459 eth_igb_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
2460                                                         void *param)
2461 {
2462         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2463
2464         eth_igb_interrupt_get_status(dev);
2465         eth_igb_interrupt_action(dev);
2466 }
2467
2468 static int
2469 eth_igb_led_on(struct rte_eth_dev *dev)
2470 {
2471         struct e1000_hw *hw;
2472
2473         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2474         return (e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
2475 }
2476
2477 static int
2478 eth_igb_led_off(struct rte_eth_dev *dev)
2479 {
2480         struct e1000_hw *hw;
2481
2482         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2483         return (e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
2484 }
2485
2486 static int
2487 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2488 {
2489         struct e1000_hw *hw;
2490         uint32_t ctrl;
2491         int tx_pause;
2492         int rx_pause;
2493
2494         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2495         fc_conf->pause_time = hw->fc.pause_time;
2496         fc_conf->high_water = hw->fc.high_water;
2497         fc_conf->low_water = hw->fc.low_water;
2498         fc_conf->send_xon = hw->fc.send_xon;
2499         fc_conf->autoneg = hw->mac.autoneg;
2500
2501         /*
2502          * Return rx_pause and tx_pause status according to actual setting of
2503          * the TFCE and RFCE bits in the CTRL register.
2504          */
2505         ctrl = E1000_READ_REG(hw, E1000_CTRL);
2506         if (ctrl & E1000_CTRL_TFCE)
2507                 tx_pause = 1;
2508         else
2509                 tx_pause = 0;
2510
2511         if (ctrl & E1000_CTRL_RFCE)
2512                 rx_pause = 1;
2513         else
2514                 rx_pause = 0;
2515
2516         if (rx_pause && tx_pause)
2517                 fc_conf->mode = RTE_FC_FULL;
2518         else if (rx_pause)
2519                 fc_conf->mode = RTE_FC_RX_PAUSE;
2520         else if (tx_pause)
2521                 fc_conf->mode = RTE_FC_TX_PAUSE;
2522         else
2523                 fc_conf->mode = RTE_FC_NONE;
2524
2525         return 0;
2526 }
2527
2528 static int
2529 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2530 {
2531         struct e1000_hw *hw;
2532         int err;
2533         enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = {
2534                 e1000_fc_none,
2535                 e1000_fc_rx_pause,
2536                 e1000_fc_tx_pause,
2537                 e1000_fc_full
2538         };
2539         uint32_t rx_buf_size;
2540         uint32_t max_high_water;
2541         uint32_t rctl;
2542
2543         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2544         if (fc_conf->autoneg != hw->mac.autoneg)
2545                 return -ENOTSUP;
2546         rx_buf_size = igb_get_rx_buffer_size(hw);
2547         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2548
2549         /* At least reserve one Ethernet frame for watermark */
2550         max_high_water = rx_buf_size - ETHER_MAX_LEN;
2551         if ((fc_conf->high_water > max_high_water) ||
2552             (fc_conf->high_water < fc_conf->low_water)) {
2553                 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value");
2554                 PMD_INIT_LOG(ERR, "high water must <=  0x%x", max_high_water);
2555                 return (-EINVAL);
2556         }
2557
2558         hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode];
2559         hw->fc.pause_time     = fc_conf->pause_time;
2560         hw->fc.high_water     = fc_conf->high_water;
2561         hw->fc.low_water      = fc_conf->low_water;
2562         hw->fc.send_xon       = fc_conf->send_xon;
2563
2564         err = e1000_setup_link_generic(hw);
2565         if (err == E1000_SUCCESS) {
2566
2567                 /* check if we want to forward MAC frames - driver doesn't have native
2568                  * capability to do that, so we'll write the registers ourselves */
2569
2570                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2571
2572                 /* set or clear MFLCN.PMCF bit depending on configuration */
2573                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2574                         rctl |= E1000_RCTL_PMCF;
2575                 else
2576                         rctl &= ~E1000_RCTL_PMCF;
2577
2578                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2579                 E1000_WRITE_FLUSH(hw);
2580
2581                 return 0;
2582         }
2583
2584         PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err);
2585         return (-EIO);
2586 }
2587
2588 #define E1000_RAH_POOLSEL_SHIFT      (18)
2589 static void
2590 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2591                 uint32_t index, __rte_unused uint32_t pool)
2592 {
2593         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2594         uint32_t rah;
2595
2596         e1000_rar_set(hw, mac_addr->addr_bytes, index);
2597         rah = E1000_READ_REG(hw, E1000_RAH(index));
2598         rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool));
2599         E1000_WRITE_REG(hw, E1000_RAH(index), rah);
2600 }
2601
2602 static void
2603 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index)
2604 {
2605         uint8_t addr[ETHER_ADDR_LEN];
2606         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2607
2608         memset(addr, 0, sizeof(addr));
2609
2610         e1000_rar_set(hw, addr, index);
2611 }
2612
2613 static void
2614 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev,
2615                                 struct ether_addr *addr)
2616 {
2617         eth_igb_rar_clear(dev, 0);
2618
2619         eth_igb_rar_set(dev, (void *)addr, 0, 0);
2620 }
2621 /*
2622  * Virtual Function operations
2623  */
2624 static void
2625 igbvf_intr_disable(struct e1000_hw *hw)
2626 {
2627         PMD_INIT_FUNC_TRACE();
2628
2629         /* Clear interrupt mask to stop from interrupts being generated */
2630         E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF);
2631
2632         E1000_WRITE_FLUSH(hw);
2633 }
2634
2635 static void
2636 igbvf_stop_adapter(struct rte_eth_dev *dev)
2637 {
2638         u32 reg_val;
2639         u16 i;
2640         struct rte_eth_dev_info dev_info;
2641         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2642
2643         memset(&dev_info, 0, sizeof(dev_info));
2644         eth_igbvf_infos_get(dev, &dev_info);
2645
2646         /* Clear interrupt mask to stop from interrupts being generated */
2647         igbvf_intr_disable(hw);
2648
2649         /* Clear any pending interrupts, flush previous writes */
2650         E1000_READ_REG(hw, E1000_EICR);
2651
2652         /* Disable the transmit unit.  Each queue must be disabled. */
2653         for (i = 0; i < dev_info.max_tx_queues; i++)
2654                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH);
2655
2656         /* Disable the receive unit by stopping each queue */
2657         for (i = 0; i < dev_info.max_rx_queues; i++) {
2658                 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i));
2659                 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE;
2660                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val);
2661                 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE)
2662                         ;
2663         }
2664
2665         /* flush all queues disables */
2666         E1000_WRITE_FLUSH(hw);
2667         msec_delay(2);
2668 }
2669
2670 static int eth_igbvf_link_update(struct e1000_hw *hw)
2671 {
2672         struct e1000_mbx_info *mbx = &hw->mbx;
2673         struct e1000_mac_info *mac = &hw->mac;
2674         int ret_val = E1000_SUCCESS;
2675
2676         PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf");
2677
2678         /*
2679          * We only want to run this if there has been a rst asserted.
2680          * in this case that could mean a link change, device reset,
2681          * or a virtual function reset
2682          */
2683
2684         /* If we were hit with a reset or timeout drop the link */
2685         if (!e1000_check_for_rst(hw, 0) || !mbx->timeout)
2686                 mac->get_link_status = TRUE;
2687
2688         if (!mac->get_link_status)
2689                 goto out;
2690
2691         /* if link status is down no point in checking to see if pf is up */
2692         if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU))
2693                 goto out;
2694
2695         /* if we passed all the tests above then the link is up and we no
2696          * longer need to check for link */
2697         mac->get_link_status = FALSE;
2698
2699 out:
2700         return ret_val;
2701 }
2702
2703
2704 static int
2705 igbvf_dev_configure(struct rte_eth_dev *dev)
2706 {
2707         struct rte_eth_conf* conf = &dev->data->dev_conf;
2708
2709         PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
2710                      dev->data->port_id);
2711
2712         /*
2713          * VF has no ability to enable/disable HW CRC
2714          * Keep the persistent behavior the same as Host PF
2715          */
2716 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC
2717         if (!conf->rxmode.hw_strip_crc) {
2718                 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip");
2719                 conf->rxmode.hw_strip_crc = 1;
2720         }
2721 #else
2722         if (conf->rxmode.hw_strip_crc) {
2723                 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip");
2724                 conf->rxmode.hw_strip_crc = 0;
2725         }
2726 #endif
2727
2728         return 0;
2729 }
2730
2731 static int
2732 igbvf_dev_start(struct rte_eth_dev *dev)
2733 {
2734         struct e1000_hw *hw =
2735                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2736         struct e1000_adapter *adapter =
2737                 E1000_DEV_PRIVATE(dev->data->dev_private);
2738         int ret;
2739
2740         PMD_INIT_FUNC_TRACE();
2741
2742         hw->mac.ops.reset_hw(hw);
2743         adapter->stopped = 0;
2744
2745         /* Set all vfta */
2746         igbvf_set_vfta_all(dev,1);
2747
2748         eth_igbvf_tx_init(dev);
2749
2750         /* This can fail when allocating mbufs for descriptor rings */
2751         ret = eth_igbvf_rx_init(dev);
2752         if (ret) {
2753                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
2754                 igb_dev_clear_queues(dev);
2755                 return ret;
2756         }
2757
2758         return 0;
2759 }
2760
2761 static void
2762 igbvf_dev_stop(struct rte_eth_dev *dev)
2763 {
2764         PMD_INIT_FUNC_TRACE();
2765
2766         igbvf_stop_adapter(dev);
2767
2768         /*
2769           * Clear what we set, but we still keep shadow_vfta to
2770           * restore after device starts
2771           */
2772         igbvf_set_vfta_all(dev,0);
2773
2774         igb_dev_clear_queues(dev);
2775 }
2776
2777 static void
2778 igbvf_dev_close(struct rte_eth_dev *dev)
2779 {
2780         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2781         struct e1000_adapter *adapter =
2782                 E1000_DEV_PRIVATE(dev->data->dev_private);
2783
2784         PMD_INIT_FUNC_TRACE();
2785
2786         e1000_reset_hw(hw);
2787
2788         igbvf_dev_stop(dev);
2789         adapter->stopped = 1;
2790         igb_dev_free_queues(dev);
2791 }
2792
2793 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on)
2794 {
2795         struct e1000_mbx_info *mbx = &hw->mbx;
2796         uint32_t msgbuf[2];
2797         s32 err;
2798
2799         /* After set vlan, vlan strip will also be enabled in igb driver*/
2800         msgbuf[0] = E1000_VF_SET_VLAN;
2801         msgbuf[1] = vid;
2802         /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */
2803         if (on)
2804                 msgbuf[0] |= E1000_VF_SET_VLAN_ADD;
2805
2806         err = mbx->ops.write_posted(hw, msgbuf, 2, 0);
2807         if (err)
2808                 goto mbx_err;
2809
2810         err = mbx->ops.read_posted(hw, msgbuf, 2, 0);
2811         if (err)
2812                 goto mbx_err;
2813
2814         msgbuf[0] &= ~E1000_VT_MSGTYPE_CTS;
2815         if (msgbuf[0] == (E1000_VF_SET_VLAN | E1000_VT_MSGTYPE_NACK))
2816                 err = -EINVAL;
2817
2818 mbx_err:
2819         return err;
2820 }
2821
2822 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2823 {
2824         struct e1000_hw *hw =
2825                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2826         struct e1000_vfta * shadow_vfta =
2827                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2828         int i = 0, j = 0, vfta = 0, mask = 1;
2829
2830         for (i = 0; i < IGB_VFTA_SIZE; i++){
2831                 vfta = shadow_vfta->vfta[i];
2832                 if(vfta){
2833                         mask = 1;
2834                         for (j = 0; j < 32; j++){
2835                                 if(vfta & mask)
2836                                         igbvf_set_vfta(hw,
2837                                                 (uint16_t)((i<<5)+j), on);
2838                                 mask<<=1;
2839                         }
2840                 }
2841         }
2842
2843 }
2844
2845 static int
2846 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2847 {
2848         struct e1000_hw *hw =
2849                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2850         struct e1000_vfta * shadow_vfta =
2851                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2852         uint32_t vid_idx = 0;
2853         uint32_t vid_bit = 0;
2854         int ret = 0;
2855
2856         PMD_INIT_FUNC_TRACE();
2857
2858         /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/
2859         ret = igbvf_set_vfta(hw, vlan_id, !!on);
2860         if(ret){
2861                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2862                 return ret;
2863         }
2864         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2865         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2866
2867         /*Save what we set and retore it after device reset*/
2868         if (on)
2869                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2870         else
2871                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2872
2873         return 0;
2874 }
2875
2876 static void
2877 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr)
2878 {
2879         struct e1000_hw *hw =
2880                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2881
2882         /* index is not used by rar_set() */
2883         hw->mac.ops.rar_set(hw, (void *)addr, 0);
2884 }
2885
2886
2887 static int
2888 eth_igb_rss_reta_update(struct rte_eth_dev *dev,
2889                         struct rte_eth_rss_reta_entry64 *reta_conf,
2890                         uint16_t reta_size)
2891 {
2892         uint8_t i, j, mask;
2893         uint32_t reta, r;
2894         uint16_t idx, shift;
2895         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2896
2897         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2898                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2899                         "(%d) doesn't match the number hardware can supported "
2900                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2901                 return -EINVAL;
2902         }
2903
2904         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
2905                 idx = i / RTE_RETA_GROUP_SIZE;
2906                 shift = i % RTE_RETA_GROUP_SIZE;
2907                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2908                                                 IGB_4_BIT_MASK);
2909                 if (!mask)
2910                         continue;
2911                 if (mask == IGB_4_BIT_MASK)
2912                         r = 0;
2913                 else
2914                         r = E1000_READ_REG(hw, E1000_RETA(i >> 2));
2915                 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) {
2916                         if (mask & (0x1 << j))
2917                                 reta |= reta_conf[idx].reta[shift + j] <<
2918                                                         (CHAR_BIT * j);
2919                         else
2920                                 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j));
2921                 }
2922                 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta);
2923         }
2924
2925         return 0;
2926 }
2927
2928 static int
2929 eth_igb_rss_reta_query(struct rte_eth_dev *dev,
2930                        struct rte_eth_rss_reta_entry64 *reta_conf,
2931                        uint16_t reta_size)
2932 {
2933         uint8_t i, j, mask;
2934         uint32_t reta;
2935         uint16_t idx, shift;
2936         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2937
2938         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2939                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2940                         "(%d) doesn't match the number hardware can supported "
2941                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2942                 return -EINVAL;
2943         }
2944
2945         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
2946                 idx = i / RTE_RETA_GROUP_SIZE;
2947                 shift = i % RTE_RETA_GROUP_SIZE;
2948                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2949                                                 IGB_4_BIT_MASK);
2950                 if (!mask)
2951                         continue;
2952                 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2));
2953                 for (j = 0; j < IGB_4_BIT_WIDTH; j++) {
2954                         if (mask & (0x1 << j))
2955                                 reta_conf[idx].reta[shift + j] =
2956                                         ((reta >> (CHAR_BIT * j)) &
2957                                                 IGB_8_BIT_MASK);
2958                 }
2959         }
2960
2961         return 0;
2962 }
2963
2964 #define MAC_TYPE_FILTER_SUP(type)    do {\
2965         if ((type) != e1000_82580 && (type) != e1000_i350 &&\
2966                 (type) != e1000_82576)\
2967                 return -ENOTSUP;\
2968 } while (0)
2969
2970 static int
2971 eth_igb_syn_filter_set(struct rte_eth_dev *dev,
2972                         struct rte_eth_syn_filter *filter,
2973                         bool add)
2974 {
2975         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2976         uint32_t synqf, rfctl;
2977
2978         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
2979                 return -EINVAL;
2980
2981         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
2982
2983         if (add) {
2984                 if (synqf & E1000_SYN_FILTER_ENABLE)
2985                         return -EINVAL;
2986
2987                 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) &
2988                         E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE);
2989
2990                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
2991                 if (filter->hig_pri)
2992                         rfctl |= E1000_RFCTL_SYNQFP;
2993                 else
2994                         rfctl &= ~E1000_RFCTL_SYNQFP;
2995
2996                 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl);
2997         } else {
2998                 if (!(synqf & E1000_SYN_FILTER_ENABLE))
2999                         return -ENOENT;
3000                 synqf = 0;
3001         }
3002
3003         E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf);
3004         E1000_WRITE_FLUSH(hw);
3005         return 0;
3006 }
3007
3008 static int
3009 eth_igb_syn_filter_get(struct rte_eth_dev *dev,
3010                         struct rte_eth_syn_filter *filter)
3011 {
3012         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3013         uint32_t synqf, rfctl;
3014
3015         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
3016         if (synqf & E1000_SYN_FILTER_ENABLE) {
3017                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
3018                 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0;
3019                 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >>
3020                                 E1000_SYN_FILTER_QUEUE_SHIFT);
3021                 return 0;
3022         }
3023
3024         return -ENOENT;
3025 }
3026
3027 static int
3028 eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
3029                         enum rte_filter_op filter_op,
3030                         void *arg)
3031 {
3032         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3033         int ret;
3034
3035         MAC_TYPE_FILTER_SUP(hw->mac.type);
3036
3037         if (filter_op == RTE_ETH_FILTER_NOP)
3038                 return 0;
3039
3040         if (arg == NULL) {
3041                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
3042                             filter_op);
3043                 return -EINVAL;
3044         }
3045
3046         switch (filter_op) {
3047         case RTE_ETH_FILTER_ADD:
3048                 ret = eth_igb_syn_filter_set(dev,
3049                                 (struct rte_eth_syn_filter *)arg,
3050                                 TRUE);
3051                 break;
3052         case RTE_ETH_FILTER_DELETE:
3053                 ret = eth_igb_syn_filter_set(dev,
3054                                 (struct rte_eth_syn_filter *)arg,
3055                                 FALSE);
3056                 break;
3057         case RTE_ETH_FILTER_GET:
3058                 ret = eth_igb_syn_filter_get(dev,
3059                                 (struct rte_eth_syn_filter *)arg);
3060                 break;
3061         default:
3062                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
3063                 ret = -EINVAL;
3064                 break;
3065         }
3066
3067         return ret;
3068 }
3069
3070 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
3071         if ((type) != e1000_82580 && (type) != e1000_i350)\
3072                 return -ENOSYS; \
3073 } while (0)
3074
3075 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/
3076 static inline int
3077 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter,
3078                         struct e1000_2tuple_filter_info *filter_info)
3079 {
3080         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
3081                 return -EINVAL;
3082         if (filter->priority > E1000_2TUPLE_MAX_PRI)
3083                 return -EINVAL;  /* filter index is out of range. */
3084         if (filter->tcp_flags > TCP_FLAG_ALL)
3085                 return -EINVAL;  /* flags is invalid. */
3086
3087         switch (filter->dst_port_mask) {
3088         case UINT16_MAX:
3089                 filter_info->dst_port_mask = 0;
3090                 filter_info->dst_port = filter->dst_port;
3091                 break;
3092         case 0:
3093                 filter_info->dst_port_mask = 1;
3094                 break;
3095         default:
3096                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
3097                 return -EINVAL;
3098         }
3099
3100         switch (filter->proto_mask) {
3101         case UINT8_MAX:
3102                 filter_info->proto_mask = 0;
3103                 filter_info->proto = filter->proto;
3104                 break;
3105         case 0:
3106                 filter_info->proto_mask = 1;
3107                 break;
3108         default:
3109                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
3110                 return -EINVAL;
3111         }
3112
3113         filter_info->priority = (uint8_t)filter->priority;
3114         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
3115                 filter_info->tcp_flags = filter->tcp_flags;
3116         else
3117                 filter_info->tcp_flags = 0;
3118
3119         return 0;
3120 }
3121
3122 static inline struct e1000_2tuple_filter *
3123 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list,
3124                         struct e1000_2tuple_filter_info *key)
3125 {
3126         struct e1000_2tuple_filter *it;
3127
3128         TAILQ_FOREACH(it, filter_list, entries) {
3129                 if (memcmp(key, &it->filter_info,
3130                         sizeof(struct e1000_2tuple_filter_info)) == 0) {
3131                         return it;
3132                 }
3133         }
3134         return NULL;
3135 }
3136
3137 /*
3138  * igb_add_2tuple_filter - add a 2tuple filter
3139  *
3140  * @param
3141  * dev: Pointer to struct rte_eth_dev.
3142  * ntuple_filter: ponter to the filter that will be added.
3143  *
3144  * @return
3145  *    - On success, zero.
3146  *    - On failure, a negative value.
3147  */
3148 static int
3149 igb_add_2tuple_filter(struct rte_eth_dev *dev,
3150                         struct rte_eth_ntuple_filter *ntuple_filter)
3151 {
3152         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3153         struct e1000_filter_info *filter_info =
3154                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3155         struct e1000_2tuple_filter *filter;
3156         uint32_t ttqf = E1000_TTQF_DISABLE_MASK;
3157         uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP;
3158         int i, ret;
3159
3160         filter = rte_zmalloc("e1000_2tuple_filter",
3161                         sizeof(struct e1000_2tuple_filter), 0);
3162         if (filter == NULL)
3163                 return -ENOMEM;
3164
3165         ret = ntuple_filter_to_2tuple(ntuple_filter,
3166                                       &filter->filter_info);
3167         if (ret < 0) {
3168                 rte_free(filter);
3169                 return ret;
3170         }
3171         if (igb_2tuple_filter_lookup(&filter_info->twotuple_list,
3172                                          &filter->filter_info) != NULL) {
3173                 PMD_DRV_LOG(ERR, "filter exists.");
3174                 rte_free(filter);
3175                 return -EEXIST;
3176         }
3177         filter->queue = ntuple_filter->queue;
3178
3179         /*
3180          * look for an unused 2tuple filter index,
3181          * and insert the filter to list.
3182          */
3183         for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) {
3184                 if (!(filter_info->twotuple_mask & (1 << i))) {
3185                         filter_info->twotuple_mask |= 1 << i;
3186                         filter->index = i;
3187                         TAILQ_INSERT_TAIL(&filter_info->twotuple_list,
3188                                           filter,
3189                                           entries);
3190                         break;
3191                 }
3192         }
3193         if (i >= E1000_MAX_TTQF_FILTERS) {
3194                 PMD_DRV_LOG(ERR, "2tuple filters are full.");
3195                 rte_free(filter);
3196                 return -ENOSYS;
3197         }
3198
3199         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
3200         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
3201                 imir |= E1000_IMIR_PORT_BP;
3202         else
3203                 imir &= ~E1000_IMIR_PORT_BP;
3204
3205         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
3206
3207         ttqf |= E1000_TTQF_QUEUE_ENABLE;
3208         ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT);
3209         ttqf |= (uint32_t)(filter->filter_info.proto & E1000_TTQF_PROTOCOL_MASK);
3210         if (filter->filter_info.proto_mask == 0)
3211                 ttqf &= ~E1000_TTQF_MASK_ENABLE;
3212
3213         /* tcp flags bits setting. */
3214         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
3215                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
3216                         imir_ext |= E1000_IMIREXT_CTRL_URG;
3217                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
3218                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
3219                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
3220                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
3221                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
3222                         imir_ext |= E1000_IMIREXT_CTRL_RST;
3223                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
3224                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
3225                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
3226                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
3227         } else
3228                 imir_ext |= E1000_IMIREXT_CTRL_BP;
3229         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
3230         E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf);
3231         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
3232         return 0;
3233 }
3234
3235 /*
3236  * igb_remove_2tuple_filter - remove a 2tuple filter
3237  *
3238  * @param
3239  * dev: Pointer to struct rte_eth_dev.
3240  * ntuple_filter: ponter to the filter that will be removed.
3241  *
3242  * @return
3243  *    - On success, zero.
3244  *    - On failure, a negative value.
3245  */
3246 static int
3247 igb_remove_2tuple_filter(struct rte_eth_dev *dev,
3248                         struct rte_eth_ntuple_filter *ntuple_filter)
3249 {
3250         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3251         struct e1000_filter_info *filter_info =
3252                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3253         struct e1000_2tuple_filter_info filter_2tuple;
3254         struct e1000_2tuple_filter *filter;
3255         int ret;
3256
3257         memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info));
3258         ret = ntuple_filter_to_2tuple(ntuple_filter,
3259                                       &filter_2tuple);
3260         if (ret < 0)
3261                 return ret;
3262
3263         filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list,
3264                                          &filter_2tuple);
3265         if (filter == NULL) {
3266                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3267                 return -ENOENT;
3268         }
3269
3270         filter_info->twotuple_mask &= ~(1 << filter->index);
3271         TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries);
3272         rte_free(filter);
3273
3274         E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK);
3275         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
3276         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
3277         return 0;
3278 }
3279
3280 static inline struct e1000_flex_filter *
3281 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list,
3282                         struct e1000_flex_filter_info *key)
3283 {
3284         struct e1000_flex_filter *it;
3285
3286         TAILQ_FOREACH(it, filter_list, entries) {
3287                 if (memcmp(key, &it->filter_info,
3288                         sizeof(struct e1000_flex_filter_info)) == 0)
3289                         return it;
3290         }
3291
3292         return NULL;
3293 }
3294
3295 static int
3296 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
3297                         struct rte_eth_flex_filter *filter,
3298                         bool add)
3299 {
3300         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3301         struct e1000_filter_info *filter_info =
3302                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3303         struct e1000_flex_filter *flex_filter, *it;
3304         uint32_t wufc, queueing, mask;
3305         uint32_t reg_off;
3306         uint8_t shift, i, j = 0;
3307
3308         flex_filter = rte_zmalloc("e1000_flex_filter",
3309                         sizeof(struct e1000_flex_filter), 0);
3310         if (flex_filter == NULL)
3311                 return -ENOMEM;
3312
3313         flex_filter->filter_info.len = filter->len;
3314         flex_filter->filter_info.priority = filter->priority;
3315         memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len);
3316         for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) {
3317                 mask = 0;
3318                 /* reverse bits in flex filter's mask*/
3319                 for (shift = 0; shift < CHAR_BIT; shift++) {
3320                         if (filter->mask[i] & (0x01 << shift))
3321                                 mask |= (0x80 >> shift);
3322                 }
3323                 flex_filter->filter_info.mask[i] = mask;
3324         }
3325
3326         wufc = E1000_READ_REG(hw, E1000_WUFC);
3327         if (flex_filter->index < E1000_MAX_FHFT)
3328                 reg_off = E1000_FHFT(flex_filter->index);
3329         else
3330                 reg_off = E1000_FHFT_EXT(flex_filter->index - E1000_MAX_FHFT);
3331
3332         if (add) {
3333                 if (eth_igb_flex_filter_lookup(&filter_info->flex_list,
3334                                 &flex_filter->filter_info) != NULL) {
3335                         PMD_DRV_LOG(ERR, "filter exists.");
3336                         rte_free(flex_filter);
3337                         return -EEXIST;
3338                 }
3339                 flex_filter->queue = filter->queue;
3340                 /*
3341                  * look for an unused flex filter index
3342                  * and insert the filter into the list.
3343                  */
3344                 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) {
3345                         if (!(filter_info->flex_mask & (1 << i))) {
3346                                 filter_info->flex_mask |= 1 << i;
3347                                 flex_filter->index = i;
3348                                 TAILQ_INSERT_TAIL(&filter_info->flex_list,
3349                                         flex_filter,
3350                                         entries);
3351                                 break;
3352                         }
3353                 }
3354                 if (i >= E1000_MAX_FLEX_FILTERS) {
3355                         PMD_DRV_LOG(ERR, "flex filters are full.");
3356                         rte_free(flex_filter);
3357                         return -ENOSYS;
3358                 }
3359
3360                 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ |
3361                                 (E1000_WUFC_FLX0 << flex_filter->index));
3362                 queueing = filter->len |
3363                         (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) |
3364                         (filter->priority << E1000_FHFT_QUEUEING_PRIO_SHIFT);
3365                 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET,
3366                                 queueing);
3367                 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) {
3368                         E1000_WRITE_REG(hw, reg_off,
3369                                         flex_filter->filter_info.dwords[j]);
3370                         reg_off += sizeof(uint32_t);
3371                         E1000_WRITE_REG(hw, reg_off,
3372                                         flex_filter->filter_info.dwords[++j]);
3373                         reg_off += sizeof(uint32_t);
3374                         E1000_WRITE_REG(hw, reg_off,
3375                                 (uint32_t)flex_filter->filter_info.mask[i]);
3376                         reg_off += sizeof(uint32_t) * 2;
3377                         ++j;
3378                 }
3379         } else {
3380                 it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
3381                                 &flex_filter->filter_info);
3382                 if (it == NULL) {
3383                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3384                         rte_free(flex_filter);
3385                         return -ENOENT;
3386                 }
3387
3388                 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++)
3389                         E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0);
3390                 E1000_WRITE_REG(hw, E1000_WUFC, wufc &
3391                         (~(E1000_WUFC_FLX0 << it->index)));
3392
3393                 filter_info->flex_mask &= ~(1 << it->index);
3394                 TAILQ_REMOVE(&filter_info->flex_list, it, entries);
3395                 rte_free(it);
3396                 rte_free(flex_filter);
3397         }
3398
3399         return 0;
3400 }
3401
3402 static int
3403 eth_igb_get_flex_filter(struct rte_eth_dev *dev,
3404                         struct rte_eth_flex_filter *filter)
3405 {
3406         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3407         struct e1000_filter_info *filter_info =
3408                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3409         struct e1000_flex_filter flex_filter, *it;
3410         uint32_t wufc, queueing, wufc_en = 0;
3411
3412         memset(&flex_filter, 0, sizeof(struct e1000_flex_filter));
3413         flex_filter.filter_info.len = filter->len;
3414         flex_filter.filter_info.priority = filter->priority;
3415         memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len);
3416         memcpy(flex_filter.filter_info.mask, filter->mask,
3417                         RTE_ALIGN(filter->len, sizeof(char)) / sizeof(char));
3418
3419         it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
3420                                 &flex_filter.filter_info);
3421         if (it == NULL) {
3422                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3423                 return -ENOENT;
3424         }
3425
3426         wufc = E1000_READ_REG(hw, E1000_WUFC);
3427         wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index);
3428
3429         if ((wufc & wufc_en) == wufc_en) {
3430                 uint32_t reg_off = 0;
3431                 if (it->index < E1000_MAX_FHFT)
3432                         reg_off = E1000_FHFT(it->index);
3433                 else
3434                         reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT);
3435
3436                 queueing = E1000_READ_REG(hw,
3437                                 reg_off + E1000_FHFT_QUEUEING_OFFSET);
3438                 filter->len = queueing & E1000_FHFT_QUEUEING_LEN;
3439                 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >>
3440                         E1000_FHFT_QUEUEING_PRIO_SHIFT;
3441                 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >>
3442                         E1000_FHFT_QUEUEING_QUEUE_SHIFT;
3443                 return 0;
3444         }
3445         return -ENOENT;
3446 }
3447
3448 static int
3449 eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
3450                         enum rte_filter_op filter_op,
3451                         void *arg)
3452 {
3453         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3454         struct rte_eth_flex_filter *filter;
3455         int ret = 0;
3456
3457         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
3458
3459         if (filter_op == RTE_ETH_FILTER_NOP)
3460                 return ret;
3461
3462         if (arg == NULL) {
3463                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
3464                             filter_op);
3465                 return -EINVAL;
3466         }
3467
3468         filter = (struct rte_eth_flex_filter *)arg;
3469         if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN
3470             || filter->len % sizeof(uint64_t) != 0) {
3471                 PMD_DRV_LOG(ERR, "filter's length is out of range");
3472                 return -EINVAL;
3473         }
3474         if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) {
3475                 PMD_DRV_LOG(ERR, "filter's priority is out of range");
3476                 return -EINVAL;
3477         }
3478
3479         switch (filter_op) {
3480         case RTE_ETH_FILTER_ADD:
3481                 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE);
3482                 break;
3483         case RTE_ETH_FILTER_DELETE:
3484                 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE);
3485                 break;
3486         case RTE_ETH_FILTER_GET:
3487                 ret = eth_igb_get_flex_filter(dev, filter);
3488                 break;
3489         default:
3490                 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op);
3491                 ret = -EINVAL;
3492                 break;
3493         }
3494
3495         return ret;
3496 }
3497
3498 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/
3499 static inline int
3500 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter,
3501                         struct e1000_5tuple_filter_info *filter_info)
3502 {
3503         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576)
3504                 return -EINVAL;
3505         if (filter->priority > E1000_2TUPLE_MAX_PRI)
3506                 return -EINVAL;  /* filter index is out of range. */
3507         if (filter->tcp_flags > TCP_FLAG_ALL)
3508                 return -EINVAL;  /* flags is invalid. */
3509
3510         switch (filter->dst_ip_mask) {
3511         case UINT32_MAX:
3512                 filter_info->dst_ip_mask = 0;
3513                 filter_info->dst_ip = filter->dst_ip;
3514                 break;
3515         case 0:
3516                 filter_info->dst_ip_mask = 1;
3517                 break;
3518         default:
3519                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
3520                 return -EINVAL;
3521         }
3522
3523         switch (filter->src_ip_mask) {
3524         case UINT32_MAX:
3525                 filter_info->src_ip_mask = 0;
3526                 filter_info->src_ip = filter->src_ip;
3527                 break;
3528         case 0:
3529                 filter_info->src_ip_mask = 1;
3530                 break;
3531         default:
3532                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
3533                 return -EINVAL;
3534         }
3535
3536         switch (filter->dst_port_mask) {
3537         case UINT16_MAX:
3538                 filter_info->dst_port_mask = 0;
3539                 filter_info->dst_port = filter->dst_port;
3540                 break;
3541         case 0:
3542                 filter_info->dst_port_mask = 1;
3543                 break;
3544         default:
3545                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
3546                 return -EINVAL;
3547         }
3548
3549         switch (filter->src_port_mask) {
3550         case UINT16_MAX:
3551                 filter_info->src_port_mask = 0;
3552                 filter_info->src_port = filter->src_port;
3553                 break;
3554         case 0:
3555                 filter_info->src_port_mask = 1;
3556                 break;
3557         default:
3558                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
3559                 return -EINVAL;
3560         }
3561
3562         switch (filter->proto_mask) {
3563         case UINT8_MAX:
3564                 filter_info->proto_mask = 0;
3565                 filter_info->proto = filter->proto;
3566                 break;
3567         case 0:
3568                 filter_info->proto_mask = 1;
3569                 break;
3570         default:
3571                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
3572                 return -EINVAL;
3573         }
3574
3575         filter_info->priority = (uint8_t)filter->priority;
3576         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
3577                 filter_info->tcp_flags = filter->tcp_flags;
3578         else
3579                 filter_info->tcp_flags = 0;
3580
3581         return 0;
3582 }
3583
3584 static inline struct e1000_5tuple_filter *
3585 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list,
3586                         struct e1000_5tuple_filter_info *key)
3587 {
3588         struct e1000_5tuple_filter *it;
3589
3590         TAILQ_FOREACH(it, filter_list, entries) {
3591                 if (memcmp(key, &it->filter_info,
3592                         sizeof(struct e1000_5tuple_filter_info)) == 0) {
3593                         return it;
3594                 }
3595         }
3596         return NULL;
3597 }
3598
3599 /*
3600  * igb_add_5tuple_filter_82576 - add a 5tuple filter
3601  *
3602  * @param
3603  * dev: Pointer to struct rte_eth_dev.
3604  * ntuple_filter: ponter to the filter that will be added.
3605  *
3606  * @return
3607  *    - On success, zero.
3608  *    - On failure, a negative value.
3609  */
3610 static int
3611 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
3612                         struct rte_eth_ntuple_filter *ntuple_filter)
3613 {
3614         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3615         struct e1000_filter_info *filter_info =
3616                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3617         struct e1000_5tuple_filter *filter;
3618         uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK;
3619         uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP;
3620         uint8_t i;
3621         int ret;
3622
3623         filter = rte_zmalloc("e1000_5tuple_filter",
3624                         sizeof(struct e1000_5tuple_filter), 0);
3625         if (filter == NULL)
3626                 return -ENOMEM;
3627
3628         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3629                                             &filter->filter_info);
3630         if (ret < 0) {
3631                 rte_free(filter);
3632                 return ret;
3633         }
3634
3635         if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
3636                                          &filter->filter_info) != NULL) {
3637                 PMD_DRV_LOG(ERR, "filter exists.");
3638                 rte_free(filter);
3639                 return -EEXIST;
3640         }
3641         filter->queue = ntuple_filter->queue;
3642
3643         /*
3644          * look for an unused 5tuple filter index,
3645          * and insert the filter to list.
3646          */
3647         for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) {
3648                 if (!(filter_info->fivetuple_mask & (1 << i))) {
3649                         filter_info->fivetuple_mask |= 1 << i;
3650                         filter->index = i;
3651                         TAILQ_INSERT_TAIL(&filter_info->fivetuple_list,
3652                                           filter,
3653                                           entries);
3654                         break;
3655                 }
3656         }
3657         if (i >= E1000_MAX_FTQF_FILTERS) {
3658                 PMD_DRV_LOG(ERR, "5tuple filters are full.");
3659                 rte_free(filter);
3660                 return -ENOSYS;
3661         }
3662
3663         ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK;
3664         if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */
3665                 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP;
3666         if (filter->filter_info.dst_ip_mask == 0)
3667                 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP;
3668         if (filter->filter_info.src_port_mask == 0)
3669                 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
3670         if (filter->filter_info.proto_mask == 0)
3671                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP;
3672         ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) &
3673                 E1000_FTQF_QUEUE_MASK;
3674         ftqf |= E1000_FTQF_QUEUE_ENABLE;
3675         E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf);
3676         E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip);
3677         E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip);
3678
3679         spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT;
3680         E1000_WRITE_REG(hw, E1000_SPQF(i), spqf);
3681
3682         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
3683         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
3684                 imir |= E1000_IMIR_PORT_BP;
3685         else
3686                 imir &= ~E1000_IMIR_PORT_BP;
3687         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
3688
3689         /* tcp flags bits setting. */
3690         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
3691                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
3692                         imir_ext |= E1000_IMIREXT_CTRL_URG;
3693                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
3694                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
3695                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
3696                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
3697                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
3698                         imir_ext |= E1000_IMIREXT_CTRL_RST;
3699                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
3700                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
3701                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
3702                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
3703         } else
3704                 imir_ext |= E1000_IMIREXT_CTRL_BP;
3705         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
3706         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
3707         return 0;
3708 }
3709
3710 /*
3711  * igb_remove_5tuple_filter_82576 - remove a 5tuple filter
3712  *
3713  * @param
3714  * dev: Pointer to struct rte_eth_dev.
3715  * ntuple_filter: ponter to the filter that will be removed.
3716  *
3717  * @return
3718  *    - On success, zero.
3719  *    - On failure, a negative value.
3720  */
3721 static int
3722 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
3723                                 struct rte_eth_ntuple_filter *ntuple_filter)
3724 {
3725         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3726         struct e1000_filter_info *filter_info =
3727                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3728         struct e1000_5tuple_filter_info filter_5tuple;
3729         struct e1000_5tuple_filter *filter;
3730         int ret;
3731
3732         memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info));
3733         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3734                                             &filter_5tuple);
3735         if (ret < 0)
3736                 return ret;
3737
3738         filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
3739                                          &filter_5tuple);
3740         if (filter == NULL) {
3741                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3742                 return -ENOENT;
3743         }
3744
3745         filter_info->fivetuple_mask &= ~(1 << filter->index);
3746         TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries);
3747         rte_free(filter);
3748
3749         E1000_WRITE_REG(hw, E1000_FTQF(filter->index),
3750                         E1000_FTQF_VF_BP | E1000_FTQF_MASK);
3751         E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0);
3752         E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0);
3753         E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0);
3754         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
3755         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
3756         return 0;
3757 }
3758
3759 static int
3760 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
3761 {
3762         uint32_t rctl;
3763         struct e1000_hw *hw;
3764         struct rte_eth_dev_info dev_info;
3765         uint32_t frame_size = mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN +
3766                                      VLAN_TAG_SIZE);
3767
3768         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3769
3770 #ifdef RTE_LIBRTE_82571_SUPPORT
3771         /* XXX: not bigger than max_rx_pktlen */
3772         if (hw->mac.type == e1000_82571)
3773                 return -ENOTSUP;
3774 #endif
3775         eth_igb_infos_get(dev, &dev_info);
3776
3777         /* check that mtu is within the allowed range */
3778         if ((mtu < ETHER_MIN_MTU) ||
3779             (frame_size > dev_info.max_rx_pktlen))
3780                 return -EINVAL;
3781
3782         /* refuse mtu that requires the support of scattered packets when this
3783          * feature has not been enabled before. */
3784         if (!dev->data->scattered_rx &&
3785             frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)
3786                 return -EINVAL;
3787
3788         rctl = E1000_READ_REG(hw, E1000_RCTL);
3789
3790         /* switch to jumbo mode if needed */
3791         if (frame_size > ETHER_MAX_LEN) {
3792                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
3793                 rctl |= E1000_RCTL_LPE;
3794         } else {
3795                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
3796                 rctl &= ~E1000_RCTL_LPE;
3797         }
3798         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3799
3800         /* update max frame size */
3801         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
3802
3803         E1000_WRITE_REG(hw, E1000_RLPML,
3804                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
3805
3806         return 0;
3807 }
3808
3809 /*
3810  * igb_add_del_ntuple_filter - add or delete a ntuple filter
3811  *
3812  * @param
3813  * dev: Pointer to struct rte_eth_dev.
3814  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
3815  * add: if true, add filter, if false, remove filter
3816  *
3817  * @return
3818  *    - On success, zero.
3819  *    - On failure, a negative value.
3820  */
3821 static int
3822 igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
3823                         struct rte_eth_ntuple_filter *ntuple_filter,
3824                         bool add)
3825 {
3826         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3827         int ret;
3828
3829         switch (ntuple_filter->flags) {
3830         case RTE_5TUPLE_FLAGS:
3831         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3832                 if (hw->mac.type != e1000_82576)
3833                         return -ENOTSUP;
3834                 if (add)
3835                         ret = igb_add_5tuple_filter_82576(dev,
3836                                                           ntuple_filter);
3837                 else
3838                         ret = igb_remove_5tuple_filter_82576(dev,
3839                                                              ntuple_filter);
3840                 break;
3841         case RTE_2TUPLE_FLAGS:
3842         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3843                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
3844                         return -ENOTSUP;
3845                 if (add)
3846                         ret = igb_add_2tuple_filter(dev, ntuple_filter);
3847                 else
3848                         ret = igb_remove_2tuple_filter(dev, ntuple_filter);
3849                 break;
3850         default:
3851                 ret = -EINVAL;
3852                 break;
3853         }
3854
3855         return ret;
3856 }
3857
3858 /*
3859  * igb_get_ntuple_filter - get a ntuple filter
3860  *
3861  * @param
3862  * dev: Pointer to struct rte_eth_dev.
3863  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
3864  *
3865  * @return
3866  *    - On success, zero.
3867  *    - On failure, a negative value.
3868  */
3869 static int
3870 igb_get_ntuple_filter(struct rte_eth_dev *dev,
3871                         struct rte_eth_ntuple_filter *ntuple_filter)
3872 {
3873         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3874         struct e1000_filter_info *filter_info =
3875                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3876         struct e1000_5tuple_filter_info filter_5tuple;
3877         struct e1000_2tuple_filter_info filter_2tuple;
3878         struct e1000_5tuple_filter *p_5tuple_filter;
3879         struct e1000_2tuple_filter *p_2tuple_filter;
3880         int ret;
3881
3882         switch (ntuple_filter->flags) {
3883         case RTE_5TUPLE_FLAGS:
3884         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3885                 if (hw->mac.type != e1000_82576)
3886                         return -ENOTSUP;
3887                 memset(&filter_5tuple,
3888                         0,
3889                         sizeof(struct e1000_5tuple_filter_info));
3890                 ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3891                                                     &filter_5tuple);
3892                 if (ret < 0)
3893                         return ret;
3894                 p_5tuple_filter = igb_5tuple_filter_lookup_82576(
3895                                         &filter_info->fivetuple_list,
3896                                         &filter_5tuple);
3897                 if (p_5tuple_filter == NULL) {
3898                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3899                         return -ENOENT;
3900                 }
3901                 ntuple_filter->queue = p_5tuple_filter->queue;
3902                 break;
3903         case RTE_2TUPLE_FLAGS:
3904         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3905                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
3906                         return -ENOTSUP;
3907                 memset(&filter_2tuple,
3908                         0,
3909                         sizeof(struct e1000_2tuple_filter_info));
3910                 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple);
3911                 if (ret < 0)
3912                         return ret;
3913                 p_2tuple_filter = igb_2tuple_filter_lookup(
3914                                         &filter_info->twotuple_list,
3915                                         &filter_2tuple);
3916                 if (p_2tuple_filter == NULL) {
3917                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3918                         return -ENOENT;
3919                 }
3920                 ntuple_filter->queue = p_2tuple_filter->queue;
3921                 break;
3922         default:
3923                 ret = -EINVAL;
3924                 break;
3925         }
3926
3927         return 0;
3928 }
3929
3930 /*
3931  * igb_ntuple_filter_handle - Handle operations for ntuple filter.
3932  * @dev: pointer to rte_eth_dev structure
3933  * @filter_op:operation will be taken.
3934  * @arg: a pointer to specific structure corresponding to the filter_op
3935  */
3936 static int
3937 igb_ntuple_filter_handle(struct rte_eth_dev *dev,
3938                                 enum rte_filter_op filter_op,
3939                                 void *arg)
3940 {
3941         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3942         int ret;
3943
3944         MAC_TYPE_FILTER_SUP(hw->mac.type);
3945
3946         if (filter_op == RTE_ETH_FILTER_NOP)
3947                 return 0;
3948
3949         if (arg == NULL) {
3950                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
3951                             filter_op);
3952                 return -EINVAL;
3953         }
3954
3955         switch (filter_op) {
3956         case RTE_ETH_FILTER_ADD:
3957                 ret = igb_add_del_ntuple_filter(dev,
3958                         (struct rte_eth_ntuple_filter *)arg,
3959                         TRUE);
3960                 break;
3961         case RTE_ETH_FILTER_DELETE:
3962                 ret = igb_add_del_ntuple_filter(dev,
3963                         (struct rte_eth_ntuple_filter *)arg,
3964                         FALSE);
3965                 break;
3966         case RTE_ETH_FILTER_GET:
3967                 ret = igb_get_ntuple_filter(dev,
3968                         (struct rte_eth_ntuple_filter *)arg);
3969                 break;
3970         default:
3971                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
3972                 ret = -EINVAL;
3973                 break;
3974         }
3975         return ret;
3976 }
3977
3978 static inline int
3979 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info,
3980                         uint16_t ethertype)
3981 {
3982         int i;
3983
3984         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
3985                 if (filter_info->ethertype_filters[i] == ethertype &&
3986                     (filter_info->ethertype_mask & (1 << i)))
3987                         return i;
3988         }
3989         return -1;
3990 }
3991
3992 static inline int
3993 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info,
3994                         uint16_t ethertype)
3995 {
3996         int i;
3997
3998         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
3999                 if (!(filter_info->ethertype_mask & (1 << i))) {
4000                         filter_info->ethertype_mask |= 1 << i;
4001                         filter_info->ethertype_filters[i] = ethertype;
4002                         return i;
4003                 }
4004         }
4005         return -1;
4006 }
4007
4008 static inline int
4009 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info,
4010                         uint8_t idx)
4011 {
4012         if (idx >= E1000_MAX_ETQF_FILTERS)
4013                 return -1;
4014         filter_info->ethertype_mask &= ~(1 << idx);
4015         filter_info->ethertype_filters[idx] = 0;
4016         return idx;
4017 }
4018
4019
4020 static int
4021 igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
4022                         struct rte_eth_ethertype_filter *filter,
4023                         bool add)
4024 {
4025         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4026         struct e1000_filter_info *filter_info =
4027                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4028         uint32_t etqf = 0;
4029         int ret;
4030
4031         if (filter->ether_type == ETHER_TYPE_IPv4 ||
4032                 filter->ether_type == ETHER_TYPE_IPv6) {
4033                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
4034                         " ethertype filter.", filter->ether_type);
4035                 return -EINVAL;
4036         }
4037
4038         if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) {
4039                 PMD_DRV_LOG(ERR, "mac compare is unsupported.");
4040                 return -EINVAL;
4041         }
4042         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) {
4043                 PMD_DRV_LOG(ERR, "drop option is unsupported.");
4044                 return -EINVAL;
4045         }
4046
4047         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
4048         if (ret >= 0 && add) {
4049                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.",
4050                             filter->ether_type);
4051                 return -EEXIST;
4052         }
4053         if (ret < 0 && !add) {
4054                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
4055                             filter->ether_type);
4056                 return -ENOENT;
4057         }
4058
4059         if (add) {
4060                 ret = igb_ethertype_filter_insert(filter_info,
4061                         filter->ether_type);
4062                 if (ret < 0) {
4063                         PMD_DRV_LOG(ERR, "ethertype filters are full.");
4064                         return -ENOSYS;
4065                 }
4066
4067                 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE;
4068                 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE);
4069                 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT;
4070         } else {
4071                 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret);
4072                 if (ret < 0)
4073                         return -ENOSYS;
4074         }
4075         E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf);
4076         E1000_WRITE_FLUSH(hw);
4077
4078         return 0;
4079 }
4080
4081 static int
4082 igb_get_ethertype_filter(struct rte_eth_dev *dev,
4083                         struct rte_eth_ethertype_filter *filter)
4084 {
4085         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4086         struct e1000_filter_info *filter_info =
4087                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4088         uint32_t etqf;
4089         int ret;
4090
4091         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
4092         if (ret < 0) {
4093                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
4094                             filter->ether_type);
4095                 return -ENOENT;
4096         }
4097
4098         etqf = E1000_READ_REG(hw, E1000_ETQF(ret));
4099         if (etqf & E1000_ETQF_FILTER_ENABLE) {
4100                 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE;
4101                 filter->flags = 0;
4102                 filter->queue = (etqf & E1000_ETQF_QUEUE) >>
4103                                 E1000_ETQF_QUEUE_SHIFT;
4104                 return 0;
4105         }
4106
4107         return -ENOENT;
4108 }
4109
4110 /*
4111  * igb_ethertype_filter_handle - Handle operations for ethertype filter.
4112  * @dev: pointer to rte_eth_dev structure
4113  * @filter_op:operation will be taken.
4114  * @arg: a pointer to specific structure corresponding to the filter_op
4115  */
4116 static int
4117 igb_ethertype_filter_handle(struct rte_eth_dev *dev,
4118                                 enum rte_filter_op filter_op,
4119                                 void *arg)
4120 {
4121         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4122         int ret;
4123
4124         MAC_TYPE_FILTER_SUP(hw->mac.type);
4125
4126         if (filter_op == RTE_ETH_FILTER_NOP)
4127                 return 0;
4128
4129         if (arg == NULL) {
4130                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
4131                             filter_op);
4132                 return -EINVAL;
4133         }
4134
4135         switch (filter_op) {
4136         case RTE_ETH_FILTER_ADD:
4137                 ret = igb_add_del_ethertype_filter(dev,
4138                         (struct rte_eth_ethertype_filter *)arg,
4139                         TRUE);
4140                 break;
4141         case RTE_ETH_FILTER_DELETE:
4142                 ret = igb_add_del_ethertype_filter(dev,
4143                         (struct rte_eth_ethertype_filter *)arg,
4144                         FALSE);
4145                 break;
4146         case RTE_ETH_FILTER_GET:
4147                 ret = igb_get_ethertype_filter(dev,
4148                         (struct rte_eth_ethertype_filter *)arg);
4149                 break;
4150         default:
4151                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
4152                 ret = -EINVAL;
4153                 break;
4154         }
4155         return ret;
4156 }
4157
4158 static int
4159 eth_igb_filter_ctrl(struct rte_eth_dev *dev,
4160                      enum rte_filter_type filter_type,
4161                      enum rte_filter_op filter_op,
4162                      void *arg)
4163 {
4164         int ret = -EINVAL;
4165
4166         switch (filter_type) {
4167         case RTE_ETH_FILTER_NTUPLE:
4168                 ret = igb_ntuple_filter_handle(dev, filter_op, arg);
4169                 break;
4170         case RTE_ETH_FILTER_ETHERTYPE:
4171                 ret = igb_ethertype_filter_handle(dev, filter_op, arg);
4172                 break;
4173         case RTE_ETH_FILTER_SYN:
4174                 ret = eth_igb_syn_filter_handle(dev, filter_op, arg);
4175                 break;
4176         case RTE_ETH_FILTER_FLEXIBLE:
4177                 ret = eth_igb_flex_filter_handle(dev, filter_op, arg);
4178                 break;
4179         default:
4180                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
4181                                                         filter_type);
4182                 break;
4183         }
4184
4185         return ret;
4186 }
4187
4188 static int
4189 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
4190                          struct ether_addr *mc_addr_set,
4191                          uint32_t nb_mc_addr)
4192 {
4193         struct e1000_hw *hw;
4194
4195         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4196         e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr);
4197         return 0;
4198 }
4199
4200 static uint64_t
4201 igb_read_systime_cyclecounter(struct rte_eth_dev *dev)
4202 {
4203         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4204         uint64_t systime_cycles;
4205
4206         switch (hw->mac.type) {
4207         case e1000_i210:
4208         case e1000_i211:
4209                 /*
4210                  * Need to read System Time Residue Register to be able
4211                  * to read the other two registers.
4212                  */
4213                 E1000_READ_REG(hw, E1000_SYSTIMR);
4214                 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */
4215                 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML);
4216                 systime_cycles += (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH)
4217                                 * NSEC_PER_SEC;
4218                 break;
4219         case e1000_82580:
4220         case e1000_i350:
4221         case e1000_i354:
4222                 /*
4223                  * Need to read System Time Residue Register to be able
4224                  * to read the other two registers.
4225                  */
4226                 E1000_READ_REG(hw, E1000_SYSTIMR);
4227                 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML);
4228                 /* Only the 8 LSB are valid. */
4229                 systime_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_SYSTIMH)
4230                                 & 0xff) << 32;
4231                 break;
4232         default:
4233                 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML);
4234                 systime_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH)
4235                                 << 32;
4236                 break;
4237         }
4238
4239         return systime_cycles;
4240 }
4241
4242 static uint64_t
4243 igb_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev)
4244 {
4245         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4246         uint64_t rx_tstamp_cycles;
4247
4248         switch (hw->mac.type) {
4249         case e1000_i210:
4250         case e1000_i211:
4251                 /* RXSTMPL stores ns and RXSTMPH stores seconds. */
4252                 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL);
4253                 rx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH)
4254                                 * NSEC_PER_SEC;
4255                 break;
4256         case e1000_82580:
4257         case e1000_i350:
4258         case e1000_i354:
4259                 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL);
4260                 /* Only the 8 LSB are valid. */
4261                 rx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_RXSTMPH)
4262                                 & 0xff) << 32;
4263                 break;
4264         default:
4265                 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL);
4266                 rx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH)
4267                                 << 32;
4268                 break;
4269         }
4270
4271         return rx_tstamp_cycles;
4272 }
4273
4274 static uint64_t
4275 igb_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev)
4276 {
4277         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4278         uint64_t tx_tstamp_cycles;
4279
4280         switch (hw->mac.type) {
4281         case e1000_i210:
4282         case e1000_i211:
4283                 /* RXSTMPL stores ns and RXSTMPH stores seconds. */
4284                 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL);
4285                 tx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH)
4286                                 * NSEC_PER_SEC;
4287                 break;
4288         case e1000_82580:
4289         case e1000_i350:
4290         case e1000_i354:
4291                 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL);
4292                 /* Only the 8 LSB are valid. */
4293                 tx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_TXSTMPH)
4294                                 & 0xff) << 32;
4295                 break;
4296         default:
4297                 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL);
4298                 tx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH)
4299                                 << 32;
4300                 break;
4301         }
4302
4303         return tx_tstamp_cycles;
4304 }
4305
4306 static void
4307 igb_start_timecounters(struct rte_eth_dev *dev)
4308 {
4309         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4310         struct e1000_adapter *adapter =
4311                 (struct e1000_adapter *)dev->data->dev_private;
4312         uint32_t incval = 1;
4313         uint32_t shift = 0;
4314         uint64_t mask = E1000_CYCLECOUNTER_MASK;
4315
4316         switch (hw->mac.type) {
4317         case e1000_82580:
4318         case e1000_i350:
4319         case e1000_i354:
4320                 /* 32 LSB bits + 8 MSB bits = 40 bits */
4321                 mask = (1ULL << 40) - 1;
4322                 /* fall-through */
4323         case e1000_i210:
4324         case e1000_i211:
4325                 /*
4326                  * Start incrementing the register
4327                  * used to timestamp PTP packets.
4328                  */
4329                 E1000_WRITE_REG(hw, E1000_TIMINCA, incval);
4330                 break;
4331         case e1000_82576:
4332                 incval = E1000_INCVALUE_82576;
4333                 shift = IGB_82576_TSYNC_SHIFT;
4334                 E1000_WRITE_REG(hw, E1000_TIMINCA,
4335                                 E1000_INCPERIOD_82576 | incval);
4336                 break;
4337         default:
4338                 /* Not supported */
4339                 return;
4340         }
4341
4342         memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter));
4343         memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
4344         memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
4345
4346         adapter->systime_tc.cc_mask = mask;
4347         adapter->systime_tc.cc_shift = shift;
4348         adapter->systime_tc.nsec_mask = (1ULL << shift) - 1;
4349
4350         adapter->rx_tstamp_tc.cc_mask = mask;
4351         adapter->rx_tstamp_tc.cc_shift = shift;
4352         adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
4353
4354         adapter->tx_tstamp_tc.cc_mask = mask;
4355         adapter->tx_tstamp_tc.cc_shift = shift;
4356         adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
4357 }
4358
4359 static int
4360 igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
4361 {
4362         struct e1000_adapter *adapter =
4363                         (struct e1000_adapter *)dev->data->dev_private;
4364
4365         adapter->systime_tc.nsec += delta;
4366         adapter->rx_tstamp_tc.nsec += delta;
4367         adapter->tx_tstamp_tc.nsec += delta;
4368
4369         return 0;
4370 }
4371
4372 static int
4373 igb_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
4374 {
4375         uint64_t ns;
4376         struct e1000_adapter *adapter =
4377                         (struct e1000_adapter *)dev->data->dev_private;
4378
4379         ns = rte_timespec_to_ns(ts);
4380
4381         /* Set the timecounters to a new value. */
4382         adapter->systime_tc.nsec = ns;
4383         adapter->rx_tstamp_tc.nsec = ns;
4384         adapter->tx_tstamp_tc.nsec = ns;
4385
4386         return 0;
4387 }
4388
4389 static int
4390 igb_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
4391 {
4392         uint64_t ns, systime_cycles;
4393         struct e1000_adapter *adapter =
4394                         (struct e1000_adapter *)dev->data->dev_private;
4395
4396         systime_cycles = igb_read_systime_cyclecounter(dev);
4397         ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles);
4398         *ts = rte_ns_to_timespec(ns);
4399
4400         return 0;
4401 }
4402
4403 static int
4404 igb_timesync_enable(struct rte_eth_dev *dev)
4405 {
4406         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4407         uint32_t tsync_ctl;
4408         uint32_t tsauxc;
4409
4410         /* Stop the timesync system time. */
4411         E1000_WRITE_REG(hw, E1000_TIMINCA, 0x0);
4412         /* Reset the timesync system time value. */
4413         switch (hw->mac.type) {
4414         case e1000_82580:
4415         case e1000_i350:
4416         case e1000_i354:
4417         case e1000_i210:
4418         case e1000_i211:
4419                 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x0);
4420                 /* fall-through */
4421         case e1000_82576:
4422                 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x0);
4423                 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x0);
4424                 break;
4425         default:
4426                 /* Not supported. */
4427                 return -ENOTSUP;
4428         }
4429
4430         /* Enable system time for it isn't on by default. */
4431         tsauxc = E1000_READ_REG(hw, E1000_TSAUXC);
4432         tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME;
4433         E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc);
4434
4435         igb_start_timecounters(dev);
4436
4437         /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
4438         E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588),
4439                         (ETHER_TYPE_1588 |
4440                          E1000_ETQF_FILTER_ENABLE |
4441                          E1000_ETQF_1588));
4442
4443         /* Enable timestamping of received PTP packets. */
4444         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
4445         tsync_ctl |= E1000_TSYNCRXCTL_ENABLED;
4446         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl);
4447
4448         /* Enable Timestamping of transmitted PTP packets. */
4449         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
4450         tsync_ctl |= E1000_TSYNCTXCTL_ENABLED;
4451         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl);
4452
4453         return 0;
4454 }
4455
4456 static int
4457 igb_timesync_disable(struct rte_eth_dev *dev)
4458 {
4459         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4460         uint32_t tsync_ctl;
4461
4462         /* Disable timestamping of transmitted PTP packets. */
4463         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
4464         tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED;
4465         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl);
4466
4467         /* Disable timestamping of received PTP packets. */
4468         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
4469         tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED;
4470         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl);
4471
4472         /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
4473         E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0);
4474
4475         /* Stop incrementating the System Time registers. */
4476         E1000_WRITE_REG(hw, E1000_TIMINCA, 0);
4477
4478         return 0;
4479 }
4480
4481 static int
4482 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
4483                                struct timespec *timestamp,
4484                                uint32_t flags __rte_unused)
4485 {
4486         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4487         struct e1000_adapter *adapter =
4488                         (struct e1000_adapter *)dev->data->dev_private;
4489         uint32_t tsync_rxctl;
4490         uint64_t rx_tstamp_cycles;
4491         uint64_t ns;
4492
4493         tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
4494         if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0)
4495                 return -EINVAL;
4496
4497         rx_tstamp_cycles = igb_read_rx_tstamp_cyclecounter(dev);
4498         ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles);
4499         *timestamp = rte_ns_to_timespec(ns);
4500
4501         return  0;
4502 }
4503
4504 static int
4505 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
4506                                struct timespec *timestamp)
4507 {
4508         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4509         struct e1000_adapter *adapter =
4510                         (struct e1000_adapter *)dev->data->dev_private;
4511         uint32_t tsync_txctl;
4512         uint64_t tx_tstamp_cycles;
4513         uint64_t ns;
4514
4515         tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
4516         if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0)
4517                 return -EINVAL;
4518
4519         tx_tstamp_cycles = igb_read_tx_tstamp_cyclecounter(dev);
4520         ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles);
4521         *timestamp = rte_ns_to_timespec(ns);
4522
4523         return  0;
4524 }
4525
4526 static int
4527 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused)
4528 {
4529         int count = 0;
4530         int g_ind = 0;
4531         const struct reg_info *reg_group;
4532
4533         while ((reg_group = igb_regs[g_ind++]))
4534                 count += igb_reg_group_count(reg_group);
4535
4536         return count;
4537 }
4538
4539 static int
4540 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused)
4541 {
4542         int count = 0;
4543         int g_ind = 0;
4544         const struct reg_info *reg_group;
4545
4546         while ((reg_group = igbvf_regs[g_ind++]))
4547                 count += igb_reg_group_count(reg_group);
4548
4549         return count;
4550 }
4551
4552 static int
4553 eth_igb_get_regs(struct rte_eth_dev *dev,
4554         struct rte_dev_reg_info *regs)
4555 {
4556         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4557         uint32_t *data = regs->data;
4558         int g_ind = 0;
4559         int count = 0;
4560         const struct reg_info *reg_group;
4561
4562         /* Support only full register dump */
4563         if ((regs->length == 0) ||
4564             (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) {
4565                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4566                         hw->device_id;
4567                 while ((reg_group = igb_regs[g_ind++]))
4568                         count += igb_read_regs_group(dev, &data[count],
4569                                                         reg_group);
4570                 return 0;
4571         }
4572
4573         return -ENOTSUP;
4574 }
4575
4576 static int
4577 igbvf_get_regs(struct rte_eth_dev *dev,
4578         struct rte_dev_reg_info *regs)
4579 {
4580         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4581         uint32_t *data = regs->data;
4582         int g_ind = 0;
4583         int count = 0;
4584         const struct reg_info *reg_group;
4585
4586         /* Support only full register dump */
4587         if ((regs->length == 0) ||
4588             (regs->length == (uint32_t)igbvf_get_reg_length(dev))) {
4589                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4590                         hw->device_id;
4591                 while ((reg_group = igbvf_regs[g_ind++]))
4592                         count += igb_read_regs_group(dev, &data[count],
4593                                                         reg_group);
4594                 return 0;
4595         }
4596
4597         return -ENOTSUP;
4598 }
4599
4600 static int
4601 eth_igb_get_eeprom_length(struct rte_eth_dev *dev)
4602 {
4603         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4604
4605         /* Return unit is byte count */
4606         return hw->nvm.word_size * 2;
4607 }
4608
4609 static int
4610 eth_igb_get_eeprom(struct rte_eth_dev *dev,
4611         struct rte_dev_eeprom_info *in_eeprom)
4612 {
4613         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4614         struct e1000_nvm_info *nvm = &hw->nvm;
4615         uint16_t *data = in_eeprom->data;
4616         int first, length;
4617
4618         first = in_eeprom->offset >> 1;
4619         length = in_eeprom->length >> 1;
4620         if ((first >= hw->nvm.word_size) ||
4621             ((first + length) >= hw->nvm.word_size))
4622                 return -EINVAL;
4623
4624         in_eeprom->magic = hw->vendor_id |
4625                 ((uint32_t)hw->device_id << 16);
4626
4627         if ((nvm->ops.read) == NULL)
4628                 return -ENOTSUP;
4629
4630         return nvm->ops.read(hw, first, length, data);
4631 }
4632
4633 static int
4634 eth_igb_set_eeprom(struct rte_eth_dev *dev,
4635         struct rte_dev_eeprom_info *in_eeprom)
4636 {
4637         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4638         struct e1000_nvm_info *nvm = &hw->nvm;
4639         uint16_t *data = in_eeprom->data;
4640         int first, length;
4641
4642         first = in_eeprom->offset >> 1;
4643         length = in_eeprom->length >> 1;
4644         if ((first >= hw->nvm.word_size) ||
4645             ((first + length) >= hw->nvm.word_size))
4646                 return -EINVAL;
4647
4648         in_eeprom->magic = (uint32_t)hw->vendor_id |
4649                 ((uint32_t)hw->device_id << 16);
4650
4651         if ((nvm->ops.write) == NULL)
4652                 return -ENOTSUP;
4653         return nvm->ops.write(hw,  first, length, data);
4654 }
4655
4656 static struct rte_driver pmd_igb_drv = {
4657         .type = PMD_PDEV,
4658         .init = rte_igb_pmd_init,
4659 };
4660
4661 static struct rte_driver pmd_igbvf_drv = {
4662         .type = PMD_PDEV,
4663         .init = rte_igbvf_pmd_init,
4664 };
4665
4666 static int
4667 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
4668 {
4669         struct e1000_hw *hw =
4670                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4671         uint32_t mask = 1 << queue_id;
4672
4673         E1000_WRITE_REG(hw, E1000_EIMC, mask);
4674         E1000_WRITE_FLUSH(hw);
4675
4676         return 0;
4677 }
4678
4679 static int
4680 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
4681 {
4682         struct e1000_hw *hw =
4683                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4684         uint32_t mask = 1 << queue_id;
4685         uint32_t regval;
4686
4687         regval = E1000_READ_REG(hw, E1000_EIMS);
4688         E1000_WRITE_REG(hw, E1000_EIMS, regval | mask);
4689         E1000_WRITE_FLUSH(hw);
4690
4691         rte_intr_enable(&dev->pci_dev->intr_handle);
4692
4693         return 0;
4694 }
4695
4696 static void
4697 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t  msix_vector,
4698                    uint8_t index, uint8_t offset)
4699 {
4700         uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4701
4702         /* clear bits */
4703         val &= ~((uint32_t)0xFF << offset);
4704
4705         /* write vector and valid bit */
4706         val |= (msix_vector | E1000_IVAR_VALID) << offset;
4707
4708         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val);
4709 }
4710
4711 static void
4712 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction,
4713                            uint8_t queue, uint8_t msix_vector)
4714 {
4715         uint32_t tmp = 0;
4716
4717         if (hw->mac.type == e1000_82575) {
4718                 if (direction == 0)
4719                         tmp = E1000_EICR_RX_QUEUE0 << queue;
4720                 else if (direction == 1)
4721                         tmp = E1000_EICR_TX_QUEUE0 << queue;
4722                 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp);
4723         } else if (hw->mac.type == e1000_82576) {
4724                 if ((direction == 0) || (direction == 1))
4725                         eth_igb_write_ivar(hw, msix_vector, queue & 0x7,
4726                                            ((queue & 0x8) << 1) +
4727                                            8 * direction);
4728         } else if ((hw->mac.type == e1000_82580) ||
4729                         (hw->mac.type == e1000_i350) ||
4730                         (hw->mac.type == e1000_i354) ||
4731                         (hw->mac.type == e1000_i210) ||
4732                         (hw->mac.type == e1000_i211)) {
4733                 if ((direction == 0) || (direction == 1))
4734                         eth_igb_write_ivar(hw, msix_vector,
4735                                            queue >> 1,
4736                                            ((queue & 0x1) << 4) +
4737                                            8 * direction);
4738         }
4739 }
4740
4741 /* Sets up the hardware to generate MSI-X interrupts properly
4742  * @hw
4743  *  board private structure
4744  */
4745 static void
4746 eth_igb_configure_msix_intr(struct rte_eth_dev *dev)
4747 {
4748         int queue_id;
4749         uint32_t tmpval, regval, intr_mask;
4750         struct e1000_hw *hw =
4751                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4752         uint32_t vec = E1000_MISC_VEC_ID;
4753         uint32_t base = E1000_MISC_VEC_ID;
4754         uint32_t misc_shift = 0;
4755
4756         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
4757
4758         /* won't configure msix register if no mapping is done
4759          * between intr vector and event fd
4760          */
4761         if (!rte_intr_dp_is_en(intr_handle))
4762                 return;
4763
4764         if (rte_intr_allow_others(intr_handle)) {
4765                 vec = base = E1000_RX_VEC_START;
4766                 misc_shift = 1;
4767         }
4768
4769         /* set interrupt vector for other causes */
4770         if (hw->mac.type == e1000_82575) {
4771                 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT);
4772                 /* enable MSI-X PBA support */
4773                 tmpval |= E1000_CTRL_EXT_PBA_CLR;
4774
4775                 /* Auto-Mask interrupts upon ICR read */
4776                 tmpval |= E1000_CTRL_EXT_EIAME;
4777                 tmpval |= E1000_CTRL_EXT_IRCA;
4778
4779                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval);
4780
4781                 /* enable msix_other interrupt */
4782                 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER);
4783                 regval = E1000_READ_REG(hw, E1000_EIAC);
4784                 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER);
4785                 regval = E1000_READ_REG(hw, E1000_EIAM);
4786                 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER);
4787         } else if ((hw->mac.type == e1000_82576) ||
4788                         (hw->mac.type == e1000_82580) ||
4789                         (hw->mac.type == e1000_i350) ||
4790                         (hw->mac.type == e1000_i354) ||
4791                         (hw->mac.type == e1000_i210) ||
4792                         (hw->mac.type == e1000_i211)) {
4793                 /* turn on MSI-X capability first */
4794                 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE |
4795                                         E1000_GPIE_PBA | E1000_GPIE_EIAME |
4796                                         E1000_GPIE_NSICR);
4797                 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) <<
4798                         misc_shift;
4799                 regval = E1000_READ_REG(hw, E1000_EIAC);
4800                 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask);
4801
4802                 /* enable msix_other interrupt */
4803                 regval = E1000_READ_REG(hw, E1000_EIMS);
4804                 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask);
4805                 tmpval = (dev->data->nb_rx_queues | E1000_IVAR_VALID) << 8;
4806                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval);
4807         }
4808
4809         /* use EIAM to auto-mask when MSI-X interrupt
4810          * is asserted, this saves a register write for every interrupt
4811          */
4812         intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) <<
4813                 misc_shift;
4814         regval = E1000_READ_REG(hw, E1000_EIAM);
4815         E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask);
4816
4817         for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) {
4818                 eth_igb_assign_msix_vector(hw, 0, queue_id, vec);
4819                 intr_handle->intr_vec[queue_id] = vec;
4820                 if (vec < base + intr_handle->nb_efd - 1)
4821                         vec++;
4822         }
4823
4824         E1000_WRITE_FLUSH(hw);
4825 }
4826
4827 PMD_REGISTER_DRIVER(pmd_igb_drv);
4828 PMD_REGISTER_DRIVER(pmd_igbvf_drv);