drivers/net/e1000/igb_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <stdarg.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_interrupts.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_log.h>
  44 #include <rte_debug.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_memory.h>
  49 #include <rte_memzone.h>
  50 #include <rte_eal.h>
  51 #include <rte_atomic.h>
  52 #include <rte_malloc.h>
  53 #include <rte_dev.h>
  54
  55 #include "e1000_logs.h"
  56 #include "base/e1000_api.h"
  57 #include "e1000_ethdev.h"
  58 #include "igb_regs.h"
  59
  60 /*
  61  * Default values for port configuration
  62  */
  63 #define IGB_DEFAULT_RX_FREE_THRESH  32
  64
  65 #define IGB_DEFAULT_RX_PTHRESH      ((hw->mac.type == e1000_i354) ? 12 : 8)
  66 #define IGB_DEFAULT_RX_HTHRESH      8
  67 #define IGB_DEFAULT_RX_WTHRESH      ((hw->mac.type == e1000_82576) ? 1 : 4)
  68
  69 #define IGB_DEFAULT_TX_PTHRESH      ((hw->mac.type == e1000_i354) ? 20 : 8)
  70 #define IGB_DEFAULT_TX_HTHRESH      1
  71 #define IGB_DEFAULT_TX_WTHRESH      ((hw->mac.type == e1000_82576) ? 1 : 16)
  72
  73 #define IGB_HKEY_MAX_INDEX 10
  74
  75 /* Bit shift and mask */
  76 #define IGB_4_BIT_WIDTH  (CHAR_BIT / 2)
  77 #define IGB_4_BIT_MASK   RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t)
  78 #define IGB_8_BIT_WIDTH  CHAR_BIT
  79 #define IGB_8_BIT_MASK   UINT8_MAX
  80
  81 /* Additional timesync values. */
  82 #define E1000_CYCLECOUNTER_MASK      0xffffffffffffffffULL
  83 #define E1000_ETQF_FILTER_1588       3
  84 #define IGB_82576_TSYNC_SHIFT        16
  85 #define E1000_INCPERIOD_82576        (1 << E1000_TIMINCA_16NS_SHIFT)
  86 #define E1000_INCVALUE_82576         (16 << IGB_82576_TSYNC_SHIFT)
  87 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000
  88
  89 static int  eth_igb_configure(struct rte_eth_dev *dev);
  90 static int  eth_igb_start(struct rte_eth_dev *dev);
  91 static void eth_igb_stop(struct rte_eth_dev *dev);
  92 static void eth_igb_close(struct rte_eth_dev *dev);
  93 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev);
  94 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev);
  95 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev);
  96 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev);
  97 static int  eth_igb_link_update(struct rte_eth_dev *dev,
  98                                 int wait_to_complete);
  99 static void eth_igb_stats_get(struct rte_eth_dev *dev,
 100                                 struct rte_eth_stats *rte_stats);
 101 static int eth_igb_xstats_get(struct rte_eth_dev *dev,
 102                               struct rte_eth_xstats *xstats, unsigned n);
 103 static int eth_igb_xstats_get_names(struct rte_eth_dev *dev,
 104                                     struct rte_eth_xstat_name *xstats_names,
 105                                     unsigned limit);
 106 static void eth_igb_stats_reset(struct rte_eth_dev *dev);
 107 static void eth_igb_xstats_reset(struct rte_eth_dev *dev);
 108 static void eth_igb_infos_get(struct rte_eth_dev *dev,
 109                               struct rte_eth_dev_info *dev_info);
 110 static const uint32_t *eth_igb_supported_ptypes_get(struct rte_eth_dev *dev);
 111 static void eth_igbvf_infos_get(struct rte_eth_dev *dev,
 112                                 struct rte_eth_dev_info *dev_info);
 113 static int  eth_igb_flow_ctrl_get(struct rte_eth_dev *dev,
 114                                 struct rte_eth_fc_conf *fc_conf);
 115 static int  eth_igb_flow_ctrl_set(struct rte_eth_dev *dev,
 116                                 struct rte_eth_fc_conf *fc_conf);
 117 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev);
 118 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev);
 119 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev);
 120 static int eth_igb_interrupt_action(struct rte_eth_dev *dev);
 121 static void eth_igb_interrupt_handler(struct rte_intr_handle *handle,
 122                                                         void *param);
 123 static int  igb_hardware_init(struct e1000_hw *hw);
 124 static void igb_hw_control_acquire(struct e1000_hw *hw);
 125 static void igb_hw_control_release(struct e1000_hw *hw);
 126 static void igb_init_manageability(struct e1000_hw *hw);
 127 static void igb_release_manageability(struct e1000_hw *hw);
 128
 129 static int  eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 130
 131 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev,
 132                 uint16_t vlan_id, int on);
 133 static int eth_igb_vlan_tpid_set(struct rte_eth_dev *dev,
 134                                  enum rte_vlan_type vlan_type,
 135                                  uint16_t tpid_id);
 136 static void eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 137
 138 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 139 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 140 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev);
 141 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev);
 142 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 143 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 144
 145 static int eth_igb_led_on(struct rte_eth_dev *dev);
 146 static int eth_igb_led_off(struct rte_eth_dev *dev);
 147
 148 static void igb_intr_disable(struct e1000_hw *hw);
 149 static int  igb_get_rx_buffer_size(struct e1000_hw *hw);
 150 static void eth_igb_rar_set(struct rte_eth_dev *dev,
 151                 struct ether_addr *mac_addr,
 152                 uint32_t index, uint32_t pool);
 153 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 154 static void eth_igb_default_mac_addr_set(struct rte_eth_dev *dev,
 155                 struct ether_addr *addr);
 156
 157 static void igbvf_intr_disable(struct e1000_hw *hw);
 158 static int igbvf_dev_configure(struct rte_eth_dev *dev);
 159 static int igbvf_dev_start(struct rte_eth_dev *dev);
 160 static void igbvf_dev_stop(struct rte_eth_dev *dev);
 161 static void igbvf_dev_close(struct rte_eth_dev *dev);
 162 static void igbvf_promiscuous_enable(struct rte_eth_dev *dev);
 163 static void igbvf_promiscuous_disable(struct rte_eth_dev *dev);
 164 static void igbvf_allmulticast_enable(struct rte_eth_dev *dev);
 165 static void igbvf_allmulticast_disable(struct rte_eth_dev *dev);
 166 static int eth_igbvf_link_update(struct e1000_hw *hw);
 167 static void eth_igbvf_stats_get(struct rte_eth_dev *dev,
 168                                 struct rte_eth_stats *rte_stats);
 169 static int eth_igbvf_xstats_get(struct rte_eth_dev *dev,
 170                                 struct rte_eth_xstats *xstats, unsigned n);
 171 static int eth_igbvf_xstats_get_names(struct rte_eth_dev *dev,
 172                                       struct rte_eth_xstat_name *xstats_names,
 173                                       unsigned limit);
 174 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev);
 175 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev,
 176                 uint16_t vlan_id, int on);
 177 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on);
 178 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 179 static void igbvf_default_mac_addr_set(struct rte_eth_dev *dev,
 180                 struct ether_addr *addr);
 181 static int igbvf_get_reg_length(struct rte_eth_dev *dev);
 182 static int igbvf_get_regs(struct rte_eth_dev *dev,
 183                 struct rte_dev_reg_info *regs);
 184
 185 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev,
 186                                    struct rte_eth_rss_reta_entry64 *reta_conf,
 187                                    uint16_t reta_size);
 188 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev,
 189                                   struct rte_eth_rss_reta_entry64 *reta_conf,
 190                                   uint16_t reta_size);
 191
 192 static int eth_igb_syn_filter_set(struct rte_eth_dev *dev,
 193                         struct rte_eth_syn_filter *filter,
 194                         bool add);
 195 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev,
 196                         struct rte_eth_syn_filter *filter);
 197 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
 198                         enum rte_filter_op filter_op,
 199                         void *arg);
 200 static int igb_add_2tuple_filter(struct rte_eth_dev *dev,
 201                         struct rte_eth_ntuple_filter *ntuple_filter);
 202 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev,
 203                         struct rte_eth_ntuple_filter *ntuple_filter);
 204 static int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
 205                         struct rte_eth_flex_filter *filter,
 206                         bool add);
 207 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev,
 208                         struct rte_eth_flex_filter *filter);
 209 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
 210                         enum rte_filter_op filter_op,
 211                         void *arg);
 212 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
 213                         struct rte_eth_ntuple_filter *ntuple_filter);
 214 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
 215                         struct rte_eth_ntuple_filter *ntuple_filter);
 216 static int igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
 217                         struct rte_eth_ntuple_filter *filter,
 218                         bool add);
 219 static int igb_get_ntuple_filter(struct rte_eth_dev *dev,
 220                         struct rte_eth_ntuple_filter *filter);
 221 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev,
 222                                 enum rte_filter_op filter_op,
 223                                 void *arg);
 224 static int igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
 225                         struct rte_eth_ethertype_filter *filter,
 226                         bool add);
 227 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev,
 228                                 enum rte_filter_op filter_op,
 229                                 void *arg);
 230 static int igb_get_ethertype_filter(struct rte_eth_dev *dev,
 231                         struct rte_eth_ethertype_filter *filter);
 232 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev,
 233                      enum rte_filter_type filter_type,
 234                      enum rte_filter_op filter_op,
 235                      void *arg);
 236 static int eth_igb_get_reg_length(struct rte_eth_dev *dev);
 237 static int eth_igb_get_regs(struct rte_eth_dev *dev,
 238                 struct rte_dev_reg_info *regs);
 239 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev);
 240 static int eth_igb_get_eeprom(struct rte_eth_dev *dev,
 241                 struct rte_dev_eeprom_info *eeprom);
 242 static int eth_igb_set_eeprom(struct rte_eth_dev *dev,
 243                 struct rte_dev_eeprom_info *eeprom);
 244 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
 245                                     struct ether_addr *mc_addr_set,
 246                                     uint32_t nb_mc_addr);
 247 static int igb_timesync_enable(struct rte_eth_dev *dev);
 248 static int igb_timesync_disable(struct rte_eth_dev *dev);
 249 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 250                                           struct timespec *timestamp,
 251                                           uint32_t flags);
 252 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 253                                           struct timespec *timestamp);
 254 static int igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta);
 255 static int igb_timesync_read_time(struct rte_eth_dev *dev,
 256                                   struct timespec *timestamp);
 257 static int igb_timesync_write_time(struct rte_eth_dev *dev,
 258                                    const struct timespec *timestamp);
 259 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev,
 260                                         uint16_t queue_id);
 261 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev,
 262                                          uint16_t queue_id);
 263 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction,
 264                                        uint8_t queue, uint8_t msix_vector);
 265 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector,
 266                                uint8_t index, uint8_t offset);
 267 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev);
 268
 269 /*
 270  * Define VF Stats MACRO for Non "cleared on read" register
 271  */
 272 #define UPDATE_VF_STAT(reg, last, cur)            \
 273 {                                                 \
 274         u32 latest = E1000_READ_REG(hw, reg);     \
 275         cur += (latest - last) & UINT_MAX;        \
 276         last = latest;                            \
 277 }
 278
 279 #define IGB_FC_PAUSE_TIME 0x0680
 280 #define IGB_LINK_UPDATE_CHECK_TIMEOUT  90  /* 9s */
 281 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */
 282
 283 #define IGBVF_PMD_NAME "rte_igbvf_pmd"     /* PMD name */
 284
 285 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full;
 286
 287 /*
 288  * The set of PCI devices this driver supports
 289  */
 290 static const struct rte_pci_id pci_id_igb_map[] = {
 291
 292 #define RTE_PCI_DEV_ID_DECL_IGB(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 293 #include "rte_pci_dev_ids.h"
 294
 295 {0},
 296 };
 297
 298 /*
 299  * The set of PCI devices this driver supports (for 82576&I350 VF)
 300  */
 301 static const struct rte_pci_id pci_id_igbvf_map[] = {
 302
 303 #define RTE_PCI_DEV_ID_DECL_IGBVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 304 #include "rte_pci_dev_ids.h"
 305
 306 {0},
 307 };
 308
 309 static const struct rte_eth_desc_lim rx_desc_lim = {
 310         .nb_max = E1000_MAX_RING_DESC,
 311         .nb_min = E1000_MIN_RING_DESC,
 312         .nb_align = IGB_RXD_ALIGN,
 313 };
 314
 315 static const struct rte_eth_desc_lim tx_desc_lim = {
 316         .nb_max = E1000_MAX_RING_DESC,
 317         .nb_min = E1000_MIN_RING_DESC,
 318         .nb_align = IGB_RXD_ALIGN,
 319 };
 320
 321 static const struct eth_dev_ops eth_igb_ops = {
 322         .dev_configure        = eth_igb_configure,
 323         .dev_start            = eth_igb_start,
 324         .dev_stop             = eth_igb_stop,
 325         .dev_close            = eth_igb_close,
 326         .promiscuous_enable   = eth_igb_promiscuous_enable,
 327         .promiscuous_disable  = eth_igb_promiscuous_disable,
 328         .allmulticast_enable  = eth_igb_allmulticast_enable,
 329         .allmulticast_disable = eth_igb_allmulticast_disable,
 330         .link_update          = eth_igb_link_update,
 331         .stats_get            = eth_igb_stats_get,
 332         .xstats_get           = eth_igb_xstats_get,
 333         .xstats_get_names     = eth_igb_xstats_get_names,
 334         .stats_reset          = eth_igb_stats_reset,
 335         .xstats_reset         = eth_igb_xstats_reset,
 336         .dev_infos_get        = eth_igb_infos_get,
 337         .dev_supported_ptypes_get = eth_igb_supported_ptypes_get,
 338         .mtu_set              = eth_igb_mtu_set,
 339         .vlan_filter_set      = eth_igb_vlan_filter_set,
 340         .vlan_tpid_set        = eth_igb_vlan_tpid_set,
 341         .vlan_offload_set     = eth_igb_vlan_offload_set,
 342         .rx_queue_setup       = eth_igb_rx_queue_setup,
 343         .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable,
 344         .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable,
 345         .rx_queue_release     = eth_igb_rx_queue_release,
 346         .rx_queue_count       = eth_igb_rx_queue_count,
 347         .rx_descriptor_done   = eth_igb_rx_descriptor_done,
 348         .tx_queue_setup       = eth_igb_tx_queue_setup,
 349         .tx_queue_release     = eth_igb_tx_queue_release,
 350         .dev_led_on           = eth_igb_led_on,
 351         .dev_led_off          = eth_igb_led_off,
 352         .flow_ctrl_get        = eth_igb_flow_ctrl_get,
 353         .flow_ctrl_set        = eth_igb_flow_ctrl_set,
 354         .mac_addr_add         = eth_igb_rar_set,
 355         .mac_addr_remove      = eth_igb_rar_clear,
 356         .mac_addr_set         = eth_igb_default_mac_addr_set,
 357         .reta_update          = eth_igb_rss_reta_update,
 358         .reta_query           = eth_igb_rss_reta_query,
 359         .rss_hash_update      = eth_igb_rss_hash_update,
 360         .rss_hash_conf_get    = eth_igb_rss_hash_conf_get,
 361         .filter_ctrl          = eth_igb_filter_ctrl,
 362         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 363         .rxq_info_get         = igb_rxq_info_get,
 364         .txq_info_get         = igb_txq_info_get,
 365         .timesync_enable      = igb_timesync_enable,
 366         .timesync_disable     = igb_timesync_disable,
 367         .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp,
 368         .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp,
 369         .get_reg_length       = eth_igb_get_reg_length,
 370         .get_reg              = eth_igb_get_regs,
 371         .get_eeprom_length    = eth_igb_get_eeprom_length,
 372         .get_eeprom           = eth_igb_get_eeprom,
 373         .set_eeprom           = eth_igb_set_eeprom,
 374         .timesync_adjust_time = igb_timesync_adjust_time,
 375         .timesync_read_time   = igb_timesync_read_time,
 376         .timesync_write_time  = igb_timesync_write_time,
 377 };
 378
 379 /*
 380  * dev_ops for virtual function, bare necessities for basic vf
 381  * operation have been implemented
 382  */
 383 static const struct eth_dev_ops igbvf_eth_dev_ops = {
 384         .dev_configure        = igbvf_dev_configure,
 385         .dev_start            = igbvf_dev_start,
 386         .dev_stop             = igbvf_dev_stop,
 387         .dev_close            = igbvf_dev_close,
 388         .promiscuous_enable   = igbvf_promiscuous_enable,
 389         .promiscuous_disable  = igbvf_promiscuous_disable,
 390         .allmulticast_enable  = igbvf_allmulticast_enable,
 391         .allmulticast_disable = igbvf_allmulticast_disable,
 392         .link_update          = eth_igb_link_update,
 393         .stats_get            = eth_igbvf_stats_get,
 394         .xstats_get           = eth_igbvf_xstats_get,
 395         .xstats_get_names     = eth_igbvf_xstats_get_names,
 396         .stats_reset          = eth_igbvf_stats_reset,
 397         .xstats_reset         = eth_igbvf_stats_reset,
 398         .vlan_filter_set      = igbvf_vlan_filter_set,
 399         .dev_infos_get        = eth_igbvf_infos_get,
 400         .dev_supported_ptypes_get = eth_igb_supported_ptypes_get,
 401         .rx_queue_setup       = eth_igb_rx_queue_setup,
 402         .rx_queue_release     = eth_igb_rx_queue_release,
 403         .tx_queue_setup       = eth_igb_tx_queue_setup,
 404         .tx_queue_release     = eth_igb_tx_queue_release,
 405         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 406         .rxq_info_get         = igb_rxq_info_get,
 407         .txq_info_get         = igb_txq_info_get,
 408         .mac_addr_set         = igbvf_default_mac_addr_set,
 409         .get_reg_length       = igbvf_get_reg_length,
 410         .get_reg              = igbvf_get_regs,
 411 };
 412
 413 /* store statistics names and its offset in stats structure */
 414 struct rte_igb_xstats_name_off {
 415         char name[RTE_ETH_XSTATS_NAME_SIZE];
 416         unsigned offset;
 417 };
 418
 419 static const struct rte_igb_xstats_name_off rte_igb_stats_strings[] = {
 420         {"rx_crc_errors", offsetof(struct e1000_hw_stats, crcerrs)},
 421         {"rx_align_errors", offsetof(struct e1000_hw_stats, algnerrc)},
 422         {"rx_symbol_errors", offsetof(struct e1000_hw_stats, symerrs)},
 423         {"rx_missed_packets", offsetof(struct e1000_hw_stats, mpc)},
 424         {"tx_single_collision_packets", offsetof(struct e1000_hw_stats, scc)},
 425         {"tx_multiple_collision_packets", offsetof(struct e1000_hw_stats, mcc)},
 426         {"tx_excessive_collision_packets", offsetof(struct e1000_hw_stats,
 427                 ecol)},
 428         {"tx_late_collisions", offsetof(struct e1000_hw_stats, latecol)},
 429         {"tx_total_collisions", offsetof(struct e1000_hw_stats, colc)},
 430         {"tx_deferred_packets", offsetof(struct e1000_hw_stats, dc)},
 431         {"tx_no_carrier_sense_packets", offsetof(struct e1000_hw_stats, tncrs)},
 432         {"rx_carrier_ext_errors", offsetof(struct e1000_hw_stats, cexterr)},
 433         {"rx_length_errors", offsetof(struct e1000_hw_stats, rlec)},
 434         {"rx_xon_packets", offsetof(struct e1000_hw_stats, xonrxc)},
 435         {"tx_xon_packets", offsetof(struct e1000_hw_stats, xontxc)},
 436         {"rx_xoff_packets", offsetof(struct e1000_hw_stats, xoffrxc)},
 437         {"tx_xoff_packets", offsetof(struct e1000_hw_stats, xofftxc)},
 438         {"rx_flow_control_unsupported_packets", offsetof(struct e1000_hw_stats,
 439                 fcruc)},
 440         {"rx_size_64_packets", offsetof(struct e1000_hw_stats, prc64)},
 441         {"rx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, prc127)},
 442         {"rx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, prc255)},
 443         {"rx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, prc511)},
 444         {"rx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats,
 445                 prc1023)},
 446         {"rx_size_1024_to_max_packets", offsetof(struct e1000_hw_stats,
 447                 prc1522)},
 448         {"rx_broadcast_packets", offsetof(struct e1000_hw_stats, bprc)},
 449         {"rx_multicast_packets", offsetof(struct e1000_hw_stats, mprc)},
 450         {"rx_undersize_errors", offsetof(struct e1000_hw_stats, ruc)},
 451         {"rx_fragment_errors", offsetof(struct e1000_hw_stats, rfc)},
 452         {"rx_oversize_errors", offsetof(struct e1000_hw_stats, roc)},
 453         {"rx_jabber_errors", offsetof(struct e1000_hw_stats, rjc)},
 454         {"rx_management_packets", offsetof(struct e1000_hw_stats, mgprc)},
 455         {"rx_management_dropped", offsetof(struct e1000_hw_stats, mgpdc)},
 456         {"tx_management_packets", offsetof(struct e1000_hw_stats, mgptc)},
 457         {"rx_total_packets", offsetof(struct e1000_hw_stats, tpr)},
 458         {"tx_total_packets", offsetof(struct e1000_hw_stats, tpt)},
 459         {"rx_total_bytes", offsetof(struct e1000_hw_stats, tor)},
 460         {"tx_total_bytes", offsetof(struct e1000_hw_stats, tot)},
 461         {"tx_size_64_packets", offsetof(struct e1000_hw_stats, ptc64)},
 462         {"tx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, ptc127)},
 463         {"tx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, ptc255)},
 464         {"tx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, ptc511)},
 465         {"tx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats,
 466                 ptc1023)},
 467         {"tx_size_1023_to_max_packets", offsetof(struct e1000_hw_stats,
 468                 ptc1522)},
 469         {"tx_multicast_packets", offsetof(struct e1000_hw_stats, mptc)},
 470         {"tx_broadcast_packets", offsetof(struct e1000_hw_stats, bptc)},
 471         {"tx_tso_packets", offsetof(struct e1000_hw_stats, tsctc)},
 472         {"tx_tso_errors", offsetof(struct e1000_hw_stats, tsctfc)},
 473         {"rx_sent_to_host_packets", offsetof(struct e1000_hw_stats, rpthc)},
 474         {"tx_sent_by_host_packets", offsetof(struct e1000_hw_stats, hgptc)},
 475         {"rx_code_violation_packets", offsetof(struct e1000_hw_stats, scvpc)},
 476
 477         {"interrupt_assert_count", offsetof(struct e1000_hw_stats, iac)},
 478 };
 479
 480 #define IGB_NB_XSTATS (sizeof(rte_igb_stats_strings) / \
 481                 sizeof(rte_igb_stats_strings[0]))
 482
 483 static const struct rte_igb_xstats_name_off rte_igbvf_stats_strings[] = {
 484         {"rx_multicast_packets", offsetof(struct e1000_vf_stats, mprc)},
 485         {"rx_good_loopback_packets", offsetof(struct e1000_vf_stats, gprlbc)},
 486         {"tx_good_loopback_packets", offsetof(struct e1000_vf_stats, gptlbc)},
 487         {"rx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gorlbc)},
 488         {"tx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gotlbc)},
 489 };
 490
 491 #define IGBVF_NB_XSTATS (sizeof(rte_igbvf_stats_strings) / \
 492                 sizeof(rte_igbvf_stats_strings[0]))
 493
 494 /**
 495  * Atomically reads the link status information from global
 496  * structure rte_eth_dev.
 497  *
 498  * @param dev
 499  *   - Pointer to the structure rte_eth_dev to read from.
 500  *   - Pointer to the buffer to be saved with the link status.
 501  *
 502  * @return
 503  *   - On success, zero.
 504  *   - On failure, negative value.
 505  */
 506 static inline int
 507 rte_igb_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 508                                 struct rte_eth_link *link)
 509 {
 510         struct rte_eth_link *dst = link;
 511         struct rte_eth_link *src = &(dev->data->dev_link);
 512
 513         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 514                                         *(uint64_t *)src) == 0)
 515                 return -1;
 516
 517         return 0;
 518 }
 519
 520 /**
 521  * Atomically writes the link status information into global
 522  * structure rte_eth_dev.
 523  *
 524  * @param dev
 525  *   - Pointer to the structure rte_eth_dev to read from.
 526  *   - Pointer to the buffer to be saved with the link status.
 527  *
 528  * @return
 529  *   - On success, zero.
 530  *   - On failure, negative value.
 531  */
 532 static inline int
 533 rte_igb_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 534                                 struct rte_eth_link *link)
 535 {
 536         struct rte_eth_link *dst = &(dev->data->dev_link);
 537         struct rte_eth_link *src = link;
 538
 539         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 540                                         *(uint64_t *)src) == 0)
 541                 return -1;
 542
 543         return 0;
 544 }
 545
 546 static inline void
 547 igb_intr_enable(struct rte_eth_dev *dev)
 548 {
 549         struct e1000_interrupt *intr =
 550                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 551         struct e1000_hw *hw =
 552                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 553
 554         E1000_WRITE_REG(hw, E1000_IMS, intr->mask);
 555         E1000_WRITE_FLUSH(hw);
 556 }
 557
 558 static void
 559 igb_intr_disable(struct e1000_hw *hw)
 560 {
 561         E1000_WRITE_REG(hw, E1000_IMC, ~0);
 562         E1000_WRITE_FLUSH(hw);
 563 }
 564
 565 static inline int32_t
 566 igb_pf_reset_hw(struct e1000_hw *hw)
 567 {
 568         uint32_t ctrl_ext;
 569         int32_t status;
 570
 571         status = e1000_reset_hw(hw);
 572
 573         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 574         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 575         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 576         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 577         E1000_WRITE_FLUSH(hw);
 578
 579         return status;
 580 }
 581
 582 static void
 583 igb_identify_hardware(struct rte_eth_dev *dev)
 584 {
 585         struct e1000_hw *hw =
 586                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 587
 588         hw->vendor_id = dev->pci_dev->id.vendor_id;
 589         hw->device_id = dev->pci_dev->id.device_id;
 590         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
 591         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
 592
 593         e1000_set_mac_type(hw);
 594
 595         /* need to check if it is a vf device below */
 596 }
 597
 598 static int
 599 igb_reset_swfw_lock(struct e1000_hw *hw)
 600 {
 601         int ret_val;
 602
 603         /*
 604          * Do mac ops initialization manually here, since we will need
 605          * some function pointers set by this call.
 606          */
 607         ret_val = e1000_init_mac_params(hw);
 608         if (ret_val)
 609                 return ret_val;
 610
 611         /*
 612          * SMBI lock should not fail in this early stage. If this is the case,
 613          * it is due to an improper exit of the application.
 614          * So force the release of the faulty lock.
 615          */
 616         if (e1000_get_hw_semaphore_generic(hw) < 0) {
 617                 PMD_DRV_LOG(DEBUG, "SMBI lock released");
 618         }
 619         e1000_put_hw_semaphore_generic(hw);
 620
 621         if (hw->mac.ops.acquire_swfw_sync != NULL) {
 622                 uint16_t mask;
 623
 624                 /*
 625                  * Phy lock should not fail in this early stage. If this is the case,
 626                  * it is due to an improper exit of the application.
 627                  * So force the release of the faulty lock.
 628                  */
 629                 mask = E1000_SWFW_PHY0_SM << hw->bus.func;
 630                 if (hw->bus.func > E1000_FUNC_1)
 631                         mask <<= 2;
 632                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 633                         PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released",
 634                                     hw->bus.func);
 635                 }
 636                 hw->mac.ops.release_swfw_sync(hw, mask);
 637
 638                 /*
 639                  * This one is more tricky since it is common to all ports; but
 640                  * swfw_sync retries last long enough (1s) to be almost sure that if
 641                  * lock can not be taken it is due to an improper lock of the
 642                  * semaphore.
 643                  */
 644                 mask = E1000_SWFW_EEP_SM;
 645                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 646                         PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 647                 }
 648                 hw->mac.ops.release_swfw_sync(hw, mask);
 649         }
 650
 651         return E1000_SUCCESS;
 652 }
 653
 654 static int
 655 eth_igb_dev_init(struct rte_eth_dev *eth_dev)
 656 {
 657         int error = 0;
 658         struct rte_pci_device *pci_dev;
 659         struct e1000_hw *hw =
 660                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 661         struct e1000_vfta * shadow_vfta =
 662                 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 663         struct e1000_filter_info *filter_info =
 664                 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
 665         struct e1000_adapter *adapter =
 666                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 667
 668         uint32_t ctrl_ext;
 669
 670         pci_dev = eth_dev->pci_dev;
 671
 672         eth_dev->dev_ops = &eth_igb_ops;
 673         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 674         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 675
 676         /* for secondary processes, we don't initialise any further as primary
 677          * has already done this work. Only check we don't need a different
 678          * RX function */
 679         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 680                 if (eth_dev->data->scattered_rx)
 681                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 682                 return 0;
 683         }
 684
 685         rte_eth_copy_pci_info(eth_dev, pci_dev);
 686
 687         hw->hw_addr= (void *)pci_dev->mem_resource[0].addr;
 688
 689         igb_identify_hardware(eth_dev);
 690         if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) {
 691                 error = -EIO;
 692                 goto err_late;
 693         }
 694
 695         e1000_get_bus_info(hw);
 696
 697         /* Reset any pending lock */
 698         if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) {
 699                 error = -EIO;
 700                 goto err_late;
 701         }
 702
 703         /* Finish initialization */
 704         if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) {
 705                 error = -EIO;
 706                 goto err_late;
 707         }
 708
 709         hw->mac.autoneg = 1;
 710         hw->phy.autoneg_wait_to_complete = 0;
 711         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 712
 713         /* Copper options */
 714         if (hw->phy.media_type == e1000_media_type_copper) {
 715                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
 716                 hw->phy.disable_polarity_correction = 0;
 717                 hw->phy.ms_type = e1000_ms_hw_default;
 718         }
 719
 720         /*
 721          * Start from a known state, this is important in reading the nvm
 722          * and mac from that.
 723          */
 724         igb_pf_reset_hw(hw);
 725
 726         /* Make sure we have a good EEPROM before we read from it */
 727         if (e1000_validate_nvm_checksum(hw) < 0) {
 728                 /*
 729                  * Some PCI-E parts fail the first check due to
 730                  * the link being in sleep state, call it again,
 731                  * if it fails a second time its a real issue.
 732                  */
 733                 if (e1000_validate_nvm_checksum(hw) < 0) {
 734                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
 735                         error = -EIO;
 736                         goto err_late;
 737                 }
 738         }
 739
 740         /* Read the permanent MAC address out of the EEPROM */
 741         if (e1000_read_mac_addr(hw) != 0) {
 742                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
 743                 error = -EIO;
 744                 goto err_late;
 745         }
 746
 747         /* Allocate memory for storing MAC addresses */
 748         eth_dev->data->mac_addrs = rte_zmalloc("e1000",
 749                 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
 750         if (eth_dev->data->mac_addrs == NULL) {
 751                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to "
 752                                                 "store MAC addresses",
 753                                 ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 754                 error = -ENOMEM;
 755                 goto err_late;
 756         }
 757
 758         /* Copy the permanent MAC address */
 759         ether_addr_copy((struct ether_addr *)hw->mac.addr, &eth_dev->data->mac_addrs[0]);
 760
 761         /* initialize the vfta */
 762         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 763
 764         /* Now initialize the hardware */
 765         if (igb_hardware_init(hw) != 0) {
 766                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
 767                 rte_free(eth_dev->data->mac_addrs);
 768                 eth_dev->data->mac_addrs = NULL;
 769                 error = -ENODEV;
 770                 goto err_late;
 771         }
 772         hw->mac.get_link_status = 1;
 773         adapter->stopped = 0;
 774
 775         /* Indicate SOL/IDER usage */
 776         if (e1000_check_reset_block(hw) < 0) {
 777                 PMD_INIT_LOG(ERR, "PHY reset is blocked due to"
 778                                         "SOL/IDER session");
 779         }
 780
 781         /* initialize PF if max_vfs not zero */
 782         igb_pf_host_init(eth_dev);
 783
 784         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 785         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 786         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 787         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 788         E1000_WRITE_FLUSH(hw);
 789
 790         PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x",
 791                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 792                      pci_dev->id.device_id);
 793
 794         rte_intr_callback_register(&pci_dev->intr_handle,
 795                                    eth_igb_interrupt_handler,
 796                                    (void *)eth_dev);
 797
 798         /* enable uio/vfio intr/eventfd mapping */
 799         rte_intr_enable(&pci_dev->intr_handle);
 800
 801         /* enable support intr */
 802         igb_intr_enable(eth_dev);
 803
 804         TAILQ_INIT(&filter_info->flex_list);
 805         filter_info->flex_mask = 0;
 806         TAILQ_INIT(&filter_info->twotuple_list);
 807         filter_info->twotuple_mask = 0;
 808         TAILQ_INIT(&filter_info->fivetuple_list);
 809         filter_info->fivetuple_mask = 0;
 810
 811         return 0;
 812
 813 err_late:
 814         igb_hw_control_release(hw);
 815
 816         return error;
 817 }
 818
 819 static int
 820 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev)
 821 {
 822         struct rte_pci_device *pci_dev;
 823         struct e1000_hw *hw;
 824         struct e1000_adapter *adapter =
 825                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 826
 827         PMD_INIT_FUNC_TRACE();
 828
 829         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 830                 return -EPERM;
 831
 832         hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 833         pci_dev = eth_dev->pci_dev;
 834
 835         if (adapter->stopped == 0)
 836                 eth_igb_close(eth_dev);
 837
 838         eth_dev->dev_ops = NULL;
 839         eth_dev->rx_pkt_burst = NULL;
 840         eth_dev->tx_pkt_burst = NULL;
 841
 842         /* Reset any pending lock */
 843         igb_reset_swfw_lock(hw);
 844
 845         rte_free(eth_dev->data->mac_addrs);
 846         eth_dev->data->mac_addrs = NULL;
 847
 848         /* uninitialize PF if max_vfs not zero */
 849         igb_pf_host_uninit(eth_dev);
 850
 851         /* disable uio intr before callback unregister */
 852         rte_intr_disable(&(pci_dev->intr_handle));
 853         rte_intr_callback_unregister(&(pci_dev->intr_handle),
 854                 eth_igb_interrupt_handler, (void *)eth_dev);
 855
 856         return 0;
 857 }
 858
 859 /*
 860  * Virtual Function device init
 861  */
 862 static int
 863 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev)
 864 {
 865         struct rte_pci_device *pci_dev;
 866         struct e1000_adapter *adapter =
 867                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 868         struct e1000_hw *hw =
 869                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 870         int diag;
 871         struct ether_addr *perm_addr = (struct ether_addr *)hw->mac.perm_addr;
 872
 873         PMD_INIT_FUNC_TRACE();
 874
 875         eth_dev->dev_ops = &igbvf_eth_dev_ops;
 876         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 877         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 878
 879         /* for secondary processes, we don't initialise any further as primary
 880          * has already done this work. Only check we don't need a different
 881          * RX function */
 882         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 883                 if (eth_dev->data->scattered_rx)
 884                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 885                 return 0;
 886         }
 887
 888         pci_dev = eth_dev->pci_dev;
 889
 890         rte_eth_copy_pci_info(eth_dev, pci_dev);
 891
 892         hw->device_id = pci_dev->id.device_id;
 893         hw->vendor_id = pci_dev->id.vendor_id;
 894         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 895         adapter->stopped = 0;
 896
 897         /* Initialize the shared code (base driver) */
 898         diag = e1000_setup_init_funcs(hw, TRUE);
 899         if (diag != 0) {
 900                 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d",
 901                         diag);
 902                 return -EIO;
 903         }
 904
 905         /* init_mailbox_params */
 906         hw->mbx.ops.init_params(hw);
 907
 908         /* Disable the interrupts for VF */
 909         igbvf_intr_disable(hw);
 910
 911         diag = hw->mac.ops.reset_hw(hw);
 912
 913         /* Allocate memory for storing MAC addresses */
 914         eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN *
 915                 hw->mac.rar_entry_count, 0);
 916         if (eth_dev->data->mac_addrs == NULL) {
 917                 PMD_INIT_LOG(ERR,
 918                         "Failed to allocate %d bytes needed to store MAC "
 919                         "addresses",
 920                         ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 921                 return -ENOMEM;
 922         }
 923
 924         /* Generate a random MAC address, if none was assigned by PF. */
 925         if (is_zero_ether_addr(perm_addr)) {
 926                 eth_random_addr(perm_addr->addr_bytes);
 927                 diag = e1000_rar_set(hw, perm_addr->addr_bytes, 0);
 928                 if (diag) {
 929                         rte_free(eth_dev->data->mac_addrs);
 930                         eth_dev->data->mac_addrs = NULL;
 931                         return diag;
 932                 }
 933                 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF");
 934                 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address "
 935                              "%02x:%02x:%02x:%02x:%02x:%02x",
 936                              perm_addr->addr_bytes[0],
 937                              perm_addr->addr_bytes[1],
 938                              perm_addr->addr_bytes[2],
 939                              perm_addr->addr_bytes[3],
 940                              perm_addr->addr_bytes[4],
 941                              perm_addr->addr_bytes[5]);
 942         }
 943
 944         /* Copy the permanent MAC address */
 945         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 946                         &eth_dev->data->mac_addrs[0]);
 947
 948         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x "
 949                      "mac.type=%s",
 950                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 951                      pci_dev->id.device_id, "igb_mac_82576_vf");
 952
 953         return 0;
 954 }
 955
 956 static int
 957 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev)
 958 {
 959         struct e1000_adapter *adapter =
 960                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 961
 962         PMD_INIT_FUNC_TRACE();
 963
 964         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 965                 return -EPERM;
 966
 967         if (adapter->stopped == 0)
 968                 igbvf_dev_close(eth_dev);
 969
 970         eth_dev->dev_ops = NULL;
 971         eth_dev->rx_pkt_burst = NULL;
 972         eth_dev->tx_pkt_burst = NULL;
 973
 974         rte_free(eth_dev->data->mac_addrs);
 975         eth_dev->data->mac_addrs = NULL;
 976
 977         return 0;
 978 }
 979
 980 static struct eth_driver rte_igb_pmd = {
 981         .pci_drv = {
 982                 .name = "rte_igb_pmd",
 983                 .id_table = pci_id_igb_map,
 984                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
 985                         RTE_PCI_DRV_DETACHABLE,
 986         },
 987         .eth_dev_init = eth_igb_dev_init,
 988         .eth_dev_uninit = eth_igb_dev_uninit,
 989         .dev_private_size = sizeof(struct e1000_adapter),
 990 };
 991
 992 /*
 993  * virtual function driver struct
 994  */
 995 static struct eth_driver rte_igbvf_pmd = {
 996         .pci_drv = {
 997                 .name = "rte_igbvf_pmd",
 998                 .id_table = pci_id_igbvf_map,
 999                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
1000         },
1001         .eth_dev_init = eth_igbvf_dev_init,
1002         .eth_dev_uninit = eth_igbvf_dev_uninit,
1003         .dev_private_size = sizeof(struct e1000_adapter),
1004 };
1005
1006 static int
1007 rte_igb_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
1008 {
1009         rte_eth_driver_register(&rte_igb_pmd);
1010         return 0;
1011 }
1012
1013 static void
1014 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1015 {
1016         struct e1000_hw *hw =
1017                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1018         /* RCTL: enable VLAN filter since VMDq always use VLAN filter */
1019         uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL);
1020         rctl |= E1000_RCTL_VFE;
1021         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1022 }
1023
1024 /*
1025  * VF Driver initialization routine.
1026  * Invoked one at EAL init time.
1027  * Register itself as the [Virtual Poll Mode] Driver of PCI IGB devices.
1028  */
1029 static int
1030 rte_igbvf_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
1031 {
1032         PMD_INIT_FUNC_TRACE();
1033
1034         rte_eth_driver_register(&rte_igbvf_pmd);
1035         return 0;
1036 }
1037
1038 static int
1039 igb_check_mq_mode(struct rte_eth_dev *dev)
1040 {
1041         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
1042         enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode;
1043         uint16_t nb_rx_q = dev->data->nb_rx_queues;
1044         uint16_t nb_tx_q = dev->data->nb_rx_queues;
1045
1046         if ((rx_mq_mode & ETH_MQ_RX_DCB_FLAG) ||
1047             tx_mq_mode == ETH_MQ_TX_DCB ||
1048             tx_mq_mode == ETH_MQ_TX_VMDQ_DCB) {
1049                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
1050                 return -EINVAL;
1051         }
1052         if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
1053                 /* Check multi-queue mode.
1054                  * To no break software we accept ETH_MQ_RX_NONE as this might
1055                  * be used to turn off VLAN filter.
1056                  */
1057
1058                 if (rx_mq_mode == ETH_MQ_RX_NONE ||
1059                     rx_mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1060                         dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY;
1061                         RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1;
1062                 } else {
1063                         /* Only support one queue on VFs.
1064                          * RSS together with SRIOV is not supported.
1065                          */
1066                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1067                                         " wrong mq_mode rx %d.",
1068                                         rx_mq_mode);
1069                         return -EINVAL;
1070                 }
1071                 /* TX mode is not used here, so mode might be ignored.*/
1072                 if (tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) {
1073                         /* SRIOV only works in VMDq enable mode */
1074                         PMD_INIT_LOG(WARNING, "SRIOV is active,"
1075                                         " TX mode %d is not supported. "
1076                                         " Driver will behave as %d mode.",
1077                                         tx_mq_mode, ETH_MQ_TX_VMDQ_ONLY);
1078                 }
1079
1080                 /* check valid queue number */
1081                 if ((nb_rx_q > 1) || (nb_tx_q > 1)) {
1082                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1083                                         " only support one queue on VFs.");
1084                         return -EINVAL;
1085                 }
1086         } else {
1087                 /* To no break software that set invalid mode, only display
1088                  * warning if invalid mode is used.
1089                  */
1090                 if (rx_mq_mode != ETH_MQ_RX_NONE &&
1091                     rx_mq_mode != ETH_MQ_RX_VMDQ_ONLY &&
1092                     rx_mq_mode != ETH_MQ_RX_RSS) {
1093                         /* RSS together with VMDq not supported*/
1094                         PMD_INIT_LOG(ERR, "RX mode %d is not supported.",
1095                                      rx_mq_mode);
1096                         return -EINVAL;
1097                 }
1098
1099                 if (tx_mq_mode != ETH_MQ_TX_NONE &&
1100                     tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) {
1101                         PMD_INIT_LOG(WARNING, "TX mode %d is not supported."
1102                                         " Due to txmode is meaningless in this"
1103                                         " driver, just ignore.",
1104                                         tx_mq_mode);
1105                 }
1106         }
1107         return 0;
1108 }
1109
1110 static int
1111 eth_igb_configure(struct rte_eth_dev *dev)
1112 {
1113         struct e1000_interrupt *intr =
1114                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1115         int ret;
1116
1117         PMD_INIT_FUNC_TRACE();
1118
1119         /* multipe queue mode checking */
1120         ret  = igb_check_mq_mode(dev);
1121         if (ret != 0) {
1122                 PMD_DRV_LOG(ERR, "igb_check_mq_mode fails with %d.",
1123                             ret);
1124                 return ret;
1125         }
1126
1127         intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
1128         PMD_INIT_FUNC_TRACE();
1129
1130         return 0;
1131 }
1132
1133 static int
1134 eth_igb_start(struct rte_eth_dev *dev)
1135 {
1136         struct e1000_hw *hw =
1137                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1138         struct e1000_adapter *adapter =
1139                 E1000_DEV_PRIVATE(dev->data->dev_private);
1140         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1141         int ret, mask;
1142         uint32_t intr_vector = 0;
1143         uint32_t ctrl_ext;
1144         uint32_t *speeds;
1145         int num_speeds;
1146         bool autoneg;
1147
1148         PMD_INIT_FUNC_TRACE();
1149
1150         /* disable uio/vfio intr/eventfd mapping */
1151         rte_intr_disable(intr_handle);
1152
1153         /* Power up the phy. Needed to make the link go Up */
1154         e1000_power_up_phy(hw);
1155
1156         /*
1157          * Packet Buffer Allocation (PBA)
1158          * Writing PBA sets the receive portion of the buffer
1159          * the remainder is used for the transmit buffer.
1160          */
1161         if (hw->mac.type == e1000_82575) {
1162                 uint32_t pba;
1163
1164                 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
1165                 E1000_WRITE_REG(hw, E1000_PBA, pba);
1166         }
1167
1168         /* Put the address into the Receive Address Array */
1169         e1000_rar_set(hw, hw->mac.addr, 0);
1170
1171         /* Initialize the hardware */
1172         if (igb_hardware_init(hw)) {
1173                 PMD_INIT_LOG(ERR, "Unable to initialize the hardware");
1174                 return -EIO;
1175         }
1176         adapter->stopped = 0;
1177
1178         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1179
1180         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1181         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
1182         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
1183         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
1184         E1000_WRITE_FLUSH(hw);
1185
1186         /* configure PF module if SRIOV enabled */
1187         igb_pf_host_configure(dev);
1188
1189         /* check and configure queue intr-vector mapping */
1190         if ((rte_intr_cap_multiple(intr_handle) ||
1191              !RTE_ETH_DEV_SRIOV(dev).active) &&
1192             dev->data->dev_conf.intr_conf.rxq != 0) {
1193                 intr_vector = dev->data->nb_rx_queues;
1194                 if (rte_intr_efd_enable(intr_handle, intr_vector))
1195                         return -1;
1196         }
1197
1198         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
1199                 intr_handle->intr_vec =
1200                         rte_zmalloc("intr_vec",
1201                                     dev->data->nb_rx_queues * sizeof(int), 0);
1202                 if (intr_handle->intr_vec == NULL) {
1203                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
1204                                      " intr_vec\n", dev->data->nb_rx_queues);
1205                         return -ENOMEM;
1206                 }
1207         }
1208
1209         /* confiugre msix for rx interrupt */
1210         eth_igb_configure_msix_intr(dev);
1211
1212         /* Configure for OS presence */
1213         igb_init_manageability(hw);
1214
1215         eth_igb_tx_init(dev);
1216
1217         /* This can fail when allocating mbufs for descriptor rings */
1218         ret = eth_igb_rx_init(dev);
1219         if (ret) {
1220                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
1221                 igb_dev_clear_queues(dev);
1222                 return ret;
1223         }
1224
1225         e1000_clear_hw_cntrs_base_generic(hw);
1226
1227         /*
1228          * VLAN Offload Settings
1229          */
1230         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1231                         ETH_VLAN_EXTEND_MASK;
1232         eth_igb_vlan_offload_set(dev, mask);
1233
1234         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1235                 /* Enable VLAN filter since VMDq always use VLAN filter */
1236                 igb_vmdq_vlan_hw_filter_enable(dev);
1237         }
1238
1239         if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) ||
1240                 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) ||
1241                 (hw->mac.type == e1000_i211)) {
1242                 /* Configure EITR with the maximum possible value (0xFFFF) */
1243                 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF);
1244         }
1245
1246         /* Setup link speed and duplex */
1247         speeds = &dev->data->dev_conf.link_speeds;
1248         if (*speeds == ETH_LINK_SPEED_AUTONEG) {
1249                 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
1250         } else {
1251                 num_speeds = 0;
1252                 autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0;
1253
1254                 /* Reset */
1255                 hw->phy.autoneg_advertised = 0;
1256
1257                 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M |
1258                                 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M |
1259                                 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_FIXED)) {
1260                         num_speeds = -1;
1261                         goto error_invalid_config;
1262                 }
1263                 if (*speeds & ETH_LINK_SPEED_10M_HD) {
1264                         hw->phy.autoneg_advertised |= ADVERTISE_10_HALF;
1265                         num_speeds++;
1266                 }
1267                 if (*speeds & ETH_LINK_SPEED_10M) {
1268                         hw->phy.autoneg_advertised |= ADVERTISE_10_FULL;
1269                         num_speeds++;
1270                 }
1271                 if (*speeds & ETH_LINK_SPEED_100M_HD) {
1272                         hw->phy.autoneg_advertised |= ADVERTISE_100_HALF;
1273                         num_speeds++;
1274                 }
1275                 if (*speeds & ETH_LINK_SPEED_100M) {
1276                         hw->phy.autoneg_advertised |= ADVERTISE_100_FULL;
1277                         num_speeds++;
1278                 }
1279                 if (*speeds & ETH_LINK_SPEED_1G) {
1280                         hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL;
1281                         num_speeds++;
1282                 }
1283                 if (num_speeds == 0 || (!autoneg && (num_speeds > 1)))
1284                         goto error_invalid_config;
1285         }
1286
1287         e1000_setup_link(hw);
1288
1289         if (rte_intr_allow_others(intr_handle)) {
1290                 /* check if lsc interrupt is enabled */
1291                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1292                         eth_igb_lsc_interrupt_setup(dev);
1293         } else {
1294                 rte_intr_callback_unregister(intr_handle,
1295                                              eth_igb_interrupt_handler,
1296                                              (void *)dev);
1297                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1298                         PMD_INIT_LOG(INFO, "lsc won't enable because of"
1299                                      " no intr multiplex\n");
1300         }
1301
1302         /* check if rxq interrupt is enabled */
1303         if (dev->data->dev_conf.intr_conf.rxq != 0 &&
1304             rte_intr_dp_is_en(intr_handle))
1305                 eth_igb_rxq_interrupt_setup(dev);
1306
1307         /* enable uio/vfio intr/eventfd mapping */
1308         rte_intr_enable(intr_handle);
1309
1310         /* resume enabled intr since hw reset */
1311         igb_intr_enable(dev);
1312
1313         PMD_INIT_LOG(DEBUG, "<<");
1314
1315         return 0;
1316
1317 error_invalid_config:
1318         PMD_INIT_LOG(ERR, "Invalid advertised speeds (%u) for port %u",
1319                      dev->data->dev_conf.link_speeds, dev->data->port_id);
1320         igb_dev_clear_queues(dev);
1321         return -EINVAL;
1322 }
1323
1324 /*********************************************************************
1325  *
1326  *  This routine disables all traffic on the adapter by issuing a
1327  *  global reset on the MAC.
1328  *
1329  **********************************************************************/
1330 static void
1331 eth_igb_stop(struct rte_eth_dev *dev)
1332 {
1333         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1334         struct e1000_filter_info *filter_info =
1335                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
1336         struct rte_eth_link link;
1337         struct e1000_flex_filter *p_flex;
1338         struct e1000_5tuple_filter *p_5tuple, *p_5tuple_next;
1339         struct e1000_2tuple_filter *p_2tuple, *p_2tuple_next;
1340         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1341
1342         igb_intr_disable(hw);
1343
1344         /* disable intr eventfd mapping */
1345         rte_intr_disable(intr_handle);
1346
1347         igb_pf_reset_hw(hw);
1348         E1000_WRITE_REG(hw, E1000_WUC, 0);
1349
1350         /* Set bit for Go Link disconnect */
1351         if (hw->mac.type >= e1000_82580) {
1352                 uint32_t phpm_reg;
1353
1354                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1355                 phpm_reg |= E1000_82580_PM_GO_LINKD;
1356                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1357         }
1358
1359         /* Power down the phy. Needed to make the link go Down */
1360         if (hw->phy.media_type == e1000_media_type_copper)
1361                 e1000_power_down_phy(hw);
1362         else
1363                 e1000_shutdown_fiber_serdes_link(hw);
1364
1365         igb_dev_clear_queues(dev);
1366
1367         /* clear the recorded link status */
1368         memset(&link, 0, sizeof(link));
1369         rte_igb_dev_atomic_write_link_status(dev, &link);
1370
1371         /* Remove all flex filters of the device */
1372         while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) {
1373                 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries);
1374                 rte_free(p_flex);
1375         }
1376         filter_info->flex_mask = 0;
1377
1378         /* Remove all ntuple filters of the device */
1379         for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list);
1380              p_5tuple != NULL; p_5tuple = p_5tuple_next) {
1381                 p_5tuple_next = TAILQ_NEXT(p_5tuple, entries);
1382                 TAILQ_REMOVE(&filter_info->fivetuple_list,
1383                              p_5tuple, entries);
1384                 rte_free(p_5tuple);
1385         }
1386         filter_info->fivetuple_mask = 0;
1387         for (p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list);
1388              p_2tuple != NULL; p_2tuple = p_2tuple_next) {
1389                 p_2tuple_next = TAILQ_NEXT(p_2tuple, entries);
1390                 TAILQ_REMOVE(&filter_info->twotuple_list,
1391                              p_2tuple, entries);
1392                 rte_free(p_2tuple);
1393         }
1394         filter_info->twotuple_mask = 0;
1395
1396         if (!rte_intr_allow_others(intr_handle))
1397                 /* resume to the default handler */
1398                 rte_intr_callback_register(intr_handle,
1399                                            eth_igb_interrupt_handler,
1400                                            (void *)dev);
1401
1402         /* Clean datapath event and queue/vec mapping */
1403         rte_intr_efd_disable(intr_handle);
1404         if (intr_handle->intr_vec != NULL) {
1405                 rte_free(intr_handle->intr_vec);
1406                 intr_handle->intr_vec = NULL;
1407         }
1408 }
1409
1410 static void
1411 eth_igb_close(struct rte_eth_dev *dev)
1412 {
1413         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1414         struct e1000_adapter *adapter =
1415                 E1000_DEV_PRIVATE(dev->data->dev_private);
1416         struct rte_eth_link link;
1417         struct rte_pci_device *pci_dev;
1418
1419         eth_igb_stop(dev);
1420         adapter->stopped = 1;
1421
1422         e1000_phy_hw_reset(hw);
1423         igb_release_manageability(hw);
1424         igb_hw_control_release(hw);
1425
1426         /* Clear bit for Go Link disconnect */
1427         if (hw->mac.type >= e1000_82580) {
1428                 uint32_t phpm_reg;
1429
1430                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1431                 phpm_reg &= ~E1000_82580_PM_GO_LINKD;
1432                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1433         }
1434
1435         igb_dev_free_queues(dev);
1436
1437         pci_dev = dev->pci_dev;
1438         if (pci_dev->intr_handle.intr_vec) {
1439                 rte_free(pci_dev->intr_handle.intr_vec);
1440                 pci_dev->intr_handle.intr_vec = NULL;
1441         }
1442
1443         memset(&link, 0, sizeof(link));
1444         rte_igb_dev_atomic_write_link_status(dev, &link);
1445 }
1446
1447 static int
1448 igb_get_rx_buffer_size(struct e1000_hw *hw)
1449 {
1450         uint32_t rx_buf_size;
1451         if (hw->mac.type == e1000_82576) {
1452                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10;
1453         } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) {
1454                 /* PBS needs to be translated according to a lookup table */
1455                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf);
1456                 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size);
1457                 rx_buf_size = (rx_buf_size << 10);
1458         } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) {
1459                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10;
1460         } else {
1461                 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10;
1462         }
1463
1464         return rx_buf_size;
1465 }
1466
1467 /*********************************************************************
1468  *
1469  *  Initialize the hardware
1470  *
1471  **********************************************************************/
1472 static int
1473 igb_hardware_init(struct e1000_hw *hw)
1474 {
1475         uint32_t rx_buf_size;
1476         int diag;
1477
1478         /* Let the firmware know the OS is in control */
1479         igb_hw_control_acquire(hw);
1480
1481         /*
1482          * These parameters control the automatic generation (Tx) and
1483          * response (Rx) to Ethernet PAUSE frames.
1484          * - High water mark should allow for at least two standard size (1518)
1485          *   frames to be received after sending an XOFF.
1486          * - Low water mark works best when it is very near the high water mark.
1487          *   This allows the receiver to restart by sending XON when it has
1488          *   drained a bit. Here we use an arbitrary value of 1500 which will
1489          *   restart after one full frame is pulled from the buffer. There
1490          *   could be several smaller frames in the buffer and if so they will
1491          *   not trigger the XON until their total number reduces the buffer
1492          *   by 1500.
1493          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
1494          */
1495         rx_buf_size = igb_get_rx_buffer_size(hw);
1496
1497         hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2);
1498         hw->fc.low_water = hw->fc.high_water - 1500;
1499         hw->fc.pause_time = IGB_FC_PAUSE_TIME;
1500         hw->fc.send_xon = 1;
1501
1502         /* Set Flow control, use the tunable location if sane */
1503         if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4))
1504                 hw->fc.requested_mode = igb_fc_setting;
1505         else
1506                 hw->fc.requested_mode = e1000_fc_none;
1507
1508         /* Issue a global reset */
1509         igb_pf_reset_hw(hw);
1510         E1000_WRITE_REG(hw, E1000_WUC, 0);
1511
1512         diag = e1000_init_hw(hw);
1513         if (diag < 0)
1514                 return diag;
1515
1516         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1517         e1000_get_phy_info(hw);
1518         e1000_check_for_link(hw);
1519
1520         return 0;
1521 }
1522
1523 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */
1524 static void
1525 igb_read_stats_registers(struct e1000_hw *hw, struct e1000_hw_stats *stats)
1526 {
1527         int pause_frames;
1528
1529         uint64_t old_gprc  = stats->gprc;
1530         uint64_t old_gptc  = stats->gptc;
1531         uint64_t old_tpr   = stats->tpr;
1532         uint64_t old_tpt   = stats->tpt;
1533         uint64_t old_rpthc = stats->rpthc;
1534         uint64_t old_hgptc = stats->hgptc;
1535
1536         if(hw->phy.media_type == e1000_media_type_copper ||
1537             (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
1538                 stats->symerrs +=
1539                     E1000_READ_REG(hw,E1000_SYMERRS);
1540                 stats->sec += E1000_READ_REG(hw, E1000_SEC);
1541         }
1542
1543         stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
1544         stats->mpc += E1000_READ_REG(hw, E1000_MPC);
1545         stats->scc += E1000_READ_REG(hw, E1000_SCC);
1546         stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
1547
1548         stats->mcc += E1000_READ_REG(hw, E1000_MCC);
1549         stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
1550         stats->colc += E1000_READ_REG(hw, E1000_COLC);
1551         stats->dc += E1000_READ_REG(hw, E1000_DC);
1552         stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
1553         stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
1554         stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
1555         /*
1556         ** For watchdog management we need to know if we have been
1557         ** paused during the last interval, so capture that here.
1558         */
1559         pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC);
1560         stats->xoffrxc += pause_frames;
1561         stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
1562         stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
1563         stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
1564         stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
1565         stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
1566         stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
1567         stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
1568         stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
1569         stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
1570         stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
1571         stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
1572         stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
1573
1574         /* For the 64-bit byte counters the low dword must be read first. */
1575         /* Both registers clear on the read of the high dword */
1576
1577         /* Workaround CRC bytes included in size, take away 4 bytes/packet */
1578         stats->gorc += E1000_READ_REG(hw, E1000_GORCL);
1579         stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32);
1580         stats->gorc -= (stats->gprc - old_gprc) * ETHER_CRC_LEN;
1581         stats->gotc += E1000_READ_REG(hw, E1000_GOTCL);
1582         stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32);
1583         stats->gotc -= (stats->gptc - old_gptc) * ETHER_CRC_LEN;
1584
1585         stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
1586         stats->ruc += E1000_READ_REG(hw, E1000_RUC);
1587         stats->rfc += E1000_READ_REG(hw, E1000_RFC);
1588         stats->roc += E1000_READ_REG(hw, E1000_ROC);
1589         stats->rjc += E1000_READ_REG(hw, E1000_RJC);
1590
1591         stats->tpr += E1000_READ_REG(hw, E1000_TPR);
1592         stats->tpt += E1000_READ_REG(hw, E1000_TPT);
1593
1594         stats->tor += E1000_READ_REG(hw, E1000_TORL);
1595         stats->tor += ((uint64_t)E1000_READ_REG(hw, E1000_TORH) << 32);
1596         stats->tor -= (stats->tpr - old_tpr) * ETHER_CRC_LEN;
1597         stats->tot += E1000_READ_REG(hw, E1000_TOTL);
1598         stats->tot += ((uint64_t)E1000_READ_REG(hw, E1000_TOTH) << 32);
1599         stats->tot -= (stats->tpt - old_tpt) * ETHER_CRC_LEN;
1600
1601         stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
1602         stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
1603         stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
1604         stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
1605         stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
1606         stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
1607         stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
1608         stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
1609
1610         /* Interrupt Counts */
1611
1612         stats->iac += E1000_READ_REG(hw, E1000_IAC);
1613         stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
1614         stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
1615         stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
1616         stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
1617         stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
1618         stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
1619         stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
1620         stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
1621
1622         /* Host to Card Statistics */
1623
1624         stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
1625         stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
1626         stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
1627         stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
1628         stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
1629         stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
1630         stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
1631         stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL);
1632         stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32);
1633         stats->hgorc -= (stats->rpthc - old_rpthc) * ETHER_CRC_LEN;
1634         stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL);
1635         stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32);
1636         stats->hgotc -= (stats->hgptc - old_hgptc) * ETHER_CRC_LEN;
1637         stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
1638         stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
1639         stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
1640
1641         stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
1642         stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
1643         stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
1644         stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
1645         stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
1646         stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
1647 }
1648
1649 static void
1650 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1651 {
1652         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1653         struct e1000_hw_stats *stats =
1654                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1655
1656         igb_read_stats_registers(hw, stats);
1657
1658         if (rte_stats == NULL)
1659                 return;
1660
1661         /* Rx Errors */
1662         rte_stats->imissed = stats->mpc;
1663         rte_stats->ierrors = stats->crcerrs +
1664                              stats->rlec + stats->ruc + stats->roc +
1665                              stats->rxerrc + stats->algnerrc + stats->cexterr;
1666
1667         /* Tx Errors */
1668         rte_stats->oerrors = stats->ecol + stats->latecol;
1669
1670         rte_stats->ipackets = stats->gprc;
1671         rte_stats->opackets = stats->gptc;
1672         rte_stats->ibytes   = stats->gorc;
1673         rte_stats->obytes   = stats->gotc;
1674 }
1675
1676 static void
1677 eth_igb_stats_reset(struct rte_eth_dev *dev)
1678 {
1679         struct e1000_hw_stats *hw_stats =
1680                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1681
1682         /* HW registers are cleared on read */
1683         eth_igb_stats_get(dev, NULL);
1684
1685         /* Reset software totals */
1686         memset(hw_stats, 0, sizeof(*hw_stats));
1687 }
1688
1689 static void
1690 eth_igb_xstats_reset(struct rte_eth_dev *dev)
1691 {
1692         struct e1000_hw_stats *stats =
1693                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1694
1695         /* HW registers are cleared on read */
1696         eth_igb_xstats_get(dev, NULL, IGB_NB_XSTATS);
1697
1698         /* Reset software totals */
1699         memset(stats, 0, sizeof(*stats));
1700 }
1701
1702 static int eth_igb_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1703         struct rte_eth_xstat_name *xstats_names,
1704         __rte_unused unsigned limit)
1705 {
1706         unsigned i;
1707
1708         if (xstats_names == NULL)
1709                 return IGB_NB_XSTATS;
1710
1711         /* Note: limit checked in rte_eth_xstats_names() */
1712
1713         for (i = 0; i < IGB_NB_XSTATS; i++) {
1714                 snprintf(xstats_names[i].name, sizeof(xstats_names[i].name),
1715                          "%s", rte_igb_stats_strings[i].name);
1716                 xstats_names[i].id = i;
1717         }
1718
1719         return IGB_NB_XSTATS;
1720 }
1721
1722 static int
1723 eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
1724                    unsigned n)
1725 {
1726         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1727         struct e1000_hw_stats *hw_stats =
1728                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1729         unsigned i;
1730
1731         if (n < IGB_NB_XSTATS)
1732                 return IGB_NB_XSTATS;
1733
1734         igb_read_stats_registers(hw, hw_stats);
1735
1736         /* If this is a reset xstats is NULL, and we have cleared the
1737          * registers by reading them.
1738          */
1739         if (!xstats)
1740                 return 0;
1741
1742         /* Extended stats */
1743         for (i = 0; i < IGB_NB_XSTATS; i++) {
1744                 xstats[i].name[0] = '\0';
1745                 xstats[i].id = i;
1746                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
1747                         rte_igb_stats_strings[i].offset);
1748         }
1749
1750         return IGB_NB_XSTATS;
1751 }
1752
1753 static void
1754 igbvf_read_stats_registers(struct e1000_hw *hw, struct e1000_vf_stats *hw_stats)
1755 {
1756         /* Good Rx packets, include VF loopback */
1757         UPDATE_VF_STAT(E1000_VFGPRC,
1758             hw_stats->last_gprc, hw_stats->gprc);
1759
1760         /* Good Rx octets, include VF loopback */
1761         UPDATE_VF_STAT(E1000_VFGORC,
1762             hw_stats->last_gorc, hw_stats->gorc);
1763
1764         /* Good Tx packets, include VF loopback */
1765         UPDATE_VF_STAT(E1000_VFGPTC,
1766             hw_stats->last_gptc, hw_stats->gptc);
1767
1768         /* Good Tx octets, include VF loopback */
1769         UPDATE_VF_STAT(E1000_VFGOTC,
1770             hw_stats->last_gotc, hw_stats->gotc);
1771
1772         /* Rx Multicst packets */
1773         UPDATE_VF_STAT(E1000_VFMPRC,
1774             hw_stats->last_mprc, hw_stats->mprc);
1775
1776         /* Good Rx loopback packets */
1777         UPDATE_VF_STAT(E1000_VFGPRLBC,
1778             hw_stats->last_gprlbc, hw_stats->gprlbc);
1779
1780         /* Good Rx loopback octets */
1781         UPDATE_VF_STAT(E1000_VFGORLBC,
1782             hw_stats->last_gorlbc, hw_stats->gorlbc);
1783
1784         /* Good Tx loopback packets */
1785         UPDATE_VF_STAT(E1000_VFGPTLBC,
1786             hw_stats->last_gptlbc, hw_stats->gptlbc);
1787
1788         /* Good Tx loopback octets */
1789         UPDATE_VF_STAT(E1000_VFGOTLBC,
1790             hw_stats->last_gotlbc, hw_stats->gotlbc);
1791 }
1792
1793 static int eth_igbvf_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1794                                      struct rte_eth_xstat_name *xstats_names,
1795                                      __rte_unused unsigned limit)
1796 {
1797         unsigned i;
1798
1799         if (xstats_names != NULL)
1800                 for (i = 0; i < IGBVF_NB_XSTATS; i++) {
1801                         snprintf(xstats_names[i].name,
1802                                 sizeof(xstats_names[i].name), "%s",
1803                                 rte_igbvf_stats_strings[i].name);
1804                         xstats_names[i].id = i;
1805                 }
1806         return IGBVF_NB_XSTATS;
1807 }
1808
1809 static int
1810 eth_igbvf_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
1811                      unsigned n)
1812 {
1813         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1814         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *)
1815                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1816         unsigned i;
1817
1818         if (n < IGBVF_NB_XSTATS)
1819                 return IGBVF_NB_XSTATS;
1820
1821         igbvf_read_stats_registers(hw, hw_stats);
1822
1823         if (!xstats)
1824                 return 0;
1825
1826         for (i = 0; i < IGBVF_NB_XSTATS; i++) {
1827                 xstats[i].name[0] = '\0';
1828                 xstats[i].id = i;
1829                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
1830                         rte_igbvf_stats_strings[i].offset);
1831         }
1832
1833         return IGBVF_NB_XSTATS;
1834 }
1835
1836 static void
1837 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1838 {
1839         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1840         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *)
1841                           E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1842
1843         igbvf_read_stats_registers(hw, hw_stats);
1844
1845         if (rte_stats == NULL)
1846                 return;
1847
1848         rte_stats->ipackets = hw_stats->gprc;
1849         rte_stats->ibytes = hw_stats->gorc;
1850         rte_stats->opackets = hw_stats->gptc;
1851         rte_stats->obytes = hw_stats->gotc;
1852 }
1853
1854 static void
1855 eth_igbvf_stats_reset(struct rte_eth_dev *dev)
1856 {
1857         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1858                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1859
1860         /* Sync HW register to the last stats */
1861         eth_igbvf_stats_get(dev, NULL);
1862
1863         /* reset HW current stats*/
1864         memset(&hw_stats->gprc, 0, sizeof(*hw_stats) -
1865                offsetof(struct e1000_vf_stats, gprc));
1866 }
1867
1868 static void
1869 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1870 {
1871         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1872
1873         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1874         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1875         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1876         dev_info->rx_offload_capa =
1877                 DEV_RX_OFFLOAD_VLAN_STRIP |
1878                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1879                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1880                 DEV_RX_OFFLOAD_TCP_CKSUM;
1881         dev_info->tx_offload_capa =
1882                 DEV_TX_OFFLOAD_VLAN_INSERT |
1883                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1884                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1885                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1886                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
1887                 DEV_TX_OFFLOAD_TCP_TSO;
1888
1889         switch (hw->mac.type) {
1890         case e1000_82575:
1891                 dev_info->max_rx_queues = 4;
1892                 dev_info->max_tx_queues = 4;
1893                 dev_info->max_vmdq_pools = 0;
1894                 break;
1895
1896         case e1000_82576:
1897                 dev_info->max_rx_queues = 16;
1898                 dev_info->max_tx_queues = 16;
1899                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1900                 dev_info->vmdq_queue_num = 16;
1901                 break;
1902
1903         case e1000_82580:
1904                 dev_info->max_rx_queues = 8;
1905                 dev_info->max_tx_queues = 8;
1906                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1907                 dev_info->vmdq_queue_num = 8;
1908                 break;
1909
1910         case e1000_i350:
1911                 dev_info->max_rx_queues = 8;
1912                 dev_info->max_tx_queues = 8;
1913                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1914                 dev_info->vmdq_queue_num = 8;
1915                 break;
1916
1917         case e1000_i354:
1918                 dev_info->max_rx_queues = 8;
1919                 dev_info->max_tx_queues = 8;
1920                 break;
1921
1922         case e1000_i210:
1923                 dev_info->max_rx_queues = 4;
1924                 dev_info->max_tx_queues = 4;
1925                 dev_info->max_vmdq_pools = 0;
1926                 break;
1927
1928         case e1000_i211:
1929                 dev_info->max_rx_queues = 2;
1930                 dev_info->max_tx_queues = 2;
1931                 dev_info->max_vmdq_pools = 0;
1932                 break;
1933
1934         default:
1935                 /* Should not happen */
1936                 break;
1937         }
1938         dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t);
1939         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
1940         dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL;
1941
1942         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1943                 .rx_thresh = {
1944                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1945                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1946                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
1947                 },
1948                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
1949                 .rx_drop_en = 0,
1950         };
1951
1952         dev_info->default_txconf = (struct rte_eth_txconf) {
1953                 .tx_thresh = {
1954                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
1955                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
1956                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
1957                 },
1958                 .txq_flags = 0,
1959         };
1960
1961         dev_info->rx_desc_lim = rx_desc_lim;
1962         dev_info->tx_desc_lim = tx_desc_lim;
1963
1964         dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M |
1965                         ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M |
1966                         ETH_LINK_SPEED_1G;
1967 }
1968
1969 static const uint32_t *
1970 eth_igb_supported_ptypes_get(struct rte_eth_dev *dev)
1971 {
1972         static const uint32_t ptypes[] = {
1973                 /* refers to igb_rxd_pkt_info_to_pkt_type() */
1974                 RTE_PTYPE_L2_ETHER,
1975                 RTE_PTYPE_L3_IPV4,
1976                 RTE_PTYPE_L3_IPV4_EXT,
1977                 RTE_PTYPE_L3_IPV6,
1978                 RTE_PTYPE_L3_IPV6_EXT,
1979                 RTE_PTYPE_L4_TCP,
1980                 RTE_PTYPE_L4_UDP,
1981                 RTE_PTYPE_L4_SCTP,
1982                 RTE_PTYPE_TUNNEL_IP,
1983                 RTE_PTYPE_INNER_L3_IPV6,
1984                 RTE_PTYPE_INNER_L3_IPV6_EXT,
1985                 RTE_PTYPE_INNER_L4_TCP,
1986                 RTE_PTYPE_INNER_L4_UDP,
1987                 RTE_PTYPE_UNKNOWN
1988         };
1989
1990         if (dev->rx_pkt_burst == eth_igb_recv_pkts ||
1991             dev->rx_pkt_burst == eth_igb_recv_scattered_pkts)
1992                 return ptypes;
1993         return NULL;
1994 }
1995
1996 static void
1997 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1998 {
1999         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2000
2001         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
2002         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
2003         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
2004         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
2005                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2006                                 DEV_RX_OFFLOAD_UDP_CKSUM  |
2007                                 DEV_RX_OFFLOAD_TCP_CKSUM;
2008         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
2009                                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
2010                                 DEV_TX_OFFLOAD_UDP_CKSUM   |
2011                                 DEV_TX_OFFLOAD_TCP_CKSUM   |
2012                                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
2013                                 DEV_TX_OFFLOAD_TCP_TSO;
2014         switch (hw->mac.type) {
2015         case e1000_vfadapt:
2016                 dev_info->max_rx_queues = 2;
2017                 dev_info->max_tx_queues = 2;
2018                 break;
2019         case e1000_vfadapt_i350:
2020                 dev_info->max_rx_queues = 1;
2021                 dev_info->max_tx_queues = 1;
2022                 break;
2023         default:
2024                 /* Should not happen */
2025                 break;
2026         }
2027
2028         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2029                 .rx_thresh = {
2030                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
2031                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
2032                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
2033                 },
2034                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
2035                 .rx_drop_en = 0,
2036         };
2037
2038         dev_info->default_txconf = (struct rte_eth_txconf) {
2039                 .tx_thresh = {
2040                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
2041                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
2042                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
2043                 },
2044                 .txq_flags = 0,
2045         };
2046
2047         dev_info->rx_desc_lim = rx_desc_lim;
2048         dev_info->tx_desc_lim = tx_desc_lim;
2049 }
2050
2051 /* return 0 means link status changed, -1 means not changed */
2052 static int
2053 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete)
2054 {
2055         struct e1000_hw *hw =
2056                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2057         struct rte_eth_link link, old;
2058         int link_check, count;
2059
2060         link_check = 0;
2061         hw->mac.get_link_status = 1;
2062
2063         /* possible wait-to-complete in up to 9 seconds */
2064         for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) {
2065                 /* Read the real link status */
2066                 switch (hw->phy.media_type) {
2067                 case e1000_media_type_copper:
2068                         /* Do the work to read phy */
2069                         e1000_check_for_link(hw);
2070                         link_check = !hw->mac.get_link_status;
2071                         break;
2072
2073                 case e1000_media_type_fiber:
2074                         e1000_check_for_link(hw);
2075                         link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2076                                       E1000_STATUS_LU);
2077                         break;
2078
2079                 case e1000_media_type_internal_serdes:
2080                         e1000_check_for_link(hw);
2081                         link_check = hw->mac.serdes_has_link;
2082                         break;
2083
2084                 /* VF device is type_unknown */
2085                 case e1000_media_type_unknown:
2086                         eth_igbvf_link_update(hw);
2087                         link_check = !hw->mac.get_link_status;
2088                         break;
2089
2090                 default:
2091                         break;
2092                 }
2093                 if (link_check || wait_to_complete == 0)
2094                         break;
2095                 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL);
2096         }
2097         memset(&link, 0, sizeof(link));
2098         rte_igb_dev_atomic_read_link_status(dev, &link);
2099         old = link;
2100
2101         /* Now we check if a transition has happened */
2102         if (link_check) {
2103                 uint16_t duplex, speed;
2104                 hw->mac.ops.get_link_up_info(hw, &speed, &duplex);
2105                 link.link_duplex = (duplex == FULL_DUPLEX) ?
2106                                 ETH_LINK_FULL_DUPLEX :
2107                                 ETH_LINK_HALF_DUPLEX;
2108                 link.link_speed = speed;
2109                 link.link_status = ETH_LINK_UP;
2110                 link.link_autoneg = !(dev->data->dev_conf.link_speeds &
2111                                 ETH_LINK_SPEED_FIXED);
2112         } else if (!link_check) {
2113                 link.link_speed = 0;
2114                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
2115                 link.link_status = ETH_LINK_DOWN;
2116                 link.link_autoneg = ETH_LINK_SPEED_FIXED;
2117         }
2118         rte_igb_dev_atomic_write_link_status(dev, &link);
2119
2120         /* not changed */
2121         if (old.link_status == link.link_status)
2122                 return -1;
2123
2124         /* changed */
2125         return 0;
2126 }
2127
2128 /*
2129  * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
2130  * For ASF and Pass Through versions of f/w this means
2131  * that the driver is loaded.
2132  */
2133 static void
2134 igb_hw_control_acquire(struct e1000_hw *hw)
2135 {
2136         uint32_t ctrl_ext;
2137
2138         /* Let firmware know the driver has taken over */
2139         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
2140         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
2141 }
2142
2143 /*
2144  * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
2145  * For ASF and Pass Through versions of f/w this means that the
2146  * driver is no longer loaded.
2147  */
2148 static void
2149 igb_hw_control_release(struct e1000_hw *hw)
2150 {
2151         uint32_t ctrl_ext;
2152
2153         /* Let firmware taken over control of h/w */
2154         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
2155         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
2156                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
2157 }
2158
2159 /*
2160  * Bit of a misnomer, what this really means is
2161  * to enable OS management of the system... aka
2162  * to disable special hardware management features.
2163  */
2164 static void
2165 igb_init_manageability(struct e1000_hw *hw)
2166 {
2167         if (e1000_enable_mng_pass_thru(hw)) {
2168                 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H);
2169                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
2170
2171                 /* disable hardware interception of ARP */
2172                 manc &= ~(E1000_MANC_ARP_EN);
2173
2174                 /* enable receiving management packets to the host */
2175                 manc |= E1000_MANC_EN_MNG2HOST;
2176                 manc2h |= 1 << 5;  /* Mng Port 623 */
2177                 manc2h |= 1 << 6;  /* Mng Port 664 */
2178                 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h);
2179                 E1000_WRITE_REG(hw, E1000_MANC, manc);
2180         }
2181 }
2182
2183 static void
2184 igb_release_manageability(struct e1000_hw *hw)
2185 {
2186         if (e1000_enable_mng_pass_thru(hw)) {
2187                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
2188
2189                 manc |= E1000_MANC_ARP_EN;
2190                 manc &= ~E1000_MANC_EN_MNG2HOST;
2191
2192                 E1000_WRITE_REG(hw, E1000_MANC, manc);
2193         }
2194 }
2195
2196 static void
2197 eth_igb_promiscuous_enable(struct rte_eth_dev *dev)
2198 {
2199         struct e1000_hw *hw =
2200                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2201         uint32_t rctl;
2202
2203         rctl = E1000_READ_REG(hw, E1000_RCTL);
2204         rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2205         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2206 }
2207
2208 static void
2209 eth_igb_promiscuous_disable(struct rte_eth_dev *dev)
2210 {
2211         struct e1000_hw *hw =
2212                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2213         uint32_t rctl;
2214
2215         rctl = E1000_READ_REG(hw, E1000_RCTL);
2216         rctl &= (~E1000_RCTL_UPE);
2217         if (dev->data->all_multicast == 1)
2218                 rctl |= E1000_RCTL_MPE;
2219         else
2220                 rctl &= (~E1000_RCTL_MPE);
2221         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2222 }
2223
2224 static void
2225 eth_igb_allmulticast_enable(struct rte_eth_dev *dev)
2226 {
2227         struct e1000_hw *hw =
2228                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2229         uint32_t rctl;
2230
2231         rctl = E1000_READ_REG(hw, E1000_RCTL);
2232         rctl |= E1000_RCTL_MPE;
2233         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2234 }
2235
2236 static void
2237 eth_igb_allmulticast_disable(struct rte_eth_dev *dev)
2238 {
2239         struct e1000_hw *hw =
2240                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2241         uint32_t rctl;
2242
2243         if (dev->data->promiscuous == 1)
2244                 return; /* must remain in all_multicast mode */
2245         rctl = E1000_READ_REG(hw, E1000_RCTL);
2246         rctl &= (~E1000_RCTL_MPE);
2247         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2248 }
2249
2250 static int
2251 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2252 {
2253         struct e1000_hw *hw =
2254                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2255         struct e1000_vfta * shadow_vfta =
2256                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2257         uint32_t vfta;
2258         uint32_t vid_idx;
2259         uint32_t vid_bit;
2260
2261         vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) &
2262                               E1000_VFTA_ENTRY_MASK);
2263         vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK));
2264         vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx);
2265         if (on)
2266                 vfta |= vid_bit;
2267         else
2268                 vfta &= ~vid_bit;
2269         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta);
2270
2271         /* update local VFTA copy */
2272         shadow_vfta->vfta[vid_idx] = vfta;
2273
2274         return 0;
2275 }
2276
2277 static int
2278 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev,
2279                       enum rte_vlan_type vlan_type,
2280                       uint16_t tpid)
2281 {
2282         struct e1000_hw *hw =
2283                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2284         uint32_t reg = ETHER_TYPE_VLAN;
2285         int ret = 0;
2286
2287         switch (vlan_type) {
2288         case ETH_VLAN_TYPE_INNER:
2289                 reg |= (tpid << 16);
2290                 E1000_WRITE_REG(hw, E1000_VET, reg);
2291                 break;
2292         default:
2293                 ret = -EINVAL;
2294                 PMD_DRV_LOG(ERR, "Unsupported vlan type %d\n", vlan_type);
2295                 break;
2296         }
2297
2298         return ret;
2299 }
2300
2301 static void
2302 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev)
2303 {
2304         struct e1000_hw *hw =
2305                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2306         uint32_t reg;
2307
2308         /* Filter Table Disable */
2309         reg = E1000_READ_REG(hw, E1000_RCTL);
2310         reg &= ~E1000_RCTL_CFIEN;
2311         reg &= ~E1000_RCTL_VFE;
2312         E1000_WRITE_REG(hw, E1000_RCTL, reg);
2313 }
2314
2315 static void
2316 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev)
2317 {
2318         struct e1000_hw *hw =
2319                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2320         struct e1000_vfta * shadow_vfta =
2321                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2322         uint32_t reg;
2323         int i;
2324
2325         /* Filter Table Enable, CFI not used for packet acceptance */
2326         reg = E1000_READ_REG(hw, E1000_RCTL);
2327         reg &= ~E1000_RCTL_CFIEN;
2328         reg |= E1000_RCTL_VFE;
2329         E1000_WRITE_REG(hw, E1000_RCTL, reg);
2330
2331         /* restore VFTA table */
2332         for (i = 0; i < IGB_VFTA_SIZE; i++)
2333                 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]);
2334 }
2335
2336 static void
2337 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev)
2338 {
2339         struct e1000_hw *hw =
2340                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2341         uint32_t reg;
2342
2343         /* VLAN Mode Disable */
2344         reg = E1000_READ_REG(hw, E1000_CTRL);
2345         reg &= ~E1000_CTRL_VME;
2346         E1000_WRITE_REG(hw, E1000_CTRL, reg);
2347 }
2348
2349 static void
2350 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev)
2351 {
2352         struct e1000_hw *hw =
2353                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2354         uint32_t reg;
2355
2356         /* VLAN Mode Enable */
2357         reg = E1000_READ_REG(hw, E1000_CTRL);
2358         reg |= E1000_CTRL_VME;
2359         E1000_WRITE_REG(hw, E1000_CTRL, reg);
2360 }
2361
2362 static void
2363 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev)
2364 {
2365         struct e1000_hw *hw =
2366                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2367         uint32_t reg;
2368
2369         /* CTRL_EXT: Extended VLAN */
2370         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
2371         reg &= ~E1000_CTRL_EXT_EXTEND_VLAN;
2372         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
2373
2374         /* Update maximum packet length */
2375         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
2376                 E1000_WRITE_REG(hw, E1000_RLPML,
2377                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
2378                                                 VLAN_TAG_SIZE);
2379 }
2380
2381 static void
2382 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev)
2383 {
2384         struct e1000_hw *hw =
2385                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2386         uint32_t reg;
2387
2388         /* CTRL_EXT: Extended VLAN */
2389         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
2390         reg |= E1000_CTRL_EXT_EXTEND_VLAN;
2391         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
2392
2393         /* Update maximum packet length */
2394         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
2395                 E1000_WRITE_REG(hw, E1000_RLPML,
2396                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
2397                                                 2 * VLAN_TAG_SIZE);
2398 }
2399
2400 static void
2401 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2402 {
2403         if(mask & ETH_VLAN_STRIP_MASK){
2404                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
2405                         igb_vlan_hw_strip_enable(dev);
2406                 else
2407                         igb_vlan_hw_strip_disable(dev);
2408         }
2409
2410         if(mask & ETH_VLAN_FILTER_MASK){
2411                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
2412                         igb_vlan_hw_filter_enable(dev);
2413                 else
2414                         igb_vlan_hw_filter_disable(dev);
2415         }
2416
2417         if(mask & ETH_VLAN_EXTEND_MASK){
2418                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
2419                         igb_vlan_hw_extend_enable(dev);
2420                 else
2421                         igb_vlan_hw_extend_disable(dev);
2422         }
2423 }
2424
2425
2426 /**
2427  * It enables the interrupt mask and then enable the interrupt.
2428  *
2429  * @param dev
2430  *  Pointer to struct rte_eth_dev.
2431  *
2432  * @return
2433  *  - On success, zero.
2434  *  - On failure, a negative value.
2435  */
2436 static int
2437 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev)
2438 {
2439         struct e1000_interrupt *intr =
2440                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2441
2442         intr->mask |= E1000_ICR_LSC;
2443
2444         return 0;
2445 }
2446
2447 /* It clears the interrupt causes and enables the interrupt.
2448  * It will be called once only during nic initialized.
2449  *
2450  * @param dev
2451  *  Pointer to struct rte_eth_dev.
2452  *
2453  * @return
2454  *  - On success, zero.
2455  *  - On failure, a negative value.
2456  */
2457 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev)
2458 {
2459         uint32_t mask, regval;
2460         struct e1000_hw *hw =
2461                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2462         struct rte_eth_dev_info dev_info;
2463
2464         memset(&dev_info, 0, sizeof(dev_info));
2465         eth_igb_infos_get(dev, &dev_info);
2466
2467         mask = 0xFFFFFFFF >> (32 - dev_info.max_rx_queues);
2468         regval = E1000_READ_REG(hw, E1000_EIMS);
2469         E1000_WRITE_REG(hw, E1000_EIMS, regval | mask);
2470
2471         return 0;
2472 }
2473
2474 /*
2475  * It reads ICR and gets interrupt causes, check it and set a bit flag
2476  * to update link status.
2477  *
2478  * @param dev
2479  *  Pointer to struct rte_eth_dev.
2480  *
2481  * @return
2482  *  - On success, zero.
2483  *  - On failure, a negative value.
2484  */
2485 static int
2486 eth_igb_interrupt_get_status(struct rte_eth_dev *dev)
2487 {
2488         uint32_t icr;
2489         struct e1000_hw *hw =
2490                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2491         struct e1000_interrupt *intr =
2492                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2493
2494         igb_intr_disable(hw);
2495
2496         /* read-on-clear nic registers here */
2497         icr = E1000_READ_REG(hw, E1000_ICR);
2498
2499         intr->flags = 0;
2500         if (icr & E1000_ICR_LSC) {
2501                 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
2502         }
2503
2504         if (icr & E1000_ICR_VMMB)
2505                 intr->flags |= E1000_FLAG_MAILBOX;
2506
2507         return 0;
2508 }
2509
2510 /*
2511  * It executes link_update after knowing an interrupt is prsent.
2512  *
2513  * @param dev
2514  *  Pointer to struct rte_eth_dev.
2515  *
2516  * @return
2517  *  - On success, zero.
2518  *  - On failure, a negative value.
2519  */
2520 static int
2521 eth_igb_interrupt_action(struct rte_eth_dev *dev)
2522 {
2523         struct e1000_hw *hw =
2524                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2525         struct e1000_interrupt *intr =
2526                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2527         uint32_t tctl, rctl;
2528         struct rte_eth_link link;
2529         int ret;
2530
2531         if (intr->flags & E1000_FLAG_MAILBOX) {
2532                 igb_pf_mbx_process(dev);
2533                 intr->flags &= ~E1000_FLAG_MAILBOX;
2534         }
2535
2536         igb_intr_enable(dev);
2537         rte_intr_enable(&(dev->pci_dev->intr_handle));
2538
2539         if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) {
2540                 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE;
2541
2542                 /* set get_link_status to check register later */
2543                 hw->mac.get_link_status = 1;
2544                 ret = eth_igb_link_update(dev, 0);
2545
2546                 /* check if link has changed */
2547                 if (ret < 0)
2548                         return 0;
2549
2550                 memset(&link, 0, sizeof(link));
2551                 rte_igb_dev_atomic_read_link_status(dev, &link);
2552                 if (link.link_status) {
2553                         PMD_INIT_LOG(INFO,
2554                                      " Port %d: Link Up - speed %u Mbps - %s",
2555                                      dev->data->port_id,
2556                                      (unsigned)link.link_speed,
2557                                      link.link_duplex == ETH_LINK_FULL_DUPLEX ?
2558                                      "full-duplex" : "half-duplex");
2559                 } else {
2560                         PMD_INIT_LOG(INFO, " Port %d: Link Down",
2561                                      dev->data->port_id);
2562                 }
2563
2564                 PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d",
2565                              dev->pci_dev->addr.domain,
2566                              dev->pci_dev->addr.bus,
2567                              dev->pci_dev->addr.devid,
2568                              dev->pci_dev->addr.function);
2569                 tctl = E1000_READ_REG(hw, E1000_TCTL);
2570                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2571                 if (link.link_status) {
2572                         /* enable Tx/Rx */
2573                         tctl |= E1000_TCTL_EN;
2574                         rctl |= E1000_RCTL_EN;
2575                 } else {
2576                         /* disable Tx/Rx */
2577                         tctl &= ~E1000_TCTL_EN;
2578                         rctl &= ~E1000_RCTL_EN;
2579                 }
2580                 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
2581                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2582                 E1000_WRITE_FLUSH(hw);
2583                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
2584         }
2585
2586         return 0;
2587 }
2588
2589 /**
2590  * Interrupt handler which shall be registered at first.
2591  *
2592  * @param handle
2593  *  Pointer to interrupt handle.
2594  * @param param
2595  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2596  *
2597  * @return
2598  *  void
2599  */
2600 static void
2601 eth_igb_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
2602                                                         void *param)
2603 {
2604         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2605
2606         eth_igb_interrupt_get_status(dev);
2607         eth_igb_interrupt_action(dev);
2608 }
2609
2610 static int
2611 eth_igb_led_on(struct rte_eth_dev *dev)
2612 {
2613         struct e1000_hw *hw;
2614
2615         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2616         return e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP;
2617 }
2618
2619 static int
2620 eth_igb_led_off(struct rte_eth_dev *dev)
2621 {
2622         struct e1000_hw *hw;
2623
2624         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2625         return e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP;
2626 }
2627
2628 static int
2629 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2630 {
2631         struct e1000_hw *hw;
2632         uint32_t ctrl;
2633         int tx_pause;
2634         int rx_pause;
2635
2636         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2637         fc_conf->pause_time = hw->fc.pause_time;
2638         fc_conf->high_water = hw->fc.high_water;
2639         fc_conf->low_water = hw->fc.low_water;
2640         fc_conf->send_xon = hw->fc.send_xon;
2641         fc_conf->autoneg = hw->mac.autoneg;
2642
2643         /*
2644          * Return rx_pause and tx_pause status according to actual setting of
2645          * the TFCE and RFCE bits in the CTRL register.
2646          */
2647         ctrl = E1000_READ_REG(hw, E1000_CTRL);
2648         if (ctrl & E1000_CTRL_TFCE)
2649                 tx_pause = 1;
2650         else
2651                 tx_pause = 0;
2652
2653         if (ctrl & E1000_CTRL_RFCE)
2654                 rx_pause = 1;
2655         else
2656                 rx_pause = 0;
2657
2658         if (rx_pause && tx_pause)
2659                 fc_conf->mode = RTE_FC_FULL;
2660         else if (rx_pause)
2661                 fc_conf->mode = RTE_FC_RX_PAUSE;
2662         else if (tx_pause)
2663                 fc_conf->mode = RTE_FC_TX_PAUSE;
2664         else
2665                 fc_conf->mode = RTE_FC_NONE;
2666
2667         return 0;
2668 }
2669
2670 static int
2671 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2672 {
2673         struct e1000_hw *hw;
2674         int err;
2675         enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = {
2676                 e1000_fc_none,
2677                 e1000_fc_rx_pause,
2678                 e1000_fc_tx_pause,
2679                 e1000_fc_full
2680         };
2681         uint32_t rx_buf_size;
2682         uint32_t max_high_water;
2683         uint32_t rctl;
2684
2685         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2686         if (fc_conf->autoneg != hw->mac.autoneg)
2687                 return -ENOTSUP;
2688         rx_buf_size = igb_get_rx_buffer_size(hw);
2689         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2690
2691         /* At least reserve one Ethernet frame for watermark */
2692         max_high_water = rx_buf_size - ETHER_MAX_LEN;
2693         if ((fc_conf->high_water > max_high_water) ||
2694             (fc_conf->high_water < fc_conf->low_water)) {
2695                 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value");
2696                 PMD_INIT_LOG(ERR, "high water must <=  0x%x", max_high_water);
2697                 return -EINVAL;
2698         }
2699
2700         hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode];
2701         hw->fc.pause_time     = fc_conf->pause_time;
2702         hw->fc.high_water     = fc_conf->high_water;
2703         hw->fc.low_water      = fc_conf->low_water;
2704         hw->fc.send_xon       = fc_conf->send_xon;
2705
2706         err = e1000_setup_link_generic(hw);
2707         if (err == E1000_SUCCESS) {
2708
2709                 /* check if we want to forward MAC frames - driver doesn't have native
2710                  * capability to do that, so we'll write the registers ourselves */
2711
2712                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2713
2714                 /* set or clear MFLCN.PMCF bit depending on configuration */
2715                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2716                         rctl |= E1000_RCTL_PMCF;
2717                 else
2718                         rctl &= ~E1000_RCTL_PMCF;
2719
2720                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2721                 E1000_WRITE_FLUSH(hw);
2722
2723                 return 0;
2724         }
2725
2726         PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err);
2727         return -EIO;
2728 }
2729
2730 #define E1000_RAH_POOLSEL_SHIFT      (18)
2731 static void
2732 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2733                 uint32_t index, __rte_unused uint32_t pool)
2734 {
2735         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2736         uint32_t rah;
2737
2738         e1000_rar_set(hw, mac_addr->addr_bytes, index);
2739         rah = E1000_READ_REG(hw, E1000_RAH(index));
2740         rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool));
2741         E1000_WRITE_REG(hw, E1000_RAH(index), rah);
2742 }
2743
2744 static void
2745 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index)
2746 {
2747         uint8_t addr[ETHER_ADDR_LEN];
2748         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2749
2750         memset(addr, 0, sizeof(addr));
2751
2752         e1000_rar_set(hw, addr, index);
2753 }
2754
2755 static void
2756 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev,
2757                                 struct ether_addr *addr)
2758 {
2759         eth_igb_rar_clear(dev, 0);
2760
2761         eth_igb_rar_set(dev, (void *)addr, 0, 0);
2762 }
2763 /*
2764  * Virtual Function operations
2765  */
2766 static void
2767 igbvf_intr_disable(struct e1000_hw *hw)
2768 {
2769         PMD_INIT_FUNC_TRACE();
2770
2771         /* Clear interrupt mask to stop from interrupts being generated */
2772         E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF);
2773
2774         E1000_WRITE_FLUSH(hw);
2775 }
2776
2777 static void
2778 igbvf_stop_adapter(struct rte_eth_dev *dev)
2779 {
2780         u32 reg_val;
2781         u16 i;
2782         struct rte_eth_dev_info dev_info;
2783         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2784
2785         memset(&dev_info, 0, sizeof(dev_info));
2786         eth_igbvf_infos_get(dev, &dev_info);
2787
2788         /* Clear interrupt mask to stop from interrupts being generated */
2789         igbvf_intr_disable(hw);
2790
2791         /* Clear any pending interrupts, flush previous writes */
2792         E1000_READ_REG(hw, E1000_EICR);
2793
2794         /* Disable the transmit unit.  Each queue must be disabled. */
2795         for (i = 0; i < dev_info.max_tx_queues; i++)
2796                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH);
2797
2798         /* Disable the receive unit by stopping each queue */
2799         for (i = 0; i < dev_info.max_rx_queues; i++) {
2800                 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i));
2801                 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE;
2802                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val);
2803                 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE)
2804                         ;
2805         }
2806
2807         /* flush all queues disables */
2808         E1000_WRITE_FLUSH(hw);
2809         msec_delay(2);
2810 }
2811
2812 static int eth_igbvf_link_update(struct e1000_hw *hw)
2813 {
2814         struct e1000_mbx_info *mbx = &hw->mbx;
2815         struct e1000_mac_info *mac = &hw->mac;
2816         int ret_val = E1000_SUCCESS;
2817
2818         PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf");
2819
2820         /*
2821          * We only want to run this if there has been a rst asserted.
2822          * in this case that could mean a link change, device reset,
2823          * or a virtual function reset
2824          */
2825
2826         /* If we were hit with a reset or timeout drop the link */
2827         if (!e1000_check_for_rst(hw, 0) || !mbx->timeout)
2828                 mac->get_link_status = TRUE;
2829
2830         if (!mac->get_link_status)
2831                 goto out;
2832
2833         /* if link status is down no point in checking to see if pf is up */
2834         if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU))
2835                 goto out;
2836
2837         /* if we passed all the tests above then the link is up and we no
2838          * longer need to check for link */
2839         mac->get_link_status = FALSE;
2840
2841 out:
2842         return ret_val;
2843 }
2844
2845
2846 static int
2847 igbvf_dev_configure(struct rte_eth_dev *dev)
2848 {
2849         struct rte_eth_conf* conf = &dev->data->dev_conf;
2850
2851         PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
2852                      dev->data->port_id);
2853
2854         /*
2855          * VF has no ability to enable/disable HW CRC
2856          * Keep the persistent behavior the same as Host PF
2857          */
2858 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC
2859         if (!conf->rxmode.hw_strip_crc) {
2860                 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip");
2861                 conf->rxmode.hw_strip_crc = 1;
2862         }
2863 #else
2864         if (conf->rxmode.hw_strip_crc) {
2865                 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip");
2866                 conf->rxmode.hw_strip_crc = 0;
2867         }
2868 #endif
2869
2870         return 0;
2871 }
2872
2873 static int
2874 igbvf_dev_start(struct rte_eth_dev *dev)
2875 {
2876         struct e1000_hw *hw =
2877                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2878         struct e1000_adapter *adapter =
2879                 E1000_DEV_PRIVATE(dev->data->dev_private);
2880         int ret;
2881
2882         PMD_INIT_FUNC_TRACE();
2883
2884         hw->mac.ops.reset_hw(hw);
2885         adapter->stopped = 0;
2886
2887         /* Set all vfta */
2888         igbvf_set_vfta_all(dev,1);
2889
2890         eth_igbvf_tx_init(dev);
2891
2892         /* This can fail when allocating mbufs for descriptor rings */
2893         ret = eth_igbvf_rx_init(dev);
2894         if (ret) {
2895                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
2896                 igb_dev_clear_queues(dev);
2897                 return ret;
2898         }
2899
2900         return 0;
2901 }
2902
2903 static void
2904 igbvf_dev_stop(struct rte_eth_dev *dev)
2905 {
2906         PMD_INIT_FUNC_TRACE();
2907
2908         igbvf_stop_adapter(dev);
2909
2910         /*
2911           * Clear what we set, but we still keep shadow_vfta to
2912           * restore after device starts
2913           */
2914         igbvf_set_vfta_all(dev,0);
2915
2916         igb_dev_clear_queues(dev);
2917 }
2918
2919 static void
2920 igbvf_dev_close(struct rte_eth_dev *dev)
2921 {
2922         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2923         struct e1000_adapter *adapter =
2924                 E1000_DEV_PRIVATE(dev->data->dev_private);
2925         struct ether_addr addr;
2926
2927         PMD_INIT_FUNC_TRACE();
2928
2929         e1000_reset_hw(hw);
2930
2931         igbvf_dev_stop(dev);
2932         adapter->stopped = 1;
2933         igb_dev_free_queues(dev);
2934
2935         /**
2936          * reprogram the RAR with a zero mac address,
2937          * to ensure that the VF traffic goes to the PF
2938          * after stop, close and detach of the VF.
2939          **/
2940
2941         memset(&addr, 0, sizeof(addr));
2942         igbvf_default_mac_addr_set(dev, &addr);
2943 }
2944
2945 static void
2946 igbvf_promiscuous_enable(struct rte_eth_dev *dev)
2947 {
2948         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2949
2950         /* Set both unicast and multicast promisc */
2951         e1000_promisc_set_vf(hw, e1000_promisc_enabled);
2952 }
2953
2954 static void
2955 igbvf_promiscuous_disable(struct rte_eth_dev *dev)
2956 {
2957         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2958
2959         /* If in allmulticast mode leave multicast promisc */
2960         if (dev->data->all_multicast == 1)
2961                 e1000_promisc_set_vf(hw, e1000_promisc_multicast);
2962         else
2963                 e1000_promisc_set_vf(hw, e1000_promisc_disabled);
2964 }
2965
2966 static void
2967 igbvf_allmulticast_enable(struct rte_eth_dev *dev)
2968 {
2969         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2970
2971         /* In promiscuous mode multicast promisc already set */
2972         if (dev->data->promiscuous == 0)
2973                 e1000_promisc_set_vf(hw, e1000_promisc_multicast);
2974 }
2975
2976 static void
2977 igbvf_allmulticast_disable(struct rte_eth_dev *dev)
2978 {
2979         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2980
2981         /* In promiscuous mode leave multicast promisc enabled */
2982         if (dev->data->promiscuous == 0)
2983                 e1000_promisc_set_vf(hw, e1000_promisc_disabled);
2984 }
2985
2986 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on)
2987 {
2988         struct e1000_mbx_info *mbx = &hw->mbx;
2989         uint32_t msgbuf[2];
2990         s32 err;
2991
2992         /* After set vlan, vlan strip will also be enabled in igb driver*/
2993         msgbuf[0] = E1000_VF_SET_VLAN;
2994         msgbuf[1] = vid;
2995         /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */
2996         if (on)
2997                 msgbuf[0] |= E1000_VF_SET_VLAN_ADD;
2998
2999         err = mbx->ops.write_posted(hw, msgbuf, 2, 0);
3000         if (err)
3001                 goto mbx_err;
3002
3003         err = mbx->ops.read_posted(hw, msgbuf, 2, 0);
3004         if (err)
3005                 goto mbx_err;
3006
3007         msgbuf[0] &= ~E1000_VT_MSGTYPE_CTS;
3008         if (msgbuf[0] == (E1000_VF_SET_VLAN | E1000_VT_MSGTYPE_NACK))
3009                 err = -EINVAL;
3010
3011 mbx_err:
3012         return err;
3013 }
3014
3015 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on)
3016 {
3017         struct e1000_hw *hw =
3018                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3019         struct e1000_vfta * shadow_vfta =
3020                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
3021         int i = 0, j = 0, vfta = 0, mask = 1;
3022
3023         for (i = 0; i < IGB_VFTA_SIZE; i++){
3024                 vfta = shadow_vfta->vfta[i];
3025                 if(vfta){
3026                         mask = 1;
3027                         for (j = 0; j < 32; j++){
3028                                 if(vfta & mask)
3029                                         igbvf_set_vfta(hw,
3030                                                 (uint16_t)((i<<5)+j), on);
3031                                 mask<<=1;
3032                         }
3033                 }
3034         }
3035
3036 }
3037
3038 static int
3039 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
3040 {
3041         struct e1000_hw *hw =
3042                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3043         struct e1000_vfta * shadow_vfta =
3044                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
3045         uint32_t vid_idx = 0;
3046         uint32_t vid_bit = 0;
3047         int ret = 0;
3048
3049         PMD_INIT_FUNC_TRACE();
3050
3051         /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/
3052         ret = igbvf_set_vfta(hw, vlan_id, !!on);
3053         if(ret){
3054                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
3055                 return ret;
3056         }
3057         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
3058         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
3059
3060         /*Save what we set and retore it after device reset*/
3061         if (on)
3062                 shadow_vfta->vfta[vid_idx] |= vid_bit;
3063         else
3064                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
3065
3066         return 0;
3067 }
3068
3069 static void
3070 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr)
3071 {
3072         struct e1000_hw *hw =
3073                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3074
3075         /* index is not used by rar_set() */
3076         hw->mac.ops.rar_set(hw, (void *)addr, 0);
3077 }
3078
3079
3080 static int
3081 eth_igb_rss_reta_update(struct rte_eth_dev *dev,
3082                         struct rte_eth_rss_reta_entry64 *reta_conf,
3083                         uint16_t reta_size)
3084 {
3085         uint8_t i, j, mask;
3086         uint32_t reta, r;
3087         uint16_t idx, shift;
3088         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3089
3090         if (reta_size != ETH_RSS_RETA_SIZE_128) {
3091                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3092                         "(%d) doesn't match the number hardware can supported "
3093                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
3094                 return -EINVAL;
3095         }
3096
3097         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
3098                 idx = i / RTE_RETA_GROUP_SIZE;
3099                 shift = i % RTE_RETA_GROUP_SIZE;
3100                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
3101                                                 IGB_4_BIT_MASK);
3102                 if (!mask)
3103                         continue;
3104                 if (mask == IGB_4_BIT_MASK)
3105                         r = 0;
3106                 else
3107                         r = E1000_READ_REG(hw, E1000_RETA(i >> 2));
3108                 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) {
3109                         if (mask & (0x1 << j))
3110                                 reta |= reta_conf[idx].reta[shift + j] <<
3111                                                         (CHAR_BIT * j);
3112                         else
3113                                 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j));
3114                 }
3115                 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta);
3116         }
3117
3118         return 0;
3119 }
3120
3121 static int
3122 eth_igb_rss_reta_query(struct rte_eth_dev *dev,
3123                        struct rte_eth_rss_reta_entry64 *reta_conf,
3124                        uint16_t reta_size)
3125 {
3126         uint8_t i, j, mask;
3127         uint32_t reta;
3128         uint16_t idx, shift;
3129         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3130
3131         if (reta_size != ETH_RSS_RETA_SIZE_128) {
3132                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3133                         "(%d) doesn't match the number hardware can supported "
3134                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
3135                 return -EINVAL;
3136         }
3137
3138         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
3139                 idx = i / RTE_RETA_GROUP_SIZE;
3140                 shift = i % RTE_RETA_GROUP_SIZE;
3141                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
3142                                                 IGB_4_BIT_MASK);
3143                 if (!mask)
3144                         continue;
3145                 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2));
3146                 for (j = 0; j < IGB_4_BIT_WIDTH; j++) {
3147                         if (mask & (0x1 << j))
3148                                 reta_conf[idx].reta[shift + j] =
3149                                         ((reta >> (CHAR_BIT * j)) &
3150                                                 IGB_8_BIT_MASK);
3151                 }
3152         }
3153
3154         return 0;
3155 }
3156
3157 #define MAC_TYPE_FILTER_SUP(type)    do {\
3158         if ((type) != e1000_82580 && (type) != e1000_i350 &&\
3159                 (type) != e1000_82576)\
3160                 return -ENOTSUP;\
3161 } while (0)
3162
3163 static int
3164 eth_igb_syn_filter_set(struct rte_eth_dev *dev,
3165                         struct rte_eth_syn_filter *filter,
3166                         bool add)
3167 {
3168         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3169         uint32_t synqf, rfctl;
3170
3171         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
3172                 return -EINVAL;
3173
3174         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
3175
3176         if (add) {
3177                 if (synqf & E1000_SYN_FILTER_ENABLE)
3178                         return -EINVAL;
3179
3180                 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) &
3181                         E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE);
3182
3183                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
3184                 if (filter->hig_pri)
3185                         rfctl |= E1000_RFCTL_SYNQFP;
3186                 else
3187                         rfctl &= ~E1000_RFCTL_SYNQFP;
3188
3189                 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl);
3190         } else {
3191                 if (!(synqf & E1000_SYN_FILTER_ENABLE))
3192                         return -ENOENT;
3193                 synqf = 0;
3194         }
3195
3196         E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf);
3197         E1000_WRITE_FLUSH(hw);
3198         return 0;
3199 }
3200
3201 static int
3202 eth_igb_syn_filter_get(struct rte_eth_dev *dev,
3203                         struct rte_eth_syn_filter *filter)
3204 {
3205         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3206         uint32_t synqf, rfctl;
3207
3208         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
3209         if (synqf & E1000_SYN_FILTER_ENABLE) {
3210                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
3211                 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0;
3212                 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >>
3213                                 E1000_SYN_FILTER_QUEUE_SHIFT);
3214                 return 0;
3215         }
3216
3217         return -ENOENT;
3218 }
3219
3220 static int
3221 eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
3222                         enum rte_filter_op filter_op,
3223                         void *arg)
3224 {
3225         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3226         int ret;
3227
3228         MAC_TYPE_FILTER_SUP(hw->mac.type);
3229
3230         if (filter_op == RTE_ETH_FILTER_NOP)
3231                 return 0;
3232
3233         if (arg == NULL) {
3234                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
3235                             filter_op);
3236                 return -EINVAL;
3237         }
3238
3239         switch (filter_op) {
3240         case RTE_ETH_FILTER_ADD:
3241                 ret = eth_igb_syn_filter_set(dev,
3242                                 (struct rte_eth_syn_filter *)arg,
3243                                 TRUE);
3244                 break;
3245         case RTE_ETH_FILTER_DELETE:
3246                 ret = eth_igb_syn_filter_set(dev,
3247                                 (struct rte_eth_syn_filter *)arg,
3248                                 FALSE);
3249                 break;
3250         case RTE_ETH_FILTER_GET:
3251                 ret = eth_igb_syn_filter_get(dev,
3252                                 (struct rte_eth_syn_filter *)arg);
3253                 break;
3254         default:
3255                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
3256                 ret = -EINVAL;
3257                 break;
3258         }
3259
3260         return ret;
3261 }
3262
3263 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
3264         if ((type) != e1000_82580 && (type) != e1000_i350)\
3265                 return -ENOSYS; \
3266 } while (0)
3267
3268 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/
3269 static inline int
3270 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter,
3271                         struct e1000_2tuple_filter_info *filter_info)
3272 {
3273         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
3274                 return -EINVAL;
3275         if (filter->priority > E1000_2TUPLE_MAX_PRI)
3276                 return -EINVAL;  /* filter index is out of range. */
3277         if (filter->tcp_flags > TCP_FLAG_ALL)
3278                 return -EINVAL;  /* flags is invalid. */
3279
3280         switch (filter->dst_port_mask) {
3281         case UINT16_MAX:
3282                 filter_info->dst_port_mask = 0;
3283                 filter_info->dst_port = filter->dst_port;
3284                 break;
3285         case 0:
3286                 filter_info->dst_port_mask = 1;
3287                 break;
3288         default:
3289                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
3290                 return -EINVAL;
3291         }
3292
3293         switch (filter->proto_mask) {
3294         case UINT8_MAX:
3295                 filter_info->proto_mask = 0;
3296                 filter_info->proto = filter->proto;
3297                 break;
3298         case 0:
3299                 filter_info->proto_mask = 1;
3300                 break;
3301         default:
3302                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
3303                 return -EINVAL;
3304         }
3305
3306         filter_info->priority = (uint8_t)filter->priority;
3307         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
3308                 filter_info->tcp_flags = filter->tcp_flags;
3309         else
3310                 filter_info->tcp_flags = 0;
3311
3312         return 0;
3313 }
3314
3315 static inline struct e1000_2tuple_filter *
3316 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list,
3317                         struct e1000_2tuple_filter_info *key)
3318 {
3319         struct e1000_2tuple_filter *it;
3320
3321         TAILQ_FOREACH(it, filter_list, entries) {
3322                 if (memcmp(key, &it->filter_info,
3323                         sizeof(struct e1000_2tuple_filter_info)) == 0) {
3324                         return it;
3325                 }
3326         }
3327         return NULL;
3328 }
3329
3330 /*
3331  * igb_add_2tuple_filter - add a 2tuple filter
3332  *
3333  * @param
3334  * dev: Pointer to struct rte_eth_dev.
3335  * ntuple_filter: ponter to the filter that will be added.
3336  *
3337  * @return
3338  *    - On success, zero.
3339  *    - On failure, a negative value.
3340  */
3341 static int
3342 igb_add_2tuple_filter(struct rte_eth_dev *dev,
3343                         struct rte_eth_ntuple_filter *ntuple_filter)
3344 {
3345         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3346         struct e1000_filter_info *filter_info =
3347                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3348         struct e1000_2tuple_filter *filter;
3349         uint32_t ttqf = E1000_TTQF_DISABLE_MASK;
3350         uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP;
3351         int i, ret;
3352
3353         filter = rte_zmalloc("e1000_2tuple_filter",
3354                         sizeof(struct e1000_2tuple_filter), 0);
3355         if (filter == NULL)
3356                 return -ENOMEM;
3357
3358         ret = ntuple_filter_to_2tuple(ntuple_filter,
3359                                       &filter->filter_info);
3360         if (ret < 0) {
3361                 rte_free(filter);
3362                 return ret;
3363         }
3364         if (igb_2tuple_filter_lookup(&filter_info->twotuple_list,
3365                                          &filter->filter_info) != NULL) {
3366                 PMD_DRV_LOG(ERR, "filter exists.");
3367                 rte_free(filter);
3368                 return -EEXIST;
3369         }
3370         filter->queue = ntuple_filter->queue;
3371
3372         /*
3373          * look for an unused 2tuple filter index,
3374          * and insert the filter to list.
3375          */
3376         for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) {
3377                 if (!(filter_info->twotuple_mask & (1 << i))) {
3378                         filter_info->twotuple_mask |= 1 << i;
3379                         filter->index = i;
3380                         TAILQ_INSERT_TAIL(&filter_info->twotuple_list,
3381                                           filter,
3382                                           entries);
3383                         break;
3384                 }
3385         }
3386         if (i >= E1000_MAX_TTQF_FILTERS) {
3387                 PMD_DRV_LOG(ERR, "2tuple filters are full.");
3388                 rte_free(filter);
3389                 return -ENOSYS;
3390         }
3391
3392         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
3393         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
3394                 imir |= E1000_IMIR_PORT_BP;
3395         else
3396                 imir &= ~E1000_IMIR_PORT_BP;
3397
3398         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
3399
3400         ttqf |= E1000_TTQF_QUEUE_ENABLE;
3401         ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT);
3402         ttqf |= (uint32_t)(filter->filter_info.proto & E1000_TTQF_PROTOCOL_MASK);
3403         if (filter->filter_info.proto_mask == 0)
3404                 ttqf &= ~E1000_TTQF_MASK_ENABLE;
3405
3406         /* tcp flags bits setting. */
3407         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
3408                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
3409                         imir_ext |= E1000_IMIREXT_CTRL_URG;
3410                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
3411                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
3412                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
3413                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
3414                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
3415                         imir_ext |= E1000_IMIREXT_CTRL_RST;
3416                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
3417                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
3418                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
3419                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
3420         } else
3421                 imir_ext |= E1000_IMIREXT_CTRL_BP;
3422         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
3423         E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf);
3424         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
3425         return 0;
3426 }
3427
3428 /*
3429  * igb_remove_2tuple_filter - remove a 2tuple filter
3430  *
3431  * @param
3432  * dev: Pointer to struct rte_eth_dev.
3433  * ntuple_filter: ponter to the filter that will be removed.
3434  *
3435  * @return
3436  *    - On success, zero.
3437  *    - On failure, a negative value.
3438  */
3439 static int
3440 igb_remove_2tuple_filter(struct rte_eth_dev *dev,
3441                         struct rte_eth_ntuple_filter *ntuple_filter)
3442 {
3443         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3444         struct e1000_filter_info *filter_info =
3445                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3446         struct e1000_2tuple_filter_info filter_2tuple;
3447         struct e1000_2tuple_filter *filter;
3448         int ret;
3449
3450         memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info));
3451         ret = ntuple_filter_to_2tuple(ntuple_filter,
3452                                       &filter_2tuple);
3453         if (ret < 0)
3454                 return ret;
3455
3456         filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list,
3457                                          &filter_2tuple);
3458         if (filter == NULL) {
3459                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3460                 return -ENOENT;
3461         }
3462
3463         filter_info->twotuple_mask &= ~(1 << filter->index);
3464         TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries);
3465         rte_free(filter);
3466
3467         E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK);
3468         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
3469         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
3470         return 0;
3471 }
3472
3473 static inline struct e1000_flex_filter *
3474 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list,
3475                         struct e1000_flex_filter_info *key)
3476 {
3477         struct e1000_flex_filter *it;
3478
3479         TAILQ_FOREACH(it, filter_list, entries) {
3480                 if (memcmp(key, &it->filter_info,
3481                         sizeof(struct e1000_flex_filter_info)) == 0)
3482                         return it;
3483         }
3484
3485         return NULL;
3486 }
3487
3488 static int
3489 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
3490                         struct rte_eth_flex_filter *filter,
3491                         bool add)
3492 {
3493         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3494         struct e1000_filter_info *filter_info =
3495                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3496         struct e1000_flex_filter *flex_filter, *it;
3497         uint32_t wufc, queueing, mask;
3498         uint32_t reg_off;
3499         uint8_t shift, i, j = 0;
3500
3501         flex_filter = rte_zmalloc("e1000_flex_filter",
3502                         sizeof(struct e1000_flex_filter), 0);
3503         if (flex_filter == NULL)
3504                 return -ENOMEM;
3505
3506         flex_filter->filter_info.len = filter->len;
3507         flex_filter->filter_info.priority = filter->priority;
3508         memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len);
3509         for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) {
3510                 mask = 0;
3511                 /* reverse bits in flex filter's mask*/
3512                 for (shift = 0; shift < CHAR_BIT; shift++) {
3513                         if (filter->mask[i] & (0x01 << shift))
3514                                 mask |= (0x80 >> shift);
3515                 }
3516                 flex_filter->filter_info.mask[i] = mask;
3517         }
3518
3519         wufc = E1000_READ_REG(hw, E1000_WUFC);
3520         if (flex_filter->index < E1000_MAX_FHFT)
3521                 reg_off = E1000_FHFT(flex_filter->index);
3522         else
3523                 reg_off = E1000_FHFT_EXT(flex_filter->index - E1000_MAX_FHFT);
3524
3525         if (add) {
3526                 if (eth_igb_flex_filter_lookup(&filter_info->flex_list,
3527                                 &flex_filter->filter_info) != NULL) {
3528                         PMD_DRV_LOG(ERR, "filter exists.");
3529                         rte_free(flex_filter);
3530                         return -EEXIST;
3531                 }
3532                 flex_filter->queue = filter->queue;
3533                 /*
3534                  * look for an unused flex filter index
3535                  * and insert the filter into the list.
3536                  */
3537                 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) {
3538                         if (!(filter_info->flex_mask & (1 << i))) {
3539                                 filter_info->flex_mask |= 1 << i;
3540                                 flex_filter->index = i;
3541                                 TAILQ_INSERT_TAIL(&filter_info->flex_list,
3542                                         flex_filter,
3543                                         entries);
3544                                 break;
3545                         }
3546                 }
3547                 if (i >= E1000_MAX_FLEX_FILTERS) {
3548                         PMD_DRV_LOG(ERR, "flex filters are full.");
3549                         rte_free(flex_filter);
3550                         return -ENOSYS;
3551                 }
3552
3553                 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ |
3554                                 (E1000_WUFC_FLX0 << flex_filter->index));
3555                 queueing = filter->len |
3556                         (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) |
3557                         (filter->priority << E1000_FHFT_QUEUEING_PRIO_SHIFT);
3558                 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET,
3559                                 queueing);
3560                 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) {
3561                         E1000_WRITE_REG(hw, reg_off,
3562                                         flex_filter->filter_info.dwords[j]);
3563                         reg_off += sizeof(uint32_t);
3564                         E1000_WRITE_REG(hw, reg_off,
3565                                         flex_filter->filter_info.dwords[++j]);
3566                         reg_off += sizeof(uint32_t);
3567                         E1000_WRITE_REG(hw, reg_off,
3568                                 (uint32_t)flex_filter->filter_info.mask[i]);
3569                         reg_off += sizeof(uint32_t) * 2;
3570                         ++j;
3571                 }
3572         } else {
3573                 it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
3574                                 &flex_filter->filter_info);
3575                 if (it == NULL) {
3576                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3577                         rte_free(flex_filter);
3578                         return -ENOENT;
3579                 }
3580
3581                 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++)
3582                         E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0);
3583                 E1000_WRITE_REG(hw, E1000_WUFC, wufc &
3584                         (~(E1000_WUFC_FLX0 << it->index)));
3585
3586                 filter_info->flex_mask &= ~(1 << it->index);
3587                 TAILQ_REMOVE(&filter_info->flex_list, it, entries);
3588                 rte_free(it);
3589                 rte_free(flex_filter);
3590         }
3591
3592         return 0;
3593 }
3594
3595 static int
3596 eth_igb_get_flex_filter(struct rte_eth_dev *dev,
3597                         struct rte_eth_flex_filter *filter)
3598 {
3599         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3600         struct e1000_filter_info *filter_info =
3601                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3602         struct e1000_flex_filter flex_filter, *it;
3603         uint32_t wufc, queueing, wufc_en = 0;
3604
3605         memset(&flex_filter, 0, sizeof(struct e1000_flex_filter));
3606         flex_filter.filter_info.len = filter->len;
3607         flex_filter.filter_info.priority = filter->priority;
3608         memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len);
3609         memcpy(flex_filter.filter_info.mask, filter->mask,
3610                         RTE_ALIGN(filter->len, sizeof(char)) / sizeof(char));
3611
3612         it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
3613                                 &flex_filter.filter_info);
3614         if (it == NULL) {
3615                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3616                 return -ENOENT;
3617         }
3618
3619         wufc = E1000_READ_REG(hw, E1000_WUFC);
3620         wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index);
3621
3622         if ((wufc & wufc_en) == wufc_en) {
3623                 uint32_t reg_off = 0;
3624                 if (it->index < E1000_MAX_FHFT)
3625                         reg_off = E1000_FHFT(it->index);
3626                 else
3627                         reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT);
3628
3629                 queueing = E1000_READ_REG(hw,
3630                                 reg_off + E1000_FHFT_QUEUEING_OFFSET);
3631                 filter->len = queueing & E1000_FHFT_QUEUEING_LEN;
3632                 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >>
3633                         E1000_FHFT_QUEUEING_PRIO_SHIFT;
3634                 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >>
3635                         E1000_FHFT_QUEUEING_QUEUE_SHIFT;
3636                 return 0;
3637         }
3638         return -ENOENT;
3639 }
3640
3641 static int
3642 eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
3643                         enum rte_filter_op filter_op,
3644                         void *arg)
3645 {
3646         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3647         struct rte_eth_flex_filter *filter;
3648         int ret = 0;
3649
3650         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
3651
3652         if (filter_op == RTE_ETH_FILTER_NOP)
3653                 return ret;
3654
3655         if (arg == NULL) {
3656                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
3657                             filter_op);
3658                 return -EINVAL;
3659         }
3660
3661         filter = (struct rte_eth_flex_filter *)arg;
3662         if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN
3663             || filter->len % sizeof(uint64_t) != 0) {
3664                 PMD_DRV_LOG(ERR, "filter's length is out of range");
3665                 return -EINVAL;
3666         }
3667         if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) {
3668                 PMD_DRV_LOG(ERR, "filter's priority is out of range");
3669                 return -EINVAL;
3670         }
3671
3672         switch (filter_op) {
3673         case RTE_ETH_FILTER_ADD:
3674                 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE);
3675                 break;
3676         case RTE_ETH_FILTER_DELETE:
3677                 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE);
3678                 break;
3679         case RTE_ETH_FILTER_GET:
3680                 ret = eth_igb_get_flex_filter(dev, filter);
3681                 break;
3682         default:
3683                 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op);
3684                 ret = -EINVAL;
3685                 break;
3686         }
3687
3688         return ret;
3689 }
3690
3691 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/
3692 static inline int
3693 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter,
3694                         struct e1000_5tuple_filter_info *filter_info)
3695 {
3696         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576)
3697                 return -EINVAL;
3698         if (filter->priority > E1000_2TUPLE_MAX_PRI)
3699                 return -EINVAL;  /* filter index is out of range. */
3700         if (filter->tcp_flags > TCP_FLAG_ALL)
3701                 return -EINVAL;  /* flags is invalid. */
3702
3703         switch (filter->dst_ip_mask) {
3704         case UINT32_MAX:
3705                 filter_info->dst_ip_mask = 0;
3706                 filter_info->dst_ip = filter->dst_ip;
3707                 break;
3708         case 0:
3709                 filter_info->dst_ip_mask = 1;
3710                 break;
3711         default:
3712                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
3713                 return -EINVAL;
3714         }
3715
3716         switch (filter->src_ip_mask) {
3717         case UINT32_MAX:
3718                 filter_info->src_ip_mask = 0;
3719                 filter_info->src_ip = filter->src_ip;
3720                 break;
3721         case 0:
3722                 filter_info->src_ip_mask = 1;
3723                 break;
3724         default:
3725                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
3726                 return -EINVAL;
3727         }
3728
3729         switch (filter->dst_port_mask) {
3730         case UINT16_MAX:
3731                 filter_info->dst_port_mask = 0;
3732                 filter_info->dst_port = filter->dst_port;
3733                 break;
3734         case 0:
3735                 filter_info->dst_port_mask = 1;
3736                 break;
3737         default:
3738                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
3739                 return -EINVAL;
3740         }
3741
3742         switch (filter->src_port_mask) {
3743         case UINT16_MAX:
3744                 filter_info->src_port_mask = 0;
3745                 filter_info->src_port = filter->src_port;
3746                 break;
3747         case 0:
3748                 filter_info->src_port_mask = 1;
3749                 break;
3750         default:
3751                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
3752                 return -EINVAL;
3753         }
3754
3755         switch (filter->proto_mask) {
3756         case UINT8_MAX:
3757                 filter_info->proto_mask = 0;
3758                 filter_info->proto = filter->proto;
3759                 break;
3760         case 0:
3761                 filter_info->proto_mask = 1;
3762                 break;
3763         default:
3764                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
3765                 return -EINVAL;
3766         }
3767
3768         filter_info->priority = (uint8_t)filter->priority;
3769         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
3770                 filter_info->tcp_flags = filter->tcp_flags;
3771         else
3772                 filter_info->tcp_flags = 0;
3773
3774         return 0;
3775 }
3776
3777 static inline struct e1000_5tuple_filter *
3778 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list,
3779                         struct e1000_5tuple_filter_info *key)
3780 {
3781         struct e1000_5tuple_filter *it;
3782
3783         TAILQ_FOREACH(it, filter_list, entries) {
3784                 if (memcmp(key, &it->filter_info,
3785                         sizeof(struct e1000_5tuple_filter_info)) == 0) {
3786                         return it;
3787                 }
3788         }
3789         return NULL;
3790 }
3791
3792 /*
3793  * igb_add_5tuple_filter_82576 - add a 5tuple filter
3794  *
3795  * @param
3796  * dev: Pointer to struct rte_eth_dev.
3797  * ntuple_filter: ponter to the filter that will be added.
3798  *
3799  * @return
3800  *    - On success, zero.
3801  *    - On failure, a negative value.
3802  */
3803 static int
3804 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
3805                         struct rte_eth_ntuple_filter *ntuple_filter)
3806 {
3807         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3808         struct e1000_filter_info *filter_info =
3809                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3810         struct e1000_5tuple_filter *filter;
3811         uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK;
3812         uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP;
3813         uint8_t i;
3814         int ret;
3815
3816         filter = rte_zmalloc("e1000_5tuple_filter",
3817                         sizeof(struct e1000_5tuple_filter), 0);
3818         if (filter == NULL)
3819                 return -ENOMEM;
3820
3821         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3822                                             &filter->filter_info);
3823         if (ret < 0) {
3824                 rte_free(filter);
3825                 return ret;
3826         }
3827
3828         if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
3829                                          &filter->filter_info) != NULL) {
3830                 PMD_DRV_LOG(ERR, "filter exists.");
3831                 rte_free(filter);
3832                 return -EEXIST;
3833         }
3834         filter->queue = ntuple_filter->queue;
3835
3836         /*
3837          * look for an unused 5tuple filter index,
3838          * and insert the filter to list.
3839          */
3840         for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) {
3841                 if (!(filter_info->fivetuple_mask & (1 << i))) {
3842                         filter_info->fivetuple_mask |= 1 << i;
3843                         filter->index = i;
3844                         TAILQ_INSERT_TAIL(&filter_info->fivetuple_list,
3845                                           filter,
3846                                           entries);
3847                         break;
3848                 }
3849         }
3850         if (i >= E1000_MAX_FTQF_FILTERS) {
3851                 PMD_DRV_LOG(ERR, "5tuple filters are full.");
3852                 rte_free(filter);
3853                 return -ENOSYS;
3854         }
3855
3856         ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK;
3857         if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */
3858                 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP;
3859         if (filter->filter_info.dst_ip_mask == 0)
3860                 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP;
3861         if (filter->filter_info.src_port_mask == 0)
3862                 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
3863         if (filter->filter_info.proto_mask == 0)
3864                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP;
3865         ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) &
3866                 E1000_FTQF_QUEUE_MASK;
3867         ftqf |= E1000_FTQF_QUEUE_ENABLE;
3868         E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf);
3869         E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip);
3870         E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip);
3871
3872         spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT;
3873         E1000_WRITE_REG(hw, E1000_SPQF(i), spqf);
3874
3875         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
3876         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
3877                 imir |= E1000_IMIR_PORT_BP;
3878         else
3879                 imir &= ~E1000_IMIR_PORT_BP;
3880         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
3881
3882         /* tcp flags bits setting. */
3883         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
3884                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
3885                         imir_ext |= E1000_IMIREXT_CTRL_URG;
3886                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
3887                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
3888                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
3889                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
3890                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
3891                         imir_ext |= E1000_IMIREXT_CTRL_RST;
3892                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
3893                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
3894                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
3895                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
3896         } else
3897                 imir_ext |= E1000_IMIREXT_CTRL_BP;
3898         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
3899         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
3900         return 0;
3901 }
3902
3903 /*
3904  * igb_remove_5tuple_filter_82576 - remove a 5tuple filter
3905  *
3906  * @param
3907  * dev: Pointer to struct rte_eth_dev.
3908  * ntuple_filter: ponter to the filter that will be removed.
3909  *
3910  * @return
3911  *    - On success, zero.
3912  *    - On failure, a negative value.
3913  */
3914 static int
3915 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
3916                                 struct rte_eth_ntuple_filter *ntuple_filter)
3917 {
3918         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3919         struct e1000_filter_info *filter_info =
3920                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3921         struct e1000_5tuple_filter_info filter_5tuple;
3922         struct e1000_5tuple_filter *filter;
3923         int ret;
3924
3925         memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info));
3926         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3927                                             &filter_5tuple);
3928         if (ret < 0)
3929                 return ret;
3930
3931         filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
3932                                          &filter_5tuple);
3933         if (filter == NULL) {
3934                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3935                 return -ENOENT;
3936         }
3937
3938         filter_info->fivetuple_mask &= ~(1 << filter->index);
3939         TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries);
3940         rte_free(filter);
3941
3942         E1000_WRITE_REG(hw, E1000_FTQF(filter->index),
3943                         E1000_FTQF_VF_BP | E1000_FTQF_MASK);
3944         E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0);
3945         E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0);
3946         E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0);
3947         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
3948         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
3949         return 0;
3950 }
3951
3952 static int
3953 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
3954 {
3955         uint32_t rctl;
3956         struct e1000_hw *hw;
3957         struct rte_eth_dev_info dev_info;
3958         uint32_t frame_size = mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN +
3959                                      VLAN_TAG_SIZE);
3960
3961         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3962
3963 #ifdef RTE_LIBRTE_82571_SUPPORT
3964         /* XXX: not bigger than max_rx_pktlen */
3965         if (hw->mac.type == e1000_82571)
3966                 return -ENOTSUP;
3967 #endif
3968         eth_igb_infos_get(dev, &dev_info);
3969
3970         /* check that mtu is within the allowed range */
3971         if ((mtu < ETHER_MIN_MTU) ||
3972             (frame_size > dev_info.max_rx_pktlen))
3973                 return -EINVAL;
3974
3975         /* refuse mtu that requires the support of scattered packets when this
3976          * feature has not been enabled before. */
3977         if (!dev->data->scattered_rx &&
3978             frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)
3979                 return -EINVAL;
3980
3981         rctl = E1000_READ_REG(hw, E1000_RCTL);
3982
3983         /* switch to jumbo mode if needed */
3984         if (frame_size > ETHER_MAX_LEN) {
3985                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
3986                 rctl |= E1000_RCTL_LPE;
3987         } else {
3988                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
3989                 rctl &= ~E1000_RCTL_LPE;
3990         }
3991         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3992
3993         /* update max frame size */
3994         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
3995
3996         E1000_WRITE_REG(hw, E1000_RLPML,
3997                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
3998
3999         return 0;
4000 }
4001
4002 /*
4003  * igb_add_del_ntuple_filter - add or delete a ntuple filter
4004  *
4005  * @param
4006  * dev: Pointer to struct rte_eth_dev.
4007  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
4008  * add: if true, add filter, if false, remove filter
4009  *
4010  * @return
4011  *    - On success, zero.
4012  *    - On failure, a negative value.
4013  */
4014 static int
4015 igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
4016                         struct rte_eth_ntuple_filter *ntuple_filter,
4017                         bool add)
4018 {
4019         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4020         int ret;
4021
4022         switch (ntuple_filter->flags) {
4023         case RTE_5TUPLE_FLAGS:
4024         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
4025                 if (hw->mac.type != e1000_82576)
4026                         return -ENOTSUP;
4027                 if (add)
4028                         ret = igb_add_5tuple_filter_82576(dev,
4029                                                           ntuple_filter);
4030                 else
4031                         ret = igb_remove_5tuple_filter_82576(dev,
4032                                                              ntuple_filter);
4033                 break;
4034         case RTE_2TUPLE_FLAGS:
4035         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
4036                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
4037                         return -ENOTSUP;
4038                 if (add)
4039                         ret = igb_add_2tuple_filter(dev, ntuple_filter);
4040                 else
4041                         ret = igb_remove_2tuple_filter(dev, ntuple_filter);
4042                 break;
4043         default:
4044                 ret = -EINVAL;
4045                 break;
4046         }
4047
4048         return ret;
4049 }
4050
4051 /*
4052  * igb_get_ntuple_filter - get a ntuple filter
4053  *
4054  * @param
4055  * dev: Pointer to struct rte_eth_dev.
4056  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
4057  *
4058  * @return
4059  *    - On success, zero.
4060  *    - On failure, a negative value.
4061  */
4062 static int
4063 igb_get_ntuple_filter(struct rte_eth_dev *dev,
4064                         struct rte_eth_ntuple_filter *ntuple_filter)
4065 {
4066         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4067         struct e1000_filter_info *filter_info =
4068                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4069         struct e1000_5tuple_filter_info filter_5tuple;
4070         struct e1000_2tuple_filter_info filter_2tuple;
4071         struct e1000_5tuple_filter *p_5tuple_filter;
4072         struct e1000_2tuple_filter *p_2tuple_filter;
4073         int ret;
4074
4075         switch (ntuple_filter->flags) {
4076         case RTE_5TUPLE_FLAGS:
4077         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
4078                 if (hw->mac.type != e1000_82576)
4079                         return -ENOTSUP;
4080                 memset(&filter_5tuple,
4081                         0,
4082                         sizeof(struct e1000_5tuple_filter_info));
4083                 ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
4084                                                     &filter_5tuple);
4085                 if (ret < 0)
4086                         return ret;
4087                 p_5tuple_filter = igb_5tuple_filter_lookup_82576(
4088                                         &filter_info->fivetuple_list,
4089                                         &filter_5tuple);
4090                 if (p_5tuple_filter == NULL) {
4091                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
4092                         return -ENOENT;
4093                 }
4094                 ntuple_filter->queue = p_5tuple_filter->queue;
4095                 break;
4096         case RTE_2TUPLE_FLAGS:
4097         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
4098                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
4099                         return -ENOTSUP;
4100                 memset(&filter_2tuple,
4101                         0,
4102                         sizeof(struct e1000_2tuple_filter_info));
4103                 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple);
4104                 if (ret < 0)
4105                         return ret;
4106                 p_2tuple_filter = igb_2tuple_filter_lookup(
4107                                         &filter_info->twotuple_list,
4108                                         &filter_2tuple);
4109                 if (p_2tuple_filter == NULL) {
4110                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
4111                         return -ENOENT;
4112                 }
4113                 ntuple_filter->queue = p_2tuple_filter->queue;
4114                 break;
4115         default:
4116                 ret = -EINVAL;
4117                 break;
4118         }
4119
4120         return 0;
4121 }
4122
4123 /*
4124  * igb_ntuple_filter_handle - Handle operations for ntuple filter.
4125  * @dev: pointer to rte_eth_dev structure
4126  * @filter_op:operation will be taken.
4127  * @arg: a pointer to specific structure corresponding to the filter_op
4128  */
4129 static int
4130 igb_ntuple_filter_handle(struct rte_eth_dev *dev,
4131                                 enum rte_filter_op filter_op,
4132                                 void *arg)
4133 {
4134         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4135         int ret;
4136
4137         MAC_TYPE_FILTER_SUP(hw->mac.type);
4138
4139         if (filter_op == RTE_ETH_FILTER_NOP)
4140                 return 0;
4141
4142         if (arg == NULL) {
4143                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
4144                             filter_op);
4145                 return -EINVAL;
4146         }
4147
4148         switch (filter_op) {
4149         case RTE_ETH_FILTER_ADD:
4150                 ret = igb_add_del_ntuple_filter(dev,
4151                         (struct rte_eth_ntuple_filter *)arg,
4152                         TRUE);
4153                 break;
4154         case RTE_ETH_FILTER_DELETE:
4155                 ret = igb_add_del_ntuple_filter(dev,
4156                         (struct rte_eth_ntuple_filter *)arg,
4157                         FALSE);
4158                 break;
4159         case RTE_ETH_FILTER_GET:
4160                 ret = igb_get_ntuple_filter(dev,
4161                         (struct rte_eth_ntuple_filter *)arg);
4162                 break;
4163         default:
4164                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
4165                 ret = -EINVAL;
4166                 break;
4167         }
4168         return ret;
4169 }
4170
4171 static inline int
4172 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info,
4173                         uint16_t ethertype)
4174 {
4175         int i;
4176
4177         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
4178                 if (filter_info->ethertype_filters[i] == ethertype &&
4179                     (filter_info->ethertype_mask & (1 << i)))
4180                         return i;
4181         }
4182         return -1;
4183 }
4184
4185 static inline int
4186 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info,
4187                         uint16_t ethertype)
4188 {
4189         int i;
4190
4191         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
4192                 if (!(filter_info->ethertype_mask & (1 << i))) {
4193                         filter_info->ethertype_mask |= 1 << i;
4194                         filter_info->ethertype_filters[i] = ethertype;
4195                         return i;
4196                 }
4197         }
4198         return -1;
4199 }
4200
4201 static inline int
4202 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info,
4203                         uint8_t idx)
4204 {
4205         if (idx >= E1000_MAX_ETQF_FILTERS)
4206                 return -1;
4207         filter_info->ethertype_mask &= ~(1 << idx);
4208         filter_info->ethertype_filters[idx] = 0;
4209         return idx;
4210 }
4211
4212
4213 static int
4214 igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
4215                         struct rte_eth_ethertype_filter *filter,
4216                         bool add)
4217 {
4218         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4219         struct e1000_filter_info *filter_info =
4220                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4221         uint32_t etqf = 0;
4222         int ret;
4223
4224         if (filter->ether_type == ETHER_TYPE_IPv4 ||
4225                 filter->ether_type == ETHER_TYPE_IPv6) {
4226                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
4227                         " ethertype filter.", filter->ether_type);
4228                 return -EINVAL;
4229         }
4230
4231         if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) {
4232                 PMD_DRV_LOG(ERR, "mac compare is unsupported.");
4233                 return -EINVAL;
4234         }
4235         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) {
4236                 PMD_DRV_LOG(ERR, "drop option is unsupported.");
4237                 return -EINVAL;
4238         }
4239
4240         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
4241         if (ret >= 0 && add) {
4242                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.",
4243                             filter->ether_type);
4244                 return -EEXIST;
4245         }
4246         if (ret < 0 && !add) {
4247                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
4248                             filter->ether_type);
4249                 return -ENOENT;
4250         }
4251
4252         if (add) {
4253                 ret = igb_ethertype_filter_insert(filter_info,
4254                         filter->ether_type);
4255                 if (ret < 0) {
4256                         PMD_DRV_LOG(ERR, "ethertype filters are full.");
4257                         return -ENOSYS;
4258                 }
4259
4260                 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE;
4261                 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE);
4262                 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT;
4263         } else {
4264                 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret);
4265                 if (ret < 0)
4266                         return -ENOSYS;
4267         }
4268         E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf);
4269         E1000_WRITE_FLUSH(hw);
4270
4271         return 0;
4272 }
4273
4274 static int
4275 igb_get_ethertype_filter(struct rte_eth_dev *dev,
4276                         struct rte_eth_ethertype_filter *filter)
4277 {
4278         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4279         struct e1000_filter_info *filter_info =
4280                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4281         uint32_t etqf;
4282         int ret;
4283
4284         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
4285         if (ret < 0) {
4286                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
4287                             filter->ether_type);
4288                 return -ENOENT;
4289         }
4290
4291         etqf = E1000_READ_REG(hw, E1000_ETQF(ret));
4292         if (etqf & E1000_ETQF_FILTER_ENABLE) {
4293                 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE;
4294                 filter->flags = 0;
4295                 filter->queue = (etqf & E1000_ETQF_QUEUE) >>
4296                                 E1000_ETQF_QUEUE_SHIFT;
4297                 return 0;
4298         }
4299
4300         return -ENOENT;
4301 }
4302
4303 /*
4304  * igb_ethertype_filter_handle - Handle operations for ethertype filter.
4305  * @dev: pointer to rte_eth_dev structure
4306  * @filter_op:operation will be taken.
4307  * @arg: a pointer to specific structure corresponding to the filter_op
4308  */
4309 static int
4310 igb_ethertype_filter_handle(struct rte_eth_dev *dev,
4311                                 enum rte_filter_op filter_op,
4312                                 void *arg)
4313 {
4314         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4315         int ret;
4316
4317         MAC_TYPE_FILTER_SUP(hw->mac.type);
4318
4319         if (filter_op == RTE_ETH_FILTER_NOP)
4320                 return 0;
4321
4322         if (arg == NULL) {
4323                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
4324                             filter_op);
4325                 return -EINVAL;
4326         }
4327
4328         switch (filter_op) {
4329         case RTE_ETH_FILTER_ADD:
4330                 ret = igb_add_del_ethertype_filter(dev,
4331                         (struct rte_eth_ethertype_filter *)arg,
4332                         TRUE);
4333                 break;
4334         case RTE_ETH_FILTER_DELETE:
4335                 ret = igb_add_del_ethertype_filter(dev,
4336                         (struct rte_eth_ethertype_filter *)arg,
4337                         FALSE);
4338                 break;
4339         case RTE_ETH_FILTER_GET:
4340                 ret = igb_get_ethertype_filter(dev,
4341                         (struct rte_eth_ethertype_filter *)arg);
4342                 break;
4343         default:
4344                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
4345                 ret = -EINVAL;
4346                 break;
4347         }
4348         return ret;
4349 }
4350
4351 static int
4352 eth_igb_filter_ctrl(struct rte_eth_dev *dev,
4353                      enum rte_filter_type filter_type,
4354                      enum rte_filter_op filter_op,
4355                      void *arg)
4356 {
4357         int ret = -EINVAL;
4358
4359         switch (filter_type) {
4360         case RTE_ETH_FILTER_NTUPLE:
4361                 ret = igb_ntuple_filter_handle(dev, filter_op, arg);
4362                 break;
4363         case RTE_ETH_FILTER_ETHERTYPE:
4364                 ret = igb_ethertype_filter_handle(dev, filter_op, arg);
4365                 break;
4366         case RTE_ETH_FILTER_SYN:
4367                 ret = eth_igb_syn_filter_handle(dev, filter_op, arg);
4368                 break;
4369         case RTE_ETH_FILTER_FLEXIBLE:
4370                 ret = eth_igb_flex_filter_handle(dev, filter_op, arg);
4371                 break;
4372         default:
4373                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
4374                                                         filter_type);
4375                 break;
4376         }
4377
4378         return ret;
4379 }
4380
4381 static int
4382 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
4383                          struct ether_addr *mc_addr_set,
4384                          uint32_t nb_mc_addr)
4385 {
4386         struct e1000_hw *hw;
4387
4388         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4389         e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr);
4390         return 0;
4391 }
4392
4393 static uint64_t
4394 igb_read_systime_cyclecounter(struct rte_eth_dev *dev)
4395 {
4396         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4397         uint64_t systime_cycles;
4398
4399         switch (hw->mac.type) {
4400         case e1000_i210:
4401         case e1000_i211:
4402                 /*
4403                  * Need to read System Time Residue Register to be able
4404                  * to read the other two registers.
4405                  */
4406                 E1000_READ_REG(hw, E1000_SYSTIMR);
4407                 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */
4408                 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML);
4409                 systime_cycles += (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH)
4410                                 * NSEC_PER_SEC;
4411                 break;
4412         case e1000_82580:
4413         case e1000_i350:
4414         case e1000_i354:
4415                 /*
4416                  * Need to read System Time Residue Register to be able
4417                  * to read the other two registers.
4418                  */
4419                 E1000_READ_REG(hw, E1000_SYSTIMR);
4420                 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML);
4421                 /* Only the 8 LSB are valid. */
4422                 systime_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_SYSTIMH)
4423                                 & 0xff) << 32;
4424                 break;
4425         default:
4426                 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML);
4427                 systime_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH)
4428                                 << 32;
4429                 break;
4430         }
4431
4432         return systime_cycles;
4433 }
4434
4435 static uint64_t
4436 igb_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev)
4437 {
4438         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4439         uint64_t rx_tstamp_cycles;
4440
4441         switch (hw->mac.type) {
4442         case e1000_i210:
4443         case e1000_i211:
4444                 /* RXSTMPL stores ns and RXSTMPH stores seconds. */
4445                 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL);
4446                 rx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH)
4447                                 * NSEC_PER_SEC;
4448                 break;
4449         case e1000_82580:
4450         case e1000_i350:
4451         case e1000_i354:
4452                 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL);
4453                 /* Only the 8 LSB are valid. */
4454                 rx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_RXSTMPH)
4455                                 & 0xff) << 32;
4456                 break;
4457         default:
4458                 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL);
4459                 rx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH)
4460                                 << 32;
4461                 break;
4462         }
4463
4464         return rx_tstamp_cycles;
4465 }
4466
4467 static uint64_t
4468 igb_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev)
4469 {
4470         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4471         uint64_t tx_tstamp_cycles;
4472
4473         switch (hw->mac.type) {
4474         case e1000_i210:
4475         case e1000_i211:
4476                 /* RXSTMPL stores ns and RXSTMPH stores seconds. */
4477                 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL);
4478                 tx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH)
4479                                 * NSEC_PER_SEC;
4480                 break;
4481         case e1000_82580:
4482         case e1000_i350:
4483         case e1000_i354:
4484                 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL);
4485                 /* Only the 8 LSB are valid. */
4486                 tx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_TXSTMPH)
4487                                 & 0xff) << 32;
4488                 break;
4489         default:
4490                 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL);
4491                 tx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH)
4492                                 << 32;
4493                 break;
4494         }
4495
4496         return tx_tstamp_cycles;
4497 }
4498
4499 static void
4500 igb_start_timecounters(struct rte_eth_dev *dev)
4501 {
4502         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4503         struct e1000_adapter *adapter =
4504                 (struct e1000_adapter *)dev->data->dev_private;
4505         uint32_t incval = 1;
4506         uint32_t shift = 0;
4507         uint64_t mask = E1000_CYCLECOUNTER_MASK;
4508
4509         switch (hw->mac.type) {
4510         case e1000_82580:
4511         case e1000_i350:
4512         case e1000_i354:
4513                 /* 32 LSB bits + 8 MSB bits = 40 bits */
4514                 mask = (1ULL << 40) - 1;
4515                 /* fall-through */
4516         case e1000_i210:
4517         case e1000_i211:
4518                 /*
4519                  * Start incrementing the register
4520                  * used to timestamp PTP packets.
4521                  */
4522                 E1000_WRITE_REG(hw, E1000_TIMINCA, incval);
4523                 break;
4524         case e1000_82576:
4525                 incval = E1000_INCVALUE_82576;
4526                 shift = IGB_82576_TSYNC_SHIFT;
4527                 E1000_WRITE_REG(hw, E1000_TIMINCA,
4528                                 E1000_INCPERIOD_82576 | incval);
4529                 break;
4530         default:
4531                 /* Not supported */
4532                 return;
4533         }
4534
4535         memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter));
4536         memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
4537         memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
4538
4539         adapter->systime_tc.cc_mask = mask;
4540         adapter->systime_tc.cc_shift = shift;
4541         adapter->systime_tc.nsec_mask = (1ULL << shift) - 1;
4542
4543         adapter->rx_tstamp_tc.cc_mask = mask;
4544         adapter->rx_tstamp_tc.cc_shift = shift;
4545         adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
4546
4547         adapter->tx_tstamp_tc.cc_mask = mask;
4548         adapter->tx_tstamp_tc.cc_shift = shift;
4549         adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
4550 }
4551
4552 static int
4553 igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
4554 {
4555         struct e1000_adapter *adapter =
4556                         (struct e1000_adapter *)dev->data->dev_private;
4557
4558         adapter->systime_tc.nsec += delta;
4559         adapter->rx_tstamp_tc.nsec += delta;
4560         adapter->tx_tstamp_tc.nsec += delta;
4561
4562         return 0;
4563 }
4564
4565 static int
4566 igb_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
4567 {
4568         uint64_t ns;
4569         struct e1000_adapter *adapter =
4570                         (struct e1000_adapter *)dev->data->dev_private;
4571
4572         ns = rte_timespec_to_ns(ts);
4573
4574         /* Set the timecounters to a new value. */
4575         adapter->systime_tc.nsec = ns;
4576         adapter->rx_tstamp_tc.nsec = ns;
4577         adapter->tx_tstamp_tc.nsec = ns;
4578
4579         return 0;
4580 }
4581
4582 static int
4583 igb_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
4584 {
4585         uint64_t ns, systime_cycles;
4586         struct e1000_adapter *adapter =
4587                         (struct e1000_adapter *)dev->data->dev_private;
4588
4589         systime_cycles = igb_read_systime_cyclecounter(dev);
4590         ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles);
4591         *ts = rte_ns_to_timespec(ns);
4592
4593         return 0;
4594 }
4595
4596 static int
4597 igb_timesync_enable(struct rte_eth_dev *dev)
4598 {
4599         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4600         uint32_t tsync_ctl;
4601         uint32_t tsauxc;
4602
4603         /* Stop the timesync system time. */
4604         E1000_WRITE_REG(hw, E1000_TIMINCA, 0x0);
4605         /* Reset the timesync system time value. */
4606         switch (hw->mac.type) {
4607         case e1000_82580:
4608         case e1000_i350:
4609         case e1000_i354:
4610         case e1000_i210:
4611         case e1000_i211:
4612                 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x0);
4613                 /* fall-through */
4614         case e1000_82576:
4615                 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x0);
4616                 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x0);
4617                 break;
4618         default:
4619                 /* Not supported. */
4620                 return -ENOTSUP;
4621         }
4622
4623         /* Enable system time for it isn't on by default. */
4624         tsauxc = E1000_READ_REG(hw, E1000_TSAUXC);
4625         tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME;
4626         E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc);
4627
4628         igb_start_timecounters(dev);
4629
4630         /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
4631         E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588),
4632                         (ETHER_TYPE_1588 |
4633                          E1000_ETQF_FILTER_ENABLE |
4634                          E1000_ETQF_1588));
4635
4636         /* Enable timestamping of received PTP packets. */
4637         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
4638         tsync_ctl |= E1000_TSYNCRXCTL_ENABLED;
4639         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl);
4640
4641         /* Enable Timestamping of transmitted PTP packets. */
4642         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
4643         tsync_ctl |= E1000_TSYNCTXCTL_ENABLED;
4644         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl);
4645
4646         return 0;
4647 }
4648
4649 static int
4650 igb_timesync_disable(struct rte_eth_dev *dev)
4651 {
4652         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4653         uint32_t tsync_ctl;
4654
4655         /* Disable timestamping of transmitted PTP packets. */
4656         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
4657         tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED;
4658         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl);
4659
4660         /* Disable timestamping of received PTP packets. */
4661         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
4662         tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED;
4663         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl);
4664
4665         /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
4666         E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0);
4667
4668         /* Stop incrementating the System Time registers. */
4669         E1000_WRITE_REG(hw, E1000_TIMINCA, 0);
4670
4671         return 0;
4672 }
4673
4674 static int
4675 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
4676                                struct timespec *timestamp,
4677                                uint32_t flags __rte_unused)
4678 {
4679         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4680         struct e1000_adapter *adapter =
4681                         (struct e1000_adapter *)dev->data->dev_private;
4682         uint32_t tsync_rxctl;
4683         uint64_t rx_tstamp_cycles;
4684         uint64_t ns;
4685
4686         tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
4687         if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0)
4688                 return -EINVAL;
4689
4690         rx_tstamp_cycles = igb_read_rx_tstamp_cyclecounter(dev);
4691         ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles);
4692         *timestamp = rte_ns_to_timespec(ns);
4693
4694         return  0;
4695 }
4696
4697 static int
4698 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
4699                                struct timespec *timestamp)
4700 {
4701         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4702         struct e1000_adapter *adapter =
4703                         (struct e1000_adapter *)dev->data->dev_private;
4704         uint32_t tsync_txctl;
4705         uint64_t tx_tstamp_cycles;
4706         uint64_t ns;
4707
4708         tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
4709         if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0)
4710                 return -EINVAL;
4711
4712         tx_tstamp_cycles = igb_read_tx_tstamp_cyclecounter(dev);
4713         ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles);
4714         *timestamp = rte_ns_to_timespec(ns);
4715
4716         return  0;
4717 }
4718
4719 static int
4720 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused)
4721 {
4722         int count = 0;
4723         int g_ind = 0;
4724         const struct reg_info *reg_group;
4725
4726         while ((reg_group = igb_regs[g_ind++]))
4727                 count += igb_reg_group_count(reg_group);
4728
4729         return count;
4730 }
4731
4732 static int
4733 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused)
4734 {
4735         int count = 0;
4736         int g_ind = 0;
4737         const struct reg_info *reg_group;
4738
4739         while ((reg_group = igbvf_regs[g_ind++]))
4740                 count += igb_reg_group_count(reg_group);
4741
4742         return count;
4743 }
4744
4745 static int
4746 eth_igb_get_regs(struct rte_eth_dev *dev,
4747         struct rte_dev_reg_info *regs)
4748 {
4749         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4750         uint32_t *data = regs->data;
4751         int g_ind = 0;
4752         int count = 0;
4753         const struct reg_info *reg_group;
4754
4755         /* Support only full register dump */
4756         if ((regs->length == 0) ||
4757             (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) {
4758                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4759                         hw->device_id;
4760                 while ((reg_group = igb_regs[g_ind++]))
4761                         count += igb_read_regs_group(dev, &data[count],
4762                                                         reg_group);
4763                 return 0;
4764         }
4765
4766         return -ENOTSUP;
4767 }
4768
4769 static int
4770 igbvf_get_regs(struct rte_eth_dev *dev,
4771         struct rte_dev_reg_info *regs)
4772 {
4773         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4774         uint32_t *data = regs->data;
4775         int g_ind = 0;
4776         int count = 0;
4777         const struct reg_info *reg_group;
4778
4779         /* Support only full register dump */
4780         if ((regs->length == 0) ||
4781             (regs->length == (uint32_t)igbvf_get_reg_length(dev))) {
4782                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4783                         hw->device_id;
4784                 while ((reg_group = igbvf_regs[g_ind++]))
4785                         count += igb_read_regs_group(dev, &data[count],
4786                                                         reg_group);
4787                 return 0;
4788         }
4789
4790         return -ENOTSUP;
4791 }
4792
4793 static int
4794 eth_igb_get_eeprom_length(struct rte_eth_dev *dev)
4795 {
4796         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4797
4798         /* Return unit is byte count */
4799         return hw->nvm.word_size * 2;
4800 }
4801
4802 static int
4803 eth_igb_get_eeprom(struct rte_eth_dev *dev,
4804         struct rte_dev_eeprom_info *in_eeprom)
4805 {
4806         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4807         struct e1000_nvm_info *nvm = &hw->nvm;
4808         uint16_t *data = in_eeprom->data;
4809         int first, length;
4810
4811         first = in_eeprom->offset >> 1;
4812         length = in_eeprom->length >> 1;
4813         if ((first >= hw->nvm.word_size) ||
4814             ((first + length) >= hw->nvm.word_size))
4815                 return -EINVAL;
4816
4817         in_eeprom->magic = hw->vendor_id |
4818                 ((uint32_t)hw->device_id << 16);
4819
4820         if ((nvm->ops.read) == NULL)
4821                 return -ENOTSUP;
4822
4823         return nvm->ops.read(hw, first, length, data);
4824 }
4825
4826 static int
4827 eth_igb_set_eeprom(struct rte_eth_dev *dev,
4828         struct rte_dev_eeprom_info *in_eeprom)
4829 {
4830         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4831         struct e1000_nvm_info *nvm = &hw->nvm;
4832         uint16_t *data = in_eeprom->data;
4833         int first, length;
4834
4835         first = in_eeprom->offset >> 1;
4836         length = in_eeprom->length >> 1;
4837         if ((first >= hw->nvm.word_size) ||
4838             ((first + length) >= hw->nvm.word_size))
4839                 return -EINVAL;
4840
4841         in_eeprom->magic = (uint32_t)hw->vendor_id |
4842                 ((uint32_t)hw->device_id << 16);
4843
4844         if ((nvm->ops.write) == NULL)
4845                 return -ENOTSUP;
4846         return nvm->ops.write(hw,  first, length, data);
4847 }
4848
4849 static struct rte_driver pmd_igb_drv = {
4850         .type = PMD_PDEV,
4851         .init = rte_igb_pmd_init,
4852 };
4853
4854 static struct rte_driver pmd_igbvf_drv = {
4855         .type = PMD_PDEV,
4856         .init = rte_igbvf_pmd_init,
4857 };
4858
4859 static int
4860 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
4861 {
4862         struct e1000_hw *hw =
4863                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4864         uint32_t mask = 1 << queue_id;
4865
4866         E1000_WRITE_REG(hw, E1000_EIMC, mask);
4867         E1000_WRITE_FLUSH(hw);
4868
4869         return 0;
4870 }
4871
4872 static int
4873 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
4874 {
4875         struct e1000_hw *hw =
4876                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4877         uint32_t mask = 1 << queue_id;
4878         uint32_t regval;
4879
4880         regval = E1000_READ_REG(hw, E1000_EIMS);
4881         E1000_WRITE_REG(hw, E1000_EIMS, regval | mask);
4882         E1000_WRITE_FLUSH(hw);
4883
4884         rte_intr_enable(&dev->pci_dev->intr_handle);
4885
4886         return 0;
4887 }
4888
4889 static void
4890 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t  msix_vector,
4891                    uint8_t index, uint8_t offset)
4892 {
4893         uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4894
4895         /* clear bits */
4896         val &= ~((uint32_t)0xFF << offset);
4897
4898         /* write vector and valid bit */
4899         val |= (msix_vector | E1000_IVAR_VALID) << offset;
4900
4901         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val);
4902 }
4903
4904 static void
4905 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction,
4906                            uint8_t queue, uint8_t msix_vector)
4907 {
4908         uint32_t tmp = 0;
4909
4910         if (hw->mac.type == e1000_82575) {
4911                 if (direction == 0)
4912                         tmp = E1000_EICR_RX_QUEUE0 << queue;
4913                 else if (direction == 1)
4914                         tmp = E1000_EICR_TX_QUEUE0 << queue;
4915                 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp);
4916         } else if (hw->mac.type == e1000_82576) {
4917                 if ((direction == 0) || (direction == 1))
4918                         eth_igb_write_ivar(hw, msix_vector, queue & 0x7,
4919                                            ((queue & 0x8) << 1) +
4920                                            8 * direction);
4921         } else if ((hw->mac.type == e1000_82580) ||
4922                         (hw->mac.type == e1000_i350) ||
4923                         (hw->mac.type == e1000_i354) ||
4924                         (hw->mac.type == e1000_i210) ||
4925                         (hw->mac.type == e1000_i211)) {
4926                 if ((direction == 0) || (direction == 1))
4927                         eth_igb_write_ivar(hw, msix_vector,
4928                                            queue >> 1,
4929                                            ((queue & 0x1) << 4) +
4930                                            8 * direction);
4931         }
4932 }
4933
4934 /* Sets up the hardware to generate MSI-X interrupts properly
4935  * @hw
4936  *  board private structure
4937  */
4938 static void
4939 eth_igb_configure_msix_intr(struct rte_eth_dev *dev)
4940 {
4941         int queue_id;
4942         uint32_t tmpval, regval, intr_mask;
4943         struct e1000_hw *hw =
4944                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4945         uint32_t vec = E1000_MISC_VEC_ID;
4946         uint32_t base = E1000_MISC_VEC_ID;
4947         uint32_t misc_shift = 0;
4948
4949         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
4950
4951         /* won't configure msix register if no mapping is done
4952          * between intr vector and event fd
4953          */
4954         if (!rte_intr_dp_is_en(intr_handle))
4955                 return;
4956
4957         if (rte_intr_allow_others(intr_handle)) {
4958                 vec = base = E1000_RX_VEC_START;
4959                 misc_shift = 1;
4960         }
4961
4962         /* set interrupt vector for other causes */
4963         if (hw->mac.type == e1000_82575) {
4964                 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT);
4965                 /* enable MSI-X PBA support */
4966                 tmpval |= E1000_CTRL_EXT_PBA_CLR;
4967
4968                 /* Auto-Mask interrupts upon ICR read */
4969                 tmpval |= E1000_CTRL_EXT_EIAME;
4970                 tmpval |= E1000_CTRL_EXT_IRCA;
4971
4972                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval);
4973
4974                 /* enable msix_other interrupt */
4975                 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER);
4976                 regval = E1000_READ_REG(hw, E1000_EIAC);
4977                 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER);
4978                 regval = E1000_READ_REG(hw, E1000_EIAM);
4979                 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER);
4980         } else if ((hw->mac.type == e1000_82576) ||
4981                         (hw->mac.type == e1000_82580) ||
4982                         (hw->mac.type == e1000_i350) ||
4983                         (hw->mac.type == e1000_i354) ||
4984                         (hw->mac.type == e1000_i210) ||
4985                         (hw->mac.type == e1000_i211)) {
4986                 /* turn on MSI-X capability first */
4987                 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE |
4988                                         E1000_GPIE_PBA | E1000_GPIE_EIAME |
4989                                         E1000_GPIE_NSICR);
4990                 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) <<
4991                         misc_shift;
4992                 regval = E1000_READ_REG(hw, E1000_EIAC);
4993                 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask);
4994
4995                 /* enable msix_other interrupt */
4996                 regval = E1000_READ_REG(hw, E1000_EIMS);
4997                 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask);
4998                 tmpval = (dev->data->nb_rx_queues | E1000_IVAR_VALID) << 8;
4999                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval);
5000         }
5001
5002         /* use EIAM to auto-mask when MSI-X interrupt
5003          * is asserted, this saves a register write for every interrupt
5004          */
5005         intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) <<
5006                 misc_shift;
5007         regval = E1000_READ_REG(hw, E1000_EIAM);
5008         E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask);
5009
5010         for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) {
5011                 eth_igb_assign_msix_vector(hw, 0, queue_id, vec);
5012                 intr_handle->intr_vec[queue_id] = vec;
5013                 if (vec < base + intr_handle->nb_efd - 1)
5014                         vec++;
5015         }
5016
5017         E1000_WRITE_FLUSH(hw);
5018 }
5019
5020 PMD_REGISTER_DRIVER(pmd_igb_drv);
5021 PMD_REGISTER_DRIVER(pmd_igbvf_drv);