drivers/net/e1000/igb_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <stdarg.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_interrupts.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_log.h>
  44 #include <rte_debug.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_memory.h>
  49 #include <rte_memzone.h>
  50 #include <rte_eal.h>
  51 #include <rte_atomic.h>
  52 #include <rte_malloc.h>
  53 #include <rte_dev.h>
  54
  55 #include "e1000_logs.h"
  56 #include "base/e1000_api.h"
  57 #include "e1000_ethdev.h"
  58 #include "igb_regs.h"
  59
  60 /*
  61  * Default values for port configuration
  62  */
  63 #define IGB_DEFAULT_RX_FREE_THRESH  32
  64
  65 #define IGB_DEFAULT_RX_PTHRESH      ((hw->mac.type == e1000_i354) ? 12 : 8)
  66 #define IGB_DEFAULT_RX_HTHRESH      8
  67 #define IGB_DEFAULT_RX_WTHRESH      ((hw->mac.type == e1000_82576) ? 1 : 4)
  68
  69 #define IGB_DEFAULT_TX_PTHRESH      ((hw->mac.type == e1000_i354) ? 20 : 8)
  70 #define IGB_DEFAULT_TX_HTHRESH      1
  71 #define IGB_DEFAULT_TX_WTHRESH      ((hw->mac.type == e1000_82576) ? 1 : 16)
  72
  73 #define IGB_HKEY_MAX_INDEX 10
  74
  75 /* Bit shift and mask */
  76 #define IGB_4_BIT_WIDTH  (CHAR_BIT / 2)
  77 #define IGB_4_BIT_MASK   RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t)
  78 #define IGB_8_BIT_WIDTH  CHAR_BIT
  79 #define IGB_8_BIT_MASK   UINT8_MAX
  80
  81 /* Additional timesync values. */
  82 #define E1000_CYCLECOUNTER_MASK      0xffffffffffffffffULL
  83 #define E1000_ETQF_FILTER_1588       3
  84 #define IGB_82576_TSYNC_SHIFT        16
  85 #define E1000_INCPERIOD_82576        (1 << E1000_TIMINCA_16NS_SHIFT)
  86 #define E1000_INCVALUE_82576         (16 << IGB_82576_TSYNC_SHIFT)
  87 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000
  88
  89 #define E1000_VTIVAR_MISC                0x01740
  90 #define E1000_VTIVAR_MISC_MASK           0xFF
  91 #define E1000_VTIVAR_VALID               0x80
  92 #define E1000_VTIVAR_MISC_MAILBOX        0
  93 #define E1000_VTIVAR_MISC_INTR_MASK      0x3
  94
  95 static int  eth_igb_configure(struct rte_eth_dev *dev);
  96 static int  eth_igb_start(struct rte_eth_dev *dev);
  97 static void eth_igb_stop(struct rte_eth_dev *dev);
  98 static void eth_igb_close(struct rte_eth_dev *dev);
  99 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev);
 100 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev);
 101 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev);
 102 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev);
 103 static int  eth_igb_link_update(struct rte_eth_dev *dev,
 104                                 int wait_to_complete);
 105 static void eth_igb_stats_get(struct rte_eth_dev *dev,
 106                                 struct rte_eth_stats *rte_stats);
 107 static int eth_igb_xstats_get(struct rte_eth_dev *dev,
 108                               struct rte_eth_xstat *xstats, unsigned n);
 109 static int eth_igb_xstats_get_names(struct rte_eth_dev *dev,
 110                                     struct rte_eth_xstat_name *xstats_names,
 111                                     unsigned limit);
 112 static void eth_igb_stats_reset(struct rte_eth_dev *dev);
 113 static void eth_igb_xstats_reset(struct rte_eth_dev *dev);
 114 static void eth_igb_infos_get(struct rte_eth_dev *dev,
 115                               struct rte_eth_dev_info *dev_info);
 116 static const uint32_t *eth_igb_supported_ptypes_get(struct rte_eth_dev *dev);
 117 static void eth_igbvf_infos_get(struct rte_eth_dev *dev,
 118                                 struct rte_eth_dev_info *dev_info);
 119 static int  eth_igb_flow_ctrl_get(struct rte_eth_dev *dev,
 120                                 struct rte_eth_fc_conf *fc_conf);
 121 static int  eth_igb_flow_ctrl_set(struct rte_eth_dev *dev,
 122                                 struct rte_eth_fc_conf *fc_conf);
 123 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev);
 124 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev);
 125 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev);
 126 static int eth_igb_interrupt_action(struct rte_eth_dev *dev);
 127 static void eth_igb_interrupt_handler(struct rte_intr_handle *handle,
 128                                                         void *param);
 129 static int  igb_hardware_init(struct e1000_hw *hw);
 130 static void igb_hw_control_acquire(struct e1000_hw *hw);
 131 static void igb_hw_control_release(struct e1000_hw *hw);
 132 static void igb_init_manageability(struct e1000_hw *hw);
 133 static void igb_release_manageability(struct e1000_hw *hw);
 134
 135 static int  eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 136
 137 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev,
 138                 uint16_t vlan_id, int on);
 139 static int eth_igb_vlan_tpid_set(struct rte_eth_dev *dev,
 140                                  enum rte_vlan_type vlan_type,
 141                                  uint16_t tpid_id);
 142 static void eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 143
 144 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 145 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 146 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev);
 147 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev);
 148 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 149 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 150
 151 static int eth_igb_led_on(struct rte_eth_dev *dev);
 152 static int eth_igb_led_off(struct rte_eth_dev *dev);
 153
 154 static void igb_intr_disable(struct e1000_hw *hw);
 155 static int  igb_get_rx_buffer_size(struct e1000_hw *hw);
 156 static void eth_igb_rar_set(struct rte_eth_dev *dev,
 157                 struct ether_addr *mac_addr,
 158                 uint32_t index, uint32_t pool);
 159 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 160 static void eth_igb_default_mac_addr_set(struct rte_eth_dev *dev,
 161                 struct ether_addr *addr);
 162
 163 static void igbvf_intr_disable(struct e1000_hw *hw);
 164 static int igbvf_dev_configure(struct rte_eth_dev *dev);
 165 static int igbvf_dev_start(struct rte_eth_dev *dev);
 166 static void igbvf_dev_stop(struct rte_eth_dev *dev);
 167 static void igbvf_dev_close(struct rte_eth_dev *dev);
 168 static void igbvf_promiscuous_enable(struct rte_eth_dev *dev);
 169 static void igbvf_promiscuous_disable(struct rte_eth_dev *dev);
 170 static void igbvf_allmulticast_enable(struct rte_eth_dev *dev);
 171 static void igbvf_allmulticast_disable(struct rte_eth_dev *dev);
 172 static int eth_igbvf_link_update(struct e1000_hw *hw);
 173 static void eth_igbvf_stats_get(struct rte_eth_dev *dev,
 174                                 struct rte_eth_stats *rte_stats);
 175 static int eth_igbvf_xstats_get(struct rte_eth_dev *dev,
 176                                 struct rte_eth_xstat *xstats, unsigned n);
 177 static int eth_igbvf_xstats_get_names(struct rte_eth_dev *dev,
 178                                       struct rte_eth_xstat_name *xstats_names,
 179                                       unsigned limit);
 180 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev);
 181 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev,
 182                 uint16_t vlan_id, int on);
 183 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on);
 184 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 185 static void igbvf_default_mac_addr_set(struct rte_eth_dev *dev,
 186                 struct ether_addr *addr);
 187 static int igbvf_get_reg_length(struct rte_eth_dev *dev);
 188 static int igbvf_get_regs(struct rte_eth_dev *dev,
 189                 struct rte_dev_reg_info *regs);
 190
 191 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev,
 192                                    struct rte_eth_rss_reta_entry64 *reta_conf,
 193                                    uint16_t reta_size);
 194 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev,
 195                                   struct rte_eth_rss_reta_entry64 *reta_conf,
 196                                   uint16_t reta_size);
 197
 198 static int eth_igb_syn_filter_set(struct rte_eth_dev *dev,
 199                         struct rte_eth_syn_filter *filter,
 200                         bool add);
 201 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev,
 202                         struct rte_eth_syn_filter *filter);
 203 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
 204                         enum rte_filter_op filter_op,
 205                         void *arg);
 206 static int igb_add_2tuple_filter(struct rte_eth_dev *dev,
 207                         struct rte_eth_ntuple_filter *ntuple_filter);
 208 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev,
 209                         struct rte_eth_ntuple_filter *ntuple_filter);
 210 static int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
 211                         struct rte_eth_flex_filter *filter,
 212                         bool add);
 213 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev,
 214                         struct rte_eth_flex_filter *filter);
 215 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
 216                         enum rte_filter_op filter_op,
 217                         void *arg);
 218 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
 219                         struct rte_eth_ntuple_filter *ntuple_filter);
 220 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
 221                         struct rte_eth_ntuple_filter *ntuple_filter);
 222 static int igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
 223                         struct rte_eth_ntuple_filter *filter,
 224                         bool add);
 225 static int igb_get_ntuple_filter(struct rte_eth_dev *dev,
 226                         struct rte_eth_ntuple_filter *filter);
 227 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev,
 228                                 enum rte_filter_op filter_op,
 229                                 void *arg);
 230 static int igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
 231                         struct rte_eth_ethertype_filter *filter,
 232                         bool add);
 233 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev,
 234                                 enum rte_filter_op filter_op,
 235                                 void *arg);
 236 static int igb_get_ethertype_filter(struct rte_eth_dev *dev,
 237                         struct rte_eth_ethertype_filter *filter);
 238 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev,
 239                      enum rte_filter_type filter_type,
 240                      enum rte_filter_op filter_op,
 241                      void *arg);
 242 static int eth_igb_get_reg_length(struct rte_eth_dev *dev);
 243 static int eth_igb_get_regs(struct rte_eth_dev *dev,
 244                 struct rte_dev_reg_info *regs);
 245 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev);
 246 static int eth_igb_get_eeprom(struct rte_eth_dev *dev,
 247                 struct rte_dev_eeprom_info *eeprom);
 248 static int eth_igb_set_eeprom(struct rte_eth_dev *dev,
 249                 struct rte_dev_eeprom_info *eeprom);
 250 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
 251                                     struct ether_addr *mc_addr_set,
 252                                     uint32_t nb_mc_addr);
 253 static int igb_timesync_enable(struct rte_eth_dev *dev);
 254 static int igb_timesync_disable(struct rte_eth_dev *dev);
 255 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 256                                           struct timespec *timestamp,
 257                                           uint32_t flags);
 258 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 259                                           struct timespec *timestamp);
 260 static int igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta);
 261 static int igb_timesync_read_time(struct rte_eth_dev *dev,
 262                                   struct timespec *timestamp);
 263 static int igb_timesync_write_time(struct rte_eth_dev *dev,
 264                                    const struct timespec *timestamp);
 265 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev,
 266                                         uint16_t queue_id);
 267 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev,
 268                                          uint16_t queue_id);
 269 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction,
 270                                        uint8_t queue, uint8_t msix_vector);
 271 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector,
 272                                uint8_t index, uint8_t offset);
 273 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev);
 274 static void eth_igbvf_interrupt_handler(struct rte_intr_handle *handle,
 275                                         void *param);
 276 static void igbvf_mbx_process(struct rte_eth_dev *dev);
 277
 278 /*
 279  * Define VF Stats MACRO for Non "cleared on read" register
 280  */
 281 #define UPDATE_VF_STAT(reg, last, cur)            \
 282 {                                                 \
 283         u32 latest = E1000_READ_REG(hw, reg);     \
 284         cur += (latest - last) & UINT_MAX;        \
 285         last = latest;                            \
 286 }
 287
 288 #define IGB_FC_PAUSE_TIME 0x0680
 289 #define IGB_LINK_UPDATE_CHECK_TIMEOUT  90  /* 9s */
 290 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */
 291
 292 #define IGBVF_PMD_NAME "rte_igbvf_pmd"     /* PMD name */
 293
 294 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full;
 295
 296 /*
 297  * The set of PCI devices this driver supports
 298  */
 299 static const struct rte_pci_id pci_id_igb_map[] = {
 300
 301 #define RTE_PCI_DEV_ID_DECL_IGB(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 302 #include "rte_pci_dev_ids.h"
 303
 304 {0},
 305 };
 306
 307 /*
 308  * The set of PCI devices this driver supports (for 82576&I350 VF)
 309  */
 310 static const struct rte_pci_id pci_id_igbvf_map[] = {
 311
 312 #define RTE_PCI_DEV_ID_DECL_IGBVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 313 #include "rte_pci_dev_ids.h"
 314
 315 {0},
 316 };
 317
 318 static const struct rte_eth_desc_lim rx_desc_lim = {
 319         .nb_max = E1000_MAX_RING_DESC,
 320         .nb_min = E1000_MIN_RING_DESC,
 321         .nb_align = IGB_RXD_ALIGN,
 322 };
 323
 324 static const struct rte_eth_desc_lim tx_desc_lim = {
 325         .nb_max = E1000_MAX_RING_DESC,
 326         .nb_min = E1000_MIN_RING_DESC,
 327         .nb_align = IGB_RXD_ALIGN,
 328 };
 329
 330 static const struct eth_dev_ops eth_igb_ops = {
 331         .dev_configure        = eth_igb_configure,
 332         .dev_start            = eth_igb_start,
 333         .dev_stop             = eth_igb_stop,
 334         .dev_close            = eth_igb_close,
 335         .promiscuous_enable   = eth_igb_promiscuous_enable,
 336         .promiscuous_disable  = eth_igb_promiscuous_disable,
 337         .allmulticast_enable  = eth_igb_allmulticast_enable,
 338         .allmulticast_disable = eth_igb_allmulticast_disable,
 339         .link_update          = eth_igb_link_update,
 340         .stats_get            = eth_igb_stats_get,
 341         .xstats_get           = eth_igb_xstats_get,
 342         .xstats_get_names     = eth_igb_xstats_get_names,
 343         .stats_reset          = eth_igb_stats_reset,
 344         .xstats_reset         = eth_igb_xstats_reset,
 345         .dev_infos_get        = eth_igb_infos_get,
 346         .dev_supported_ptypes_get = eth_igb_supported_ptypes_get,
 347         .mtu_set              = eth_igb_mtu_set,
 348         .vlan_filter_set      = eth_igb_vlan_filter_set,
 349         .vlan_tpid_set        = eth_igb_vlan_tpid_set,
 350         .vlan_offload_set     = eth_igb_vlan_offload_set,
 351         .rx_queue_setup       = eth_igb_rx_queue_setup,
 352         .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable,
 353         .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable,
 354         .rx_queue_release     = eth_igb_rx_queue_release,
 355         .rx_queue_count       = eth_igb_rx_queue_count,
 356         .rx_descriptor_done   = eth_igb_rx_descriptor_done,
 357         .tx_queue_setup       = eth_igb_tx_queue_setup,
 358         .tx_queue_release     = eth_igb_tx_queue_release,
 359         .dev_led_on           = eth_igb_led_on,
 360         .dev_led_off          = eth_igb_led_off,
 361         .flow_ctrl_get        = eth_igb_flow_ctrl_get,
 362         .flow_ctrl_set        = eth_igb_flow_ctrl_set,
 363         .mac_addr_add         = eth_igb_rar_set,
 364         .mac_addr_remove      = eth_igb_rar_clear,
 365         .mac_addr_set         = eth_igb_default_mac_addr_set,
 366         .reta_update          = eth_igb_rss_reta_update,
 367         .reta_query           = eth_igb_rss_reta_query,
 368         .rss_hash_update      = eth_igb_rss_hash_update,
 369         .rss_hash_conf_get    = eth_igb_rss_hash_conf_get,
 370         .filter_ctrl          = eth_igb_filter_ctrl,
 371         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 372         .rxq_info_get         = igb_rxq_info_get,
 373         .txq_info_get         = igb_txq_info_get,
 374         .timesync_enable      = igb_timesync_enable,
 375         .timesync_disable     = igb_timesync_disable,
 376         .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp,
 377         .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp,
 378         .get_reg_length       = eth_igb_get_reg_length,
 379         .get_reg              = eth_igb_get_regs,
 380         .get_eeprom_length    = eth_igb_get_eeprom_length,
 381         .get_eeprom           = eth_igb_get_eeprom,
 382         .set_eeprom           = eth_igb_set_eeprom,
 383         .timesync_adjust_time = igb_timesync_adjust_time,
 384         .timesync_read_time   = igb_timesync_read_time,
 385         .timesync_write_time  = igb_timesync_write_time,
 386 };
 387
 388 /*
 389  * dev_ops for virtual function, bare necessities for basic vf
 390  * operation have been implemented
 391  */
 392 static const struct eth_dev_ops igbvf_eth_dev_ops = {
 393         .dev_configure        = igbvf_dev_configure,
 394         .dev_start            = igbvf_dev_start,
 395         .dev_stop             = igbvf_dev_stop,
 396         .dev_close            = igbvf_dev_close,
 397         .promiscuous_enable   = igbvf_promiscuous_enable,
 398         .promiscuous_disable  = igbvf_promiscuous_disable,
 399         .allmulticast_enable  = igbvf_allmulticast_enable,
 400         .allmulticast_disable = igbvf_allmulticast_disable,
 401         .link_update          = eth_igb_link_update,
 402         .stats_get            = eth_igbvf_stats_get,
 403         .xstats_get           = eth_igbvf_xstats_get,
 404         .xstats_get_names     = eth_igbvf_xstats_get_names,
 405         .stats_reset          = eth_igbvf_stats_reset,
 406         .xstats_reset         = eth_igbvf_stats_reset,
 407         .vlan_filter_set      = igbvf_vlan_filter_set,
 408         .dev_infos_get        = eth_igbvf_infos_get,
 409         .dev_supported_ptypes_get = eth_igb_supported_ptypes_get,
 410         .rx_queue_setup       = eth_igb_rx_queue_setup,
 411         .rx_queue_release     = eth_igb_rx_queue_release,
 412         .tx_queue_setup       = eth_igb_tx_queue_setup,
 413         .tx_queue_release     = eth_igb_tx_queue_release,
 414         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 415         .rxq_info_get         = igb_rxq_info_get,
 416         .txq_info_get         = igb_txq_info_get,
 417         .mac_addr_set         = igbvf_default_mac_addr_set,
 418         .get_reg_length       = igbvf_get_reg_length,
 419         .get_reg              = igbvf_get_regs,
 420 };
 421
 422 /* store statistics names and its offset in stats structure */
 423 struct rte_igb_xstats_name_off {
 424         char name[RTE_ETH_XSTATS_NAME_SIZE];
 425         unsigned offset;
 426 };
 427
 428 static const struct rte_igb_xstats_name_off rte_igb_stats_strings[] = {
 429         {"rx_crc_errors", offsetof(struct e1000_hw_stats, crcerrs)},
 430         {"rx_align_errors", offsetof(struct e1000_hw_stats, algnerrc)},
 431         {"rx_symbol_errors", offsetof(struct e1000_hw_stats, symerrs)},
 432         {"rx_missed_packets", offsetof(struct e1000_hw_stats, mpc)},
 433         {"tx_single_collision_packets", offsetof(struct e1000_hw_stats, scc)},
 434         {"tx_multiple_collision_packets", offsetof(struct e1000_hw_stats, mcc)},
 435         {"tx_excessive_collision_packets", offsetof(struct e1000_hw_stats,
 436                 ecol)},
 437         {"tx_late_collisions", offsetof(struct e1000_hw_stats, latecol)},
 438         {"tx_total_collisions", offsetof(struct e1000_hw_stats, colc)},
 439         {"tx_deferred_packets", offsetof(struct e1000_hw_stats, dc)},
 440         {"tx_no_carrier_sense_packets", offsetof(struct e1000_hw_stats, tncrs)},
 441         {"rx_carrier_ext_errors", offsetof(struct e1000_hw_stats, cexterr)},
 442         {"rx_length_errors", offsetof(struct e1000_hw_stats, rlec)},
 443         {"rx_xon_packets", offsetof(struct e1000_hw_stats, xonrxc)},
 444         {"tx_xon_packets", offsetof(struct e1000_hw_stats, xontxc)},
 445         {"rx_xoff_packets", offsetof(struct e1000_hw_stats, xoffrxc)},
 446         {"tx_xoff_packets", offsetof(struct e1000_hw_stats, xofftxc)},
 447         {"rx_flow_control_unsupported_packets", offsetof(struct e1000_hw_stats,
 448                 fcruc)},
 449         {"rx_size_64_packets", offsetof(struct e1000_hw_stats, prc64)},
 450         {"rx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, prc127)},
 451         {"rx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, prc255)},
 452         {"rx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, prc511)},
 453         {"rx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats,
 454                 prc1023)},
 455         {"rx_size_1024_to_max_packets", offsetof(struct e1000_hw_stats,
 456                 prc1522)},
 457         {"rx_broadcast_packets", offsetof(struct e1000_hw_stats, bprc)},
 458         {"rx_multicast_packets", offsetof(struct e1000_hw_stats, mprc)},
 459         {"rx_undersize_errors", offsetof(struct e1000_hw_stats, ruc)},
 460         {"rx_fragment_errors", offsetof(struct e1000_hw_stats, rfc)},
 461         {"rx_oversize_errors", offsetof(struct e1000_hw_stats, roc)},
 462         {"rx_jabber_errors", offsetof(struct e1000_hw_stats, rjc)},
 463         {"rx_management_packets", offsetof(struct e1000_hw_stats, mgprc)},
 464         {"rx_management_dropped", offsetof(struct e1000_hw_stats, mgpdc)},
 465         {"tx_management_packets", offsetof(struct e1000_hw_stats, mgptc)},
 466         {"rx_total_packets", offsetof(struct e1000_hw_stats, tpr)},
 467         {"tx_total_packets", offsetof(struct e1000_hw_stats, tpt)},
 468         {"rx_total_bytes", offsetof(struct e1000_hw_stats, tor)},
 469         {"tx_total_bytes", offsetof(struct e1000_hw_stats, tot)},
 470         {"tx_size_64_packets", offsetof(struct e1000_hw_stats, ptc64)},
 471         {"tx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, ptc127)},
 472         {"tx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, ptc255)},
 473         {"tx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, ptc511)},
 474         {"tx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats,
 475                 ptc1023)},
 476         {"tx_size_1023_to_max_packets", offsetof(struct e1000_hw_stats,
 477                 ptc1522)},
 478         {"tx_multicast_packets", offsetof(struct e1000_hw_stats, mptc)},
 479         {"tx_broadcast_packets", offsetof(struct e1000_hw_stats, bptc)},
 480         {"tx_tso_packets", offsetof(struct e1000_hw_stats, tsctc)},
 481         {"tx_tso_errors", offsetof(struct e1000_hw_stats, tsctfc)},
 482         {"rx_sent_to_host_packets", offsetof(struct e1000_hw_stats, rpthc)},
 483         {"tx_sent_by_host_packets", offsetof(struct e1000_hw_stats, hgptc)},
 484         {"rx_code_violation_packets", offsetof(struct e1000_hw_stats, scvpc)},
 485
 486         {"interrupt_assert_count", offsetof(struct e1000_hw_stats, iac)},
 487 };
 488
 489 #define IGB_NB_XSTATS (sizeof(rte_igb_stats_strings) / \
 490                 sizeof(rte_igb_stats_strings[0]))
 491
 492 static const struct rte_igb_xstats_name_off rte_igbvf_stats_strings[] = {
 493         {"rx_multicast_packets", offsetof(struct e1000_vf_stats, mprc)},
 494         {"rx_good_loopback_packets", offsetof(struct e1000_vf_stats, gprlbc)},
 495         {"tx_good_loopback_packets", offsetof(struct e1000_vf_stats, gptlbc)},
 496         {"rx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gorlbc)},
 497         {"tx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gotlbc)},
 498 };
 499
 500 #define IGBVF_NB_XSTATS (sizeof(rte_igbvf_stats_strings) / \
 501                 sizeof(rte_igbvf_stats_strings[0]))
 502
 503 /**
 504  * Atomically reads the link status information from global
 505  * structure rte_eth_dev.
 506  *
 507  * @param dev
 508  *   - Pointer to the structure rte_eth_dev to read from.
 509  *   - Pointer to the buffer to be saved with the link status.
 510  *
 511  * @return
 512  *   - On success, zero.
 513  *   - On failure, negative value.
 514  */
 515 static inline int
 516 rte_igb_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 517                                 struct rte_eth_link *link)
 518 {
 519         struct rte_eth_link *dst = link;
 520         struct rte_eth_link *src = &(dev->data->dev_link);
 521
 522         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 523                                         *(uint64_t *)src) == 0)
 524                 return -1;
 525
 526         return 0;
 527 }
 528
 529 /**
 530  * Atomically writes the link status information into global
 531  * structure rte_eth_dev.
 532  *
 533  * @param dev
 534  *   - Pointer to the structure rte_eth_dev to read from.
 535  *   - Pointer to the buffer to be saved with the link status.
 536  *
 537  * @return
 538  *   - On success, zero.
 539  *   - On failure, negative value.
 540  */
 541 static inline int
 542 rte_igb_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 543                                 struct rte_eth_link *link)
 544 {
 545         struct rte_eth_link *dst = &(dev->data->dev_link);
 546         struct rte_eth_link *src = link;
 547
 548         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 549                                         *(uint64_t *)src) == 0)
 550                 return -1;
 551
 552         return 0;
 553 }
 554
 555 static inline void
 556 igb_intr_enable(struct rte_eth_dev *dev)
 557 {
 558         struct e1000_interrupt *intr =
 559                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 560         struct e1000_hw *hw =
 561                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 562
 563         E1000_WRITE_REG(hw, E1000_IMS, intr->mask);
 564         E1000_WRITE_FLUSH(hw);
 565 }
 566
 567 static void
 568 igb_intr_disable(struct e1000_hw *hw)
 569 {
 570         E1000_WRITE_REG(hw, E1000_IMC, ~0);
 571         E1000_WRITE_FLUSH(hw);
 572 }
 573
 574 static inline void
 575 igbvf_intr_enable(struct rte_eth_dev *dev)
 576 {
 577         struct e1000_hw *hw =
 578                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 579
 580         /* only for mailbox */
 581         E1000_WRITE_REG(hw, E1000_EIAM, 1 << E1000_VTIVAR_MISC_MAILBOX);
 582         E1000_WRITE_REG(hw, E1000_EIAC, 1 << E1000_VTIVAR_MISC_MAILBOX);
 583         E1000_WRITE_REG(hw, E1000_EIMS, 1 << E1000_VTIVAR_MISC_MAILBOX);
 584         E1000_WRITE_FLUSH(hw);
 585 }
 586
 587 /* only for mailbox now. If RX/TX needed, should extend this function.  */
 588 static void
 589 igbvf_set_ivar_map(struct e1000_hw *hw, uint8_t msix_vector)
 590 {
 591         uint32_t tmp = 0;
 592
 593         /* mailbox */
 594         tmp |= (msix_vector & E1000_VTIVAR_MISC_INTR_MASK);
 595         tmp |= E1000_VTIVAR_VALID;
 596         E1000_WRITE_REG(hw, E1000_VTIVAR_MISC, tmp);
 597 }
 598
 599 static void
 600 eth_igbvf_configure_msix_intr(struct rte_eth_dev *dev)
 601 {
 602         struct e1000_hw *hw =
 603                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 604
 605         /* Configure VF other cause ivar */
 606         igbvf_set_ivar_map(hw, E1000_VTIVAR_MISC_MAILBOX);
 607 }
 608
 609 static inline int32_t
 610 igb_pf_reset_hw(struct e1000_hw *hw)
 611 {
 612         uint32_t ctrl_ext;
 613         int32_t status;
 614
 615         status = e1000_reset_hw(hw);
 616
 617         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 618         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 619         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 620         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 621         E1000_WRITE_FLUSH(hw);
 622
 623         return status;
 624 }
 625
 626 static void
 627 igb_identify_hardware(struct rte_eth_dev *dev)
 628 {
 629         struct e1000_hw *hw =
 630                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 631
 632         hw->vendor_id = dev->pci_dev->id.vendor_id;
 633         hw->device_id = dev->pci_dev->id.device_id;
 634         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
 635         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
 636
 637         e1000_set_mac_type(hw);
 638
 639         /* need to check if it is a vf device below */
 640 }
 641
 642 static int
 643 igb_reset_swfw_lock(struct e1000_hw *hw)
 644 {
 645         int ret_val;
 646
 647         /*
 648          * Do mac ops initialization manually here, since we will need
 649          * some function pointers set by this call.
 650          */
 651         ret_val = e1000_init_mac_params(hw);
 652         if (ret_val)
 653                 return ret_val;
 654
 655         /*
 656          * SMBI lock should not fail in this early stage. If this is the case,
 657          * it is due to an improper exit of the application.
 658          * So force the release of the faulty lock.
 659          */
 660         if (e1000_get_hw_semaphore_generic(hw) < 0) {
 661                 PMD_DRV_LOG(DEBUG, "SMBI lock released");
 662         }
 663         e1000_put_hw_semaphore_generic(hw);
 664
 665         if (hw->mac.ops.acquire_swfw_sync != NULL) {
 666                 uint16_t mask;
 667
 668                 /*
 669                  * Phy lock should not fail in this early stage. If this is the case,
 670                  * it is due to an improper exit of the application.
 671                  * So force the release of the faulty lock.
 672                  */
 673                 mask = E1000_SWFW_PHY0_SM << hw->bus.func;
 674                 if (hw->bus.func > E1000_FUNC_1)
 675                         mask <<= 2;
 676                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 677                         PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released",
 678                                     hw->bus.func);
 679                 }
 680                 hw->mac.ops.release_swfw_sync(hw, mask);
 681
 682                 /*
 683                  * This one is more tricky since it is common to all ports; but
 684                  * swfw_sync retries last long enough (1s) to be almost sure that if
 685                  * lock can not be taken it is due to an improper lock of the
 686                  * semaphore.
 687                  */
 688                 mask = E1000_SWFW_EEP_SM;
 689                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 690                         PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 691                 }
 692                 hw->mac.ops.release_swfw_sync(hw, mask);
 693         }
 694
 695         return E1000_SUCCESS;
 696 }
 697
 698 static int
 699 eth_igb_dev_init(struct rte_eth_dev *eth_dev)
 700 {
 701         int error = 0;
 702         struct rte_pci_device *pci_dev;
 703         struct e1000_hw *hw =
 704                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 705         struct e1000_vfta * shadow_vfta =
 706                 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 707         struct e1000_filter_info *filter_info =
 708                 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
 709         struct e1000_adapter *adapter =
 710                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 711
 712         uint32_t ctrl_ext;
 713
 714         pci_dev = eth_dev->pci_dev;
 715
 716         eth_dev->dev_ops = &eth_igb_ops;
 717         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 718         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 719
 720         /* for secondary processes, we don't initialise any further as primary
 721          * has already done this work. Only check we don't need a different
 722          * RX function */
 723         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 724                 if (eth_dev->data->scattered_rx)
 725                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 726                 return 0;
 727         }
 728
 729         rte_eth_copy_pci_info(eth_dev, pci_dev);
 730
 731         hw->hw_addr= (void *)pci_dev->mem_resource[0].addr;
 732
 733         igb_identify_hardware(eth_dev);
 734         if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) {
 735                 error = -EIO;
 736                 goto err_late;
 737         }
 738
 739         e1000_get_bus_info(hw);
 740
 741         /* Reset any pending lock */
 742         if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) {
 743                 error = -EIO;
 744                 goto err_late;
 745         }
 746
 747         /* Finish initialization */
 748         if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) {
 749                 error = -EIO;
 750                 goto err_late;
 751         }
 752
 753         hw->mac.autoneg = 1;
 754         hw->phy.autoneg_wait_to_complete = 0;
 755         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 756
 757         /* Copper options */
 758         if (hw->phy.media_type == e1000_media_type_copper) {
 759                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
 760                 hw->phy.disable_polarity_correction = 0;
 761                 hw->phy.ms_type = e1000_ms_hw_default;
 762         }
 763
 764         /*
 765          * Start from a known state, this is important in reading the nvm
 766          * and mac from that.
 767          */
 768         igb_pf_reset_hw(hw);
 769
 770         /* Make sure we have a good EEPROM before we read from it */
 771         if (e1000_validate_nvm_checksum(hw) < 0) {
 772                 /*
 773                  * Some PCI-E parts fail the first check due to
 774                  * the link being in sleep state, call it again,
 775                  * if it fails a second time its a real issue.
 776                  */
 777                 if (e1000_validate_nvm_checksum(hw) < 0) {
 778                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
 779                         error = -EIO;
 780                         goto err_late;
 781                 }
 782         }
 783
 784         /* Read the permanent MAC address out of the EEPROM */
 785         if (e1000_read_mac_addr(hw) != 0) {
 786                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
 787                 error = -EIO;
 788                 goto err_late;
 789         }
 790
 791         /* Allocate memory for storing MAC addresses */
 792         eth_dev->data->mac_addrs = rte_zmalloc("e1000",
 793                 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
 794         if (eth_dev->data->mac_addrs == NULL) {
 795                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to "
 796                                                 "store MAC addresses",
 797                                 ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 798                 error = -ENOMEM;
 799                 goto err_late;
 800         }
 801
 802         /* Copy the permanent MAC address */
 803         ether_addr_copy((struct ether_addr *)hw->mac.addr, &eth_dev->data->mac_addrs[0]);
 804
 805         /* initialize the vfta */
 806         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 807
 808         /* Now initialize the hardware */
 809         if (igb_hardware_init(hw) != 0) {
 810                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
 811                 rte_free(eth_dev->data->mac_addrs);
 812                 eth_dev->data->mac_addrs = NULL;
 813                 error = -ENODEV;
 814                 goto err_late;
 815         }
 816         hw->mac.get_link_status = 1;
 817         adapter->stopped = 0;
 818
 819         /* Indicate SOL/IDER usage */
 820         if (e1000_check_reset_block(hw) < 0) {
 821                 PMD_INIT_LOG(ERR, "PHY reset is blocked due to"
 822                                         "SOL/IDER session");
 823         }
 824
 825         /* initialize PF if max_vfs not zero */
 826         igb_pf_host_init(eth_dev);
 827
 828         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 829         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 830         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 831         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 832         E1000_WRITE_FLUSH(hw);
 833
 834         PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x",
 835                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 836                      pci_dev->id.device_id);
 837
 838         rte_intr_callback_register(&pci_dev->intr_handle,
 839                                    eth_igb_interrupt_handler,
 840                                    (void *)eth_dev);
 841
 842         /* enable uio/vfio intr/eventfd mapping */
 843         rte_intr_enable(&pci_dev->intr_handle);
 844
 845         /* enable support intr */
 846         igb_intr_enable(eth_dev);
 847
 848         TAILQ_INIT(&filter_info->flex_list);
 849         filter_info->flex_mask = 0;
 850         TAILQ_INIT(&filter_info->twotuple_list);
 851         filter_info->twotuple_mask = 0;
 852         TAILQ_INIT(&filter_info->fivetuple_list);
 853         filter_info->fivetuple_mask = 0;
 854
 855         return 0;
 856
 857 err_late:
 858         igb_hw_control_release(hw);
 859
 860         return error;
 861 }
 862
 863 static int
 864 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev)
 865 {
 866         struct rte_pci_device *pci_dev;
 867         struct e1000_hw *hw;
 868         struct e1000_adapter *adapter =
 869                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 870
 871         PMD_INIT_FUNC_TRACE();
 872
 873         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 874                 return -EPERM;
 875
 876         hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 877         pci_dev = eth_dev->pci_dev;
 878
 879         if (adapter->stopped == 0)
 880                 eth_igb_close(eth_dev);
 881
 882         eth_dev->dev_ops = NULL;
 883         eth_dev->rx_pkt_burst = NULL;
 884         eth_dev->tx_pkt_burst = NULL;
 885
 886         /* Reset any pending lock */
 887         igb_reset_swfw_lock(hw);
 888
 889         rte_free(eth_dev->data->mac_addrs);
 890         eth_dev->data->mac_addrs = NULL;
 891
 892         /* uninitialize PF if max_vfs not zero */
 893         igb_pf_host_uninit(eth_dev);
 894
 895         /* disable uio intr before callback unregister */
 896         rte_intr_disable(&(pci_dev->intr_handle));
 897         rte_intr_callback_unregister(&(pci_dev->intr_handle),
 898                 eth_igb_interrupt_handler, (void *)eth_dev);
 899
 900         return 0;
 901 }
 902
 903 /*
 904  * Virtual Function device init
 905  */
 906 static int
 907 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev)
 908 {
 909         struct rte_pci_device *pci_dev;
 910         struct e1000_adapter *adapter =
 911                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 912         struct e1000_hw *hw =
 913                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 914         int diag;
 915         struct ether_addr *perm_addr = (struct ether_addr *)hw->mac.perm_addr;
 916
 917         PMD_INIT_FUNC_TRACE();
 918
 919         eth_dev->dev_ops = &igbvf_eth_dev_ops;
 920         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 921         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 922
 923         /* for secondary processes, we don't initialise any further as primary
 924          * has already done this work. Only check we don't need a different
 925          * RX function */
 926         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 927                 if (eth_dev->data->scattered_rx)
 928                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 929                 return 0;
 930         }
 931
 932         pci_dev = eth_dev->pci_dev;
 933
 934         rte_eth_copy_pci_info(eth_dev, pci_dev);
 935
 936         hw->device_id = pci_dev->id.device_id;
 937         hw->vendor_id = pci_dev->id.vendor_id;
 938         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 939         adapter->stopped = 0;
 940
 941         /* Initialize the shared code (base driver) */
 942         diag = e1000_setup_init_funcs(hw, TRUE);
 943         if (diag != 0) {
 944                 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d",
 945                         diag);
 946                 return -EIO;
 947         }
 948
 949         /* init_mailbox_params */
 950         hw->mbx.ops.init_params(hw);
 951
 952         /* Disable the interrupts for VF */
 953         igbvf_intr_disable(hw);
 954
 955         diag = hw->mac.ops.reset_hw(hw);
 956
 957         /* Allocate memory for storing MAC addresses */
 958         eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN *
 959                 hw->mac.rar_entry_count, 0);
 960         if (eth_dev->data->mac_addrs == NULL) {
 961                 PMD_INIT_LOG(ERR,
 962                         "Failed to allocate %d bytes needed to store MAC "
 963                         "addresses",
 964                         ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 965                 return -ENOMEM;
 966         }
 967
 968         /* Generate a random MAC address, if none was assigned by PF. */
 969         if (is_zero_ether_addr(perm_addr)) {
 970                 eth_random_addr(perm_addr->addr_bytes);
 971                 diag = e1000_rar_set(hw, perm_addr->addr_bytes, 0);
 972                 if (diag) {
 973                         rte_free(eth_dev->data->mac_addrs);
 974                         eth_dev->data->mac_addrs = NULL;
 975                         return diag;
 976                 }
 977                 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF");
 978                 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address "
 979                              "%02x:%02x:%02x:%02x:%02x:%02x",
 980                              perm_addr->addr_bytes[0],
 981                              perm_addr->addr_bytes[1],
 982                              perm_addr->addr_bytes[2],
 983                              perm_addr->addr_bytes[3],
 984                              perm_addr->addr_bytes[4],
 985                              perm_addr->addr_bytes[5]);
 986         }
 987
 988         /* Copy the permanent MAC address */
 989         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 990                         &eth_dev->data->mac_addrs[0]);
 991
 992         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x "
 993                      "mac.type=%s",
 994                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 995                      pci_dev->id.device_id, "igb_mac_82576_vf");
 996
 997         rte_intr_callback_register(&pci_dev->intr_handle,
 998                                    eth_igbvf_interrupt_handler,
 999                                    (void *)eth_dev);
1000
1001         return 0;
1002 }
1003
1004 static int
1005 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev)
1006 {
1007         struct e1000_adapter *adapter =
1008                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
1009         struct rte_pci_device *pci_dev = eth_dev->pci_dev;
1010
1011         PMD_INIT_FUNC_TRACE();
1012
1013         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1014                 return -EPERM;
1015
1016         if (adapter->stopped == 0)
1017                 igbvf_dev_close(eth_dev);
1018
1019         eth_dev->dev_ops = NULL;
1020         eth_dev->rx_pkt_burst = NULL;
1021         eth_dev->tx_pkt_burst = NULL;
1022
1023         rte_free(eth_dev->data->mac_addrs);
1024         eth_dev->data->mac_addrs = NULL;
1025
1026         /* disable uio intr before callback unregister */
1027         rte_intr_disable(&pci_dev->intr_handle);
1028         rte_intr_callback_unregister(&pci_dev->intr_handle,
1029                                      eth_igbvf_interrupt_handler,
1030                                      (void *)eth_dev);
1031
1032         return 0;
1033 }
1034
1035 static struct eth_driver rte_igb_pmd = {
1036         .pci_drv = {
1037                 .name = "rte_igb_pmd",
1038                 .id_table = pci_id_igb_map,
1039                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
1040                         RTE_PCI_DRV_DETACHABLE,
1041         },
1042         .eth_dev_init = eth_igb_dev_init,
1043         .eth_dev_uninit = eth_igb_dev_uninit,
1044         .dev_private_size = sizeof(struct e1000_adapter),
1045 };
1046
1047 /*
1048  * virtual function driver struct
1049  */
1050 static struct eth_driver rte_igbvf_pmd = {
1051         .pci_drv = {
1052                 .name = "rte_igbvf_pmd",
1053                 .id_table = pci_id_igbvf_map,
1054                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
1055         },
1056         .eth_dev_init = eth_igbvf_dev_init,
1057         .eth_dev_uninit = eth_igbvf_dev_uninit,
1058         .dev_private_size = sizeof(struct e1000_adapter),
1059 };
1060
1061 static int
1062 rte_igb_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
1063 {
1064         rte_eth_driver_register(&rte_igb_pmd);
1065         return 0;
1066 }
1067
1068 static void
1069 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1070 {
1071         struct e1000_hw *hw =
1072                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1073         /* RCTL: enable VLAN filter since VMDq always use VLAN filter */
1074         uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL);
1075         rctl |= E1000_RCTL_VFE;
1076         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1077 }
1078
1079 /*
1080  * VF Driver initialization routine.
1081  * Invoked one at EAL init time.
1082  * Register itself as the [Virtual Poll Mode] Driver of PCI IGB devices.
1083  */
1084 static int
1085 rte_igbvf_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
1086 {
1087         PMD_INIT_FUNC_TRACE();
1088
1089         rte_eth_driver_register(&rte_igbvf_pmd);
1090         return 0;
1091 }
1092
1093 static int
1094 igb_check_mq_mode(struct rte_eth_dev *dev)
1095 {
1096         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
1097         enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode;
1098         uint16_t nb_rx_q = dev->data->nb_rx_queues;
1099         uint16_t nb_tx_q = dev->data->nb_rx_queues;
1100
1101         if ((rx_mq_mode & ETH_MQ_RX_DCB_FLAG) ||
1102             tx_mq_mode == ETH_MQ_TX_DCB ||
1103             tx_mq_mode == ETH_MQ_TX_VMDQ_DCB) {
1104                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
1105                 return -EINVAL;
1106         }
1107         if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
1108                 /* Check multi-queue mode.
1109                  * To no break software we accept ETH_MQ_RX_NONE as this might
1110                  * be used to turn off VLAN filter.
1111                  */
1112
1113                 if (rx_mq_mode == ETH_MQ_RX_NONE ||
1114                     rx_mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1115                         dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY;
1116                         RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1;
1117                 } else {
1118                         /* Only support one queue on VFs.
1119                          * RSS together with SRIOV is not supported.
1120                          */
1121                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1122                                         " wrong mq_mode rx %d.",
1123                                         rx_mq_mode);
1124                         return -EINVAL;
1125                 }
1126                 /* TX mode is not used here, so mode might be ignored.*/
1127                 if (tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) {
1128                         /* SRIOV only works in VMDq enable mode */
1129                         PMD_INIT_LOG(WARNING, "SRIOV is active,"
1130                                         " TX mode %d is not supported. "
1131                                         " Driver will behave as %d mode.",
1132                                         tx_mq_mode, ETH_MQ_TX_VMDQ_ONLY);
1133                 }
1134
1135                 /* check valid queue number */
1136                 if ((nb_rx_q > 1) || (nb_tx_q > 1)) {
1137                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1138                                         " only support one queue on VFs.");
1139                         return -EINVAL;
1140                 }
1141         } else {
1142                 /* To no break software that set invalid mode, only display
1143                  * warning if invalid mode is used.
1144                  */
1145                 if (rx_mq_mode != ETH_MQ_RX_NONE &&
1146                     rx_mq_mode != ETH_MQ_RX_VMDQ_ONLY &&
1147                     rx_mq_mode != ETH_MQ_RX_RSS) {
1148                         /* RSS together with VMDq not supported*/
1149                         PMD_INIT_LOG(ERR, "RX mode %d is not supported.",
1150                                      rx_mq_mode);
1151                         return -EINVAL;
1152                 }
1153
1154                 if (tx_mq_mode != ETH_MQ_TX_NONE &&
1155                     tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) {
1156                         PMD_INIT_LOG(WARNING, "TX mode %d is not supported."
1157                                         " Due to txmode is meaningless in this"
1158                                         " driver, just ignore.",
1159                                         tx_mq_mode);
1160                 }
1161         }
1162         return 0;
1163 }
1164
1165 static int
1166 eth_igb_configure(struct rte_eth_dev *dev)
1167 {
1168         struct e1000_interrupt *intr =
1169                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1170         int ret;
1171
1172         PMD_INIT_FUNC_TRACE();
1173
1174         /* multipe queue mode checking */
1175         ret  = igb_check_mq_mode(dev);
1176         if (ret != 0) {
1177                 PMD_DRV_LOG(ERR, "igb_check_mq_mode fails with %d.",
1178                             ret);
1179                 return ret;
1180         }
1181
1182         intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
1183         PMD_INIT_FUNC_TRACE();
1184
1185         return 0;
1186 }
1187
1188 static int
1189 eth_igb_start(struct rte_eth_dev *dev)
1190 {
1191         struct e1000_hw *hw =
1192                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1193         struct e1000_adapter *adapter =
1194                 E1000_DEV_PRIVATE(dev->data->dev_private);
1195         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1196         int ret, mask;
1197         uint32_t intr_vector = 0;
1198         uint32_t ctrl_ext;
1199         uint32_t *speeds;
1200         int num_speeds;
1201         bool autoneg;
1202
1203         PMD_INIT_FUNC_TRACE();
1204
1205         /* disable uio/vfio intr/eventfd mapping */
1206         rte_intr_disable(intr_handle);
1207
1208         /* Power up the phy. Needed to make the link go Up */
1209         e1000_power_up_phy(hw);
1210
1211         /*
1212          * Packet Buffer Allocation (PBA)
1213          * Writing PBA sets the receive portion of the buffer
1214          * the remainder is used for the transmit buffer.
1215          */
1216         if (hw->mac.type == e1000_82575) {
1217                 uint32_t pba;
1218
1219                 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
1220                 E1000_WRITE_REG(hw, E1000_PBA, pba);
1221         }
1222
1223         /* Put the address into the Receive Address Array */
1224         e1000_rar_set(hw, hw->mac.addr, 0);
1225
1226         /* Initialize the hardware */
1227         if (igb_hardware_init(hw)) {
1228                 PMD_INIT_LOG(ERR, "Unable to initialize the hardware");
1229                 return -EIO;
1230         }
1231         adapter->stopped = 0;
1232
1233         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1234
1235         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1236         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
1237         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
1238         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
1239         E1000_WRITE_FLUSH(hw);
1240
1241         /* configure PF module if SRIOV enabled */
1242         igb_pf_host_configure(dev);
1243
1244         /* check and configure queue intr-vector mapping */
1245         if ((rte_intr_cap_multiple(intr_handle) ||
1246              !RTE_ETH_DEV_SRIOV(dev).active) &&
1247             dev->data->dev_conf.intr_conf.rxq != 0) {
1248                 intr_vector = dev->data->nb_rx_queues;
1249                 if (rte_intr_efd_enable(intr_handle, intr_vector))
1250                         return -1;
1251         }
1252
1253         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
1254                 intr_handle->intr_vec =
1255                         rte_zmalloc("intr_vec",
1256                                     dev->data->nb_rx_queues * sizeof(int), 0);
1257                 if (intr_handle->intr_vec == NULL) {
1258                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
1259                                      " intr_vec\n", dev->data->nb_rx_queues);
1260                         return -ENOMEM;
1261                 }
1262         }
1263
1264         /* confiugre msix for rx interrupt */
1265         eth_igb_configure_msix_intr(dev);
1266
1267         /* Configure for OS presence */
1268         igb_init_manageability(hw);
1269
1270         eth_igb_tx_init(dev);
1271
1272         /* This can fail when allocating mbufs for descriptor rings */
1273         ret = eth_igb_rx_init(dev);
1274         if (ret) {
1275                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
1276                 igb_dev_clear_queues(dev);
1277                 return ret;
1278         }
1279
1280         e1000_clear_hw_cntrs_base_generic(hw);
1281
1282         /*
1283          * VLAN Offload Settings
1284          */
1285         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1286                         ETH_VLAN_EXTEND_MASK;
1287         eth_igb_vlan_offload_set(dev, mask);
1288
1289         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1290                 /* Enable VLAN filter since VMDq always use VLAN filter */
1291                 igb_vmdq_vlan_hw_filter_enable(dev);
1292         }
1293
1294         if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) ||
1295                 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) ||
1296                 (hw->mac.type == e1000_i211)) {
1297                 /* Configure EITR with the maximum possible value (0xFFFF) */
1298                 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF);
1299         }
1300
1301         /* Setup link speed and duplex */
1302         speeds = &dev->data->dev_conf.link_speeds;
1303         if (*speeds == ETH_LINK_SPEED_AUTONEG) {
1304                 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
1305         } else {
1306                 num_speeds = 0;
1307                 autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0;
1308
1309                 /* Reset */
1310                 hw->phy.autoneg_advertised = 0;
1311
1312                 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M |
1313                                 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M |
1314                                 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_FIXED)) {
1315                         num_speeds = -1;
1316                         goto error_invalid_config;
1317                 }
1318                 if (*speeds & ETH_LINK_SPEED_10M_HD) {
1319                         hw->phy.autoneg_advertised |= ADVERTISE_10_HALF;
1320                         num_speeds++;
1321                 }
1322                 if (*speeds & ETH_LINK_SPEED_10M) {
1323                         hw->phy.autoneg_advertised |= ADVERTISE_10_FULL;
1324                         num_speeds++;
1325                 }
1326                 if (*speeds & ETH_LINK_SPEED_100M_HD) {
1327                         hw->phy.autoneg_advertised |= ADVERTISE_100_HALF;
1328                         num_speeds++;
1329                 }
1330                 if (*speeds & ETH_LINK_SPEED_100M) {
1331                         hw->phy.autoneg_advertised |= ADVERTISE_100_FULL;
1332                         num_speeds++;
1333                 }
1334                 if (*speeds & ETH_LINK_SPEED_1G) {
1335                         hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL;
1336                         num_speeds++;
1337                 }
1338                 if (num_speeds == 0 || (!autoneg && (num_speeds > 1)))
1339                         goto error_invalid_config;
1340         }
1341
1342         e1000_setup_link(hw);
1343
1344         if (rte_intr_allow_others(intr_handle)) {
1345                 /* check if lsc interrupt is enabled */
1346                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1347                         eth_igb_lsc_interrupt_setup(dev);
1348         } else {
1349                 rte_intr_callback_unregister(intr_handle,
1350                                              eth_igb_interrupt_handler,
1351                                              (void *)dev);
1352                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1353                         PMD_INIT_LOG(INFO, "lsc won't enable because of"
1354                                      " no intr multiplex\n");
1355         }
1356
1357         /* check if rxq interrupt is enabled */
1358         if (dev->data->dev_conf.intr_conf.rxq != 0 &&
1359             rte_intr_dp_is_en(intr_handle))
1360                 eth_igb_rxq_interrupt_setup(dev);
1361
1362         /* enable uio/vfio intr/eventfd mapping */
1363         rte_intr_enable(intr_handle);
1364
1365         /* resume enabled intr since hw reset */
1366         igb_intr_enable(dev);
1367
1368         PMD_INIT_LOG(DEBUG, "<<");
1369
1370         return 0;
1371
1372 error_invalid_config:
1373         PMD_INIT_LOG(ERR, "Invalid advertised speeds (%u) for port %u",
1374                      dev->data->dev_conf.link_speeds, dev->data->port_id);
1375         igb_dev_clear_queues(dev);
1376         return -EINVAL;
1377 }
1378
1379 /*********************************************************************
1380  *
1381  *  This routine disables all traffic on the adapter by issuing a
1382  *  global reset on the MAC.
1383  *
1384  **********************************************************************/
1385 static void
1386 eth_igb_stop(struct rte_eth_dev *dev)
1387 {
1388         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1389         struct e1000_filter_info *filter_info =
1390                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
1391         struct rte_eth_link link;
1392         struct e1000_flex_filter *p_flex;
1393         struct e1000_5tuple_filter *p_5tuple, *p_5tuple_next;
1394         struct e1000_2tuple_filter *p_2tuple, *p_2tuple_next;
1395         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1396
1397         igb_intr_disable(hw);
1398
1399         /* disable intr eventfd mapping */
1400         rte_intr_disable(intr_handle);
1401
1402         igb_pf_reset_hw(hw);
1403         E1000_WRITE_REG(hw, E1000_WUC, 0);
1404
1405         /* Set bit for Go Link disconnect */
1406         if (hw->mac.type >= e1000_82580) {
1407                 uint32_t phpm_reg;
1408
1409                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1410                 phpm_reg |= E1000_82580_PM_GO_LINKD;
1411                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1412         }
1413
1414         /* Power down the phy. Needed to make the link go Down */
1415         if (hw->phy.media_type == e1000_media_type_copper)
1416                 e1000_power_down_phy(hw);
1417         else
1418                 e1000_shutdown_fiber_serdes_link(hw);
1419
1420         igb_dev_clear_queues(dev);
1421
1422         /* clear the recorded link status */
1423         memset(&link, 0, sizeof(link));
1424         rte_igb_dev_atomic_write_link_status(dev, &link);
1425
1426         /* Remove all flex filters of the device */
1427         while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) {
1428                 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries);
1429                 rte_free(p_flex);
1430         }
1431         filter_info->flex_mask = 0;
1432
1433         /* Remove all ntuple filters of the device */
1434         for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list);
1435              p_5tuple != NULL; p_5tuple = p_5tuple_next) {
1436                 p_5tuple_next = TAILQ_NEXT(p_5tuple, entries);
1437                 TAILQ_REMOVE(&filter_info->fivetuple_list,
1438                              p_5tuple, entries);
1439                 rte_free(p_5tuple);
1440         }
1441         filter_info->fivetuple_mask = 0;
1442         for (p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list);
1443              p_2tuple != NULL; p_2tuple = p_2tuple_next) {
1444                 p_2tuple_next = TAILQ_NEXT(p_2tuple, entries);
1445                 TAILQ_REMOVE(&filter_info->twotuple_list,
1446                              p_2tuple, entries);
1447                 rte_free(p_2tuple);
1448         }
1449         filter_info->twotuple_mask = 0;
1450
1451         if (!rte_intr_allow_others(intr_handle))
1452                 /* resume to the default handler */
1453                 rte_intr_callback_register(intr_handle,
1454                                            eth_igb_interrupt_handler,
1455                                            (void *)dev);
1456
1457         /* Clean datapath event and queue/vec mapping */
1458         rte_intr_efd_disable(intr_handle);
1459         if (intr_handle->intr_vec != NULL) {
1460                 rte_free(intr_handle->intr_vec);
1461                 intr_handle->intr_vec = NULL;
1462         }
1463 }
1464
1465 static void
1466 eth_igb_close(struct rte_eth_dev *dev)
1467 {
1468         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1469         struct e1000_adapter *adapter =
1470                 E1000_DEV_PRIVATE(dev->data->dev_private);
1471         struct rte_eth_link link;
1472         struct rte_pci_device *pci_dev;
1473
1474         eth_igb_stop(dev);
1475         adapter->stopped = 1;
1476
1477         e1000_phy_hw_reset(hw);
1478         igb_release_manageability(hw);
1479         igb_hw_control_release(hw);
1480
1481         /* Clear bit for Go Link disconnect */
1482         if (hw->mac.type >= e1000_82580) {
1483                 uint32_t phpm_reg;
1484
1485                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1486                 phpm_reg &= ~E1000_82580_PM_GO_LINKD;
1487                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1488         }
1489
1490         igb_dev_free_queues(dev);
1491
1492         pci_dev = dev->pci_dev;
1493         if (pci_dev->intr_handle.intr_vec) {
1494                 rte_free(pci_dev->intr_handle.intr_vec);
1495                 pci_dev->intr_handle.intr_vec = NULL;
1496         }
1497
1498         memset(&link, 0, sizeof(link));
1499         rte_igb_dev_atomic_write_link_status(dev, &link);
1500 }
1501
1502 static int
1503 igb_get_rx_buffer_size(struct e1000_hw *hw)
1504 {
1505         uint32_t rx_buf_size;
1506         if (hw->mac.type == e1000_82576) {
1507                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10;
1508         } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) {
1509                 /* PBS needs to be translated according to a lookup table */
1510                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf);
1511                 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size);
1512                 rx_buf_size = (rx_buf_size << 10);
1513         } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) {
1514                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10;
1515         } else {
1516                 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10;
1517         }
1518
1519         return rx_buf_size;
1520 }
1521
1522 /*********************************************************************
1523  *
1524  *  Initialize the hardware
1525  *
1526  **********************************************************************/
1527 static int
1528 igb_hardware_init(struct e1000_hw *hw)
1529 {
1530         uint32_t rx_buf_size;
1531         int diag;
1532
1533         /* Let the firmware know the OS is in control */
1534         igb_hw_control_acquire(hw);
1535
1536         /*
1537          * These parameters control the automatic generation (Tx) and
1538          * response (Rx) to Ethernet PAUSE frames.
1539          * - High water mark should allow for at least two standard size (1518)
1540          *   frames to be received after sending an XOFF.
1541          * - Low water mark works best when it is very near the high water mark.
1542          *   This allows the receiver to restart by sending XON when it has
1543          *   drained a bit. Here we use an arbitrary value of 1500 which will
1544          *   restart after one full frame is pulled from the buffer. There
1545          *   could be several smaller frames in the buffer and if so they will
1546          *   not trigger the XON until their total number reduces the buffer
1547          *   by 1500.
1548          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
1549          */
1550         rx_buf_size = igb_get_rx_buffer_size(hw);
1551
1552         hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2);
1553         hw->fc.low_water = hw->fc.high_water - 1500;
1554         hw->fc.pause_time = IGB_FC_PAUSE_TIME;
1555         hw->fc.send_xon = 1;
1556
1557         /* Set Flow control, use the tunable location if sane */
1558         if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4))
1559                 hw->fc.requested_mode = igb_fc_setting;
1560         else
1561                 hw->fc.requested_mode = e1000_fc_none;
1562
1563         /* Issue a global reset */
1564         igb_pf_reset_hw(hw);
1565         E1000_WRITE_REG(hw, E1000_WUC, 0);
1566
1567         diag = e1000_init_hw(hw);
1568         if (diag < 0)
1569                 return diag;
1570
1571         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1572         e1000_get_phy_info(hw);
1573         e1000_check_for_link(hw);
1574
1575         return 0;
1576 }
1577
1578 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */
1579 static void
1580 igb_read_stats_registers(struct e1000_hw *hw, struct e1000_hw_stats *stats)
1581 {
1582         int pause_frames;
1583
1584         uint64_t old_gprc  = stats->gprc;
1585         uint64_t old_gptc  = stats->gptc;
1586         uint64_t old_tpr   = stats->tpr;
1587         uint64_t old_tpt   = stats->tpt;
1588         uint64_t old_rpthc = stats->rpthc;
1589         uint64_t old_hgptc = stats->hgptc;
1590
1591         if(hw->phy.media_type == e1000_media_type_copper ||
1592             (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
1593                 stats->symerrs +=
1594                     E1000_READ_REG(hw,E1000_SYMERRS);
1595                 stats->sec += E1000_READ_REG(hw, E1000_SEC);
1596         }
1597
1598         stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
1599         stats->mpc += E1000_READ_REG(hw, E1000_MPC);
1600         stats->scc += E1000_READ_REG(hw, E1000_SCC);
1601         stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
1602
1603         stats->mcc += E1000_READ_REG(hw, E1000_MCC);
1604         stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
1605         stats->colc += E1000_READ_REG(hw, E1000_COLC);
1606         stats->dc += E1000_READ_REG(hw, E1000_DC);
1607         stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
1608         stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
1609         stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
1610         /*
1611         ** For watchdog management we need to know if we have been
1612         ** paused during the last interval, so capture that here.
1613         */
1614         pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC);
1615         stats->xoffrxc += pause_frames;
1616         stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
1617         stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
1618         stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
1619         stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
1620         stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
1621         stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
1622         stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
1623         stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
1624         stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
1625         stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
1626         stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
1627         stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
1628
1629         /* For the 64-bit byte counters the low dword must be read first. */
1630         /* Both registers clear on the read of the high dword */
1631
1632         /* Workaround CRC bytes included in size, take away 4 bytes/packet */
1633         stats->gorc += E1000_READ_REG(hw, E1000_GORCL);
1634         stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32);
1635         stats->gorc -= (stats->gprc - old_gprc) * ETHER_CRC_LEN;
1636         stats->gotc += E1000_READ_REG(hw, E1000_GOTCL);
1637         stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32);
1638         stats->gotc -= (stats->gptc - old_gptc) * ETHER_CRC_LEN;
1639
1640         stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
1641         stats->ruc += E1000_READ_REG(hw, E1000_RUC);
1642         stats->rfc += E1000_READ_REG(hw, E1000_RFC);
1643         stats->roc += E1000_READ_REG(hw, E1000_ROC);
1644         stats->rjc += E1000_READ_REG(hw, E1000_RJC);
1645
1646         stats->tpr += E1000_READ_REG(hw, E1000_TPR);
1647         stats->tpt += E1000_READ_REG(hw, E1000_TPT);
1648
1649         stats->tor += E1000_READ_REG(hw, E1000_TORL);
1650         stats->tor += ((uint64_t)E1000_READ_REG(hw, E1000_TORH) << 32);
1651         stats->tor -= (stats->tpr - old_tpr) * ETHER_CRC_LEN;
1652         stats->tot += E1000_READ_REG(hw, E1000_TOTL);
1653         stats->tot += ((uint64_t)E1000_READ_REG(hw, E1000_TOTH) << 32);
1654         stats->tot -= (stats->tpt - old_tpt) * ETHER_CRC_LEN;
1655
1656         stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
1657         stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
1658         stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
1659         stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
1660         stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
1661         stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
1662         stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
1663         stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
1664
1665         /* Interrupt Counts */
1666
1667         stats->iac += E1000_READ_REG(hw, E1000_IAC);
1668         stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
1669         stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
1670         stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
1671         stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
1672         stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
1673         stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
1674         stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
1675         stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
1676
1677         /* Host to Card Statistics */
1678
1679         stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
1680         stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
1681         stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
1682         stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
1683         stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
1684         stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
1685         stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
1686         stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL);
1687         stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32);
1688         stats->hgorc -= (stats->rpthc - old_rpthc) * ETHER_CRC_LEN;
1689         stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL);
1690         stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32);
1691         stats->hgotc -= (stats->hgptc - old_hgptc) * ETHER_CRC_LEN;
1692         stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
1693         stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
1694         stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
1695
1696         stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
1697         stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
1698         stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
1699         stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
1700         stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
1701         stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
1702 }
1703
1704 static void
1705 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1706 {
1707         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1708         struct e1000_hw_stats *stats =
1709                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1710
1711         igb_read_stats_registers(hw, stats);
1712
1713         if (rte_stats == NULL)
1714                 return;
1715
1716         /* Rx Errors */
1717         rte_stats->imissed = stats->mpc;
1718         rte_stats->ierrors = stats->crcerrs +
1719                              stats->rlec + stats->ruc + stats->roc +
1720                              stats->rxerrc + stats->algnerrc + stats->cexterr;
1721
1722         /* Tx Errors */
1723         rte_stats->oerrors = stats->ecol + stats->latecol;
1724
1725         rte_stats->ipackets = stats->gprc;
1726         rte_stats->opackets = stats->gptc;
1727         rte_stats->ibytes   = stats->gorc;
1728         rte_stats->obytes   = stats->gotc;
1729 }
1730
1731 static void
1732 eth_igb_stats_reset(struct rte_eth_dev *dev)
1733 {
1734         struct e1000_hw_stats *hw_stats =
1735                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1736
1737         /* HW registers are cleared on read */
1738         eth_igb_stats_get(dev, NULL);
1739
1740         /* Reset software totals */
1741         memset(hw_stats, 0, sizeof(*hw_stats));
1742 }
1743
1744 static void
1745 eth_igb_xstats_reset(struct rte_eth_dev *dev)
1746 {
1747         struct e1000_hw_stats *stats =
1748                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1749
1750         /* HW registers are cleared on read */
1751         eth_igb_xstats_get(dev, NULL, IGB_NB_XSTATS);
1752
1753         /* Reset software totals */
1754         memset(stats, 0, sizeof(*stats));
1755 }
1756
1757 static int eth_igb_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1758         struct rte_eth_xstat_name *xstats_names,
1759         __rte_unused unsigned limit)
1760 {
1761         unsigned i;
1762
1763         if (xstats_names == NULL)
1764                 return IGB_NB_XSTATS;
1765
1766         /* Note: limit checked in rte_eth_xstats_names() */
1767
1768         for (i = 0; i < IGB_NB_XSTATS; i++) {
1769                 snprintf(xstats_names[i].name, sizeof(xstats_names[i].name),
1770                          "%s", rte_igb_stats_strings[i].name);
1771                 xstats_names[i].id = i;
1772         }
1773
1774         return IGB_NB_XSTATS;
1775 }
1776
1777 static int
1778 eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1779                    unsigned n)
1780 {
1781         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1782         struct e1000_hw_stats *hw_stats =
1783                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1784         unsigned i;
1785
1786         if (n < IGB_NB_XSTATS)
1787                 return IGB_NB_XSTATS;
1788
1789         igb_read_stats_registers(hw, hw_stats);
1790
1791         /* If this is a reset xstats is NULL, and we have cleared the
1792          * registers by reading them.
1793          */
1794         if (!xstats)
1795                 return 0;
1796
1797         /* Extended stats */
1798         for (i = 0; i < IGB_NB_XSTATS; i++) {
1799                 xstats[i].id = i;
1800                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
1801                         rte_igb_stats_strings[i].offset);
1802         }
1803
1804         return IGB_NB_XSTATS;
1805 }
1806
1807 static void
1808 igbvf_read_stats_registers(struct e1000_hw *hw, struct e1000_vf_stats *hw_stats)
1809 {
1810         /* Good Rx packets, include VF loopback */
1811         UPDATE_VF_STAT(E1000_VFGPRC,
1812             hw_stats->last_gprc, hw_stats->gprc);
1813
1814         /* Good Rx octets, include VF loopback */
1815         UPDATE_VF_STAT(E1000_VFGORC,
1816             hw_stats->last_gorc, hw_stats->gorc);
1817
1818         /* Good Tx packets, include VF loopback */
1819         UPDATE_VF_STAT(E1000_VFGPTC,
1820             hw_stats->last_gptc, hw_stats->gptc);
1821
1822         /* Good Tx octets, include VF loopback */
1823         UPDATE_VF_STAT(E1000_VFGOTC,
1824             hw_stats->last_gotc, hw_stats->gotc);
1825
1826         /* Rx Multicst packets */
1827         UPDATE_VF_STAT(E1000_VFMPRC,
1828             hw_stats->last_mprc, hw_stats->mprc);
1829
1830         /* Good Rx loopback packets */
1831         UPDATE_VF_STAT(E1000_VFGPRLBC,
1832             hw_stats->last_gprlbc, hw_stats->gprlbc);
1833
1834         /* Good Rx loopback octets */
1835         UPDATE_VF_STAT(E1000_VFGORLBC,
1836             hw_stats->last_gorlbc, hw_stats->gorlbc);
1837
1838         /* Good Tx loopback packets */
1839         UPDATE_VF_STAT(E1000_VFGPTLBC,
1840             hw_stats->last_gptlbc, hw_stats->gptlbc);
1841
1842         /* Good Tx loopback octets */
1843         UPDATE_VF_STAT(E1000_VFGOTLBC,
1844             hw_stats->last_gotlbc, hw_stats->gotlbc);
1845 }
1846
1847 static int eth_igbvf_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1848                                      struct rte_eth_xstat_name *xstats_names,
1849                                      __rte_unused unsigned limit)
1850 {
1851         unsigned i;
1852
1853         if (xstats_names != NULL)
1854                 for (i = 0; i < IGBVF_NB_XSTATS; i++) {
1855                         snprintf(xstats_names[i].name,
1856                                 sizeof(xstats_names[i].name), "%s",
1857                                 rte_igbvf_stats_strings[i].name);
1858                         xstats_names[i].id = i;
1859                 }
1860         return IGBVF_NB_XSTATS;
1861 }
1862
1863 static int
1864 eth_igbvf_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1865                      unsigned n)
1866 {
1867         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1868         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *)
1869                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1870         unsigned i;
1871
1872         if (n < IGBVF_NB_XSTATS)
1873                 return IGBVF_NB_XSTATS;
1874
1875         igbvf_read_stats_registers(hw, hw_stats);
1876
1877         if (!xstats)
1878                 return 0;
1879
1880         for (i = 0; i < IGBVF_NB_XSTATS; i++) {
1881                 xstats[i].id = i;
1882                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
1883                         rte_igbvf_stats_strings[i].offset);
1884         }
1885
1886         return IGBVF_NB_XSTATS;
1887 }
1888
1889 static void
1890 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1891 {
1892         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1893         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *)
1894                           E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1895
1896         igbvf_read_stats_registers(hw, hw_stats);
1897
1898         if (rte_stats == NULL)
1899                 return;
1900
1901         rte_stats->ipackets = hw_stats->gprc;
1902         rte_stats->ibytes = hw_stats->gorc;
1903         rte_stats->opackets = hw_stats->gptc;
1904         rte_stats->obytes = hw_stats->gotc;
1905 }
1906
1907 static void
1908 eth_igbvf_stats_reset(struct rte_eth_dev *dev)
1909 {
1910         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1911                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1912
1913         /* Sync HW register to the last stats */
1914         eth_igbvf_stats_get(dev, NULL);
1915
1916         /* reset HW current stats*/
1917         memset(&hw_stats->gprc, 0, sizeof(*hw_stats) -
1918                offsetof(struct e1000_vf_stats, gprc));
1919 }
1920
1921 static void
1922 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1923 {
1924         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1925
1926         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1927         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1928         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1929         dev_info->rx_offload_capa =
1930                 DEV_RX_OFFLOAD_VLAN_STRIP |
1931                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1932                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1933                 DEV_RX_OFFLOAD_TCP_CKSUM;
1934         dev_info->tx_offload_capa =
1935                 DEV_TX_OFFLOAD_VLAN_INSERT |
1936                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1937                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1938                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1939                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
1940                 DEV_TX_OFFLOAD_TCP_TSO;
1941
1942         switch (hw->mac.type) {
1943         case e1000_82575:
1944                 dev_info->max_rx_queues = 4;
1945                 dev_info->max_tx_queues = 4;
1946                 dev_info->max_vmdq_pools = 0;
1947                 break;
1948
1949         case e1000_82576:
1950                 dev_info->max_rx_queues = 16;
1951                 dev_info->max_tx_queues = 16;
1952                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1953                 dev_info->vmdq_queue_num = 16;
1954                 break;
1955
1956         case e1000_82580:
1957                 dev_info->max_rx_queues = 8;
1958                 dev_info->max_tx_queues = 8;
1959                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1960                 dev_info->vmdq_queue_num = 8;
1961                 break;
1962
1963         case e1000_i350:
1964                 dev_info->max_rx_queues = 8;
1965                 dev_info->max_tx_queues = 8;
1966                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1967                 dev_info->vmdq_queue_num = 8;
1968                 break;
1969
1970         case e1000_i354:
1971                 dev_info->max_rx_queues = 8;
1972                 dev_info->max_tx_queues = 8;
1973                 break;
1974
1975         case e1000_i210:
1976                 dev_info->max_rx_queues = 4;
1977                 dev_info->max_tx_queues = 4;
1978                 dev_info->max_vmdq_pools = 0;
1979                 break;
1980
1981         case e1000_i211:
1982                 dev_info->max_rx_queues = 2;
1983                 dev_info->max_tx_queues = 2;
1984                 dev_info->max_vmdq_pools = 0;
1985                 break;
1986
1987         default:
1988                 /* Should not happen */
1989                 break;
1990         }
1991         dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t);
1992         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
1993         dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL;
1994
1995         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1996                 .rx_thresh = {
1997                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1998                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1999                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
2000                 },
2001                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
2002                 .rx_drop_en = 0,
2003         };
2004
2005         dev_info->default_txconf = (struct rte_eth_txconf) {
2006                 .tx_thresh = {
2007                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
2008                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
2009                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
2010                 },
2011                 .txq_flags = 0,
2012         };
2013
2014         dev_info->rx_desc_lim = rx_desc_lim;
2015         dev_info->tx_desc_lim = tx_desc_lim;
2016
2017         dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M |
2018                         ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M |
2019                         ETH_LINK_SPEED_1G;
2020 }
2021
2022 static const uint32_t *
2023 eth_igb_supported_ptypes_get(struct rte_eth_dev *dev)
2024 {
2025         static const uint32_t ptypes[] = {
2026                 /* refers to igb_rxd_pkt_info_to_pkt_type() */
2027                 RTE_PTYPE_L2_ETHER,
2028                 RTE_PTYPE_L3_IPV4,
2029                 RTE_PTYPE_L3_IPV4_EXT,
2030                 RTE_PTYPE_L3_IPV6,
2031                 RTE_PTYPE_L3_IPV6_EXT,
2032                 RTE_PTYPE_L4_TCP,
2033                 RTE_PTYPE_L4_UDP,
2034                 RTE_PTYPE_L4_SCTP,
2035                 RTE_PTYPE_TUNNEL_IP,
2036                 RTE_PTYPE_INNER_L3_IPV6,
2037                 RTE_PTYPE_INNER_L3_IPV6_EXT,
2038                 RTE_PTYPE_INNER_L4_TCP,
2039                 RTE_PTYPE_INNER_L4_UDP,
2040                 RTE_PTYPE_UNKNOWN
2041         };
2042
2043         if (dev->rx_pkt_burst == eth_igb_recv_pkts ||
2044             dev->rx_pkt_burst == eth_igb_recv_scattered_pkts)
2045                 return ptypes;
2046         return NULL;
2047 }
2048
2049 static void
2050 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2051 {
2052         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2053
2054         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
2055         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
2056         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
2057         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
2058                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2059                                 DEV_RX_OFFLOAD_UDP_CKSUM  |
2060                                 DEV_RX_OFFLOAD_TCP_CKSUM;
2061         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
2062                                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
2063                                 DEV_TX_OFFLOAD_UDP_CKSUM   |
2064                                 DEV_TX_OFFLOAD_TCP_CKSUM   |
2065                                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
2066                                 DEV_TX_OFFLOAD_TCP_TSO;
2067         switch (hw->mac.type) {
2068         case e1000_vfadapt:
2069                 dev_info->max_rx_queues = 2;
2070                 dev_info->max_tx_queues = 2;
2071                 break;
2072         case e1000_vfadapt_i350:
2073                 dev_info->max_rx_queues = 1;
2074                 dev_info->max_tx_queues = 1;
2075                 break;
2076         default:
2077                 /* Should not happen */
2078                 break;
2079         }
2080
2081         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2082                 .rx_thresh = {
2083                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
2084                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
2085                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
2086                 },
2087                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
2088                 .rx_drop_en = 0,
2089         };
2090
2091         dev_info->default_txconf = (struct rte_eth_txconf) {
2092                 .tx_thresh = {
2093                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
2094                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
2095                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
2096                 },
2097                 .txq_flags = 0,
2098         };
2099
2100         dev_info->rx_desc_lim = rx_desc_lim;
2101         dev_info->tx_desc_lim = tx_desc_lim;
2102 }
2103
2104 /* return 0 means link status changed, -1 means not changed */
2105 static int
2106 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete)
2107 {
2108         struct e1000_hw *hw =
2109                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2110         struct rte_eth_link link, old;
2111         int link_check, count;
2112
2113         link_check = 0;
2114         hw->mac.get_link_status = 1;
2115
2116         /* possible wait-to-complete in up to 9 seconds */
2117         for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) {
2118                 /* Read the real link status */
2119                 switch (hw->phy.media_type) {
2120                 case e1000_media_type_copper:
2121                         /* Do the work to read phy */
2122                         e1000_check_for_link(hw);
2123                         link_check = !hw->mac.get_link_status;
2124                         break;
2125
2126                 case e1000_media_type_fiber:
2127                         e1000_check_for_link(hw);
2128                         link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2129                                       E1000_STATUS_LU);
2130                         break;
2131
2132                 case e1000_media_type_internal_serdes:
2133                         e1000_check_for_link(hw);
2134                         link_check = hw->mac.serdes_has_link;
2135                         break;
2136
2137                 /* VF device is type_unknown */
2138                 case e1000_media_type_unknown:
2139                         eth_igbvf_link_update(hw);
2140                         link_check = !hw->mac.get_link_status;
2141                         break;
2142
2143                 default:
2144                         break;
2145                 }
2146                 if (link_check || wait_to_complete == 0)
2147                         break;
2148                 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL);
2149         }
2150         memset(&link, 0, sizeof(link));
2151         rte_igb_dev_atomic_read_link_status(dev, &link);
2152         old = link;
2153
2154         /* Now we check if a transition has happened */
2155         if (link_check) {
2156                 uint16_t duplex, speed;
2157                 hw->mac.ops.get_link_up_info(hw, &speed, &duplex);
2158                 link.link_duplex = (duplex == FULL_DUPLEX) ?
2159                                 ETH_LINK_FULL_DUPLEX :
2160                                 ETH_LINK_HALF_DUPLEX;
2161                 link.link_speed = speed;
2162                 link.link_status = ETH_LINK_UP;
2163                 link.link_autoneg = !(dev->data->dev_conf.link_speeds &
2164                                 ETH_LINK_SPEED_FIXED);
2165         } else if (!link_check) {
2166                 link.link_speed = 0;
2167                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
2168                 link.link_status = ETH_LINK_DOWN;
2169                 link.link_autoneg = ETH_LINK_SPEED_FIXED;
2170         }
2171         rte_igb_dev_atomic_write_link_status(dev, &link);
2172
2173         /* not changed */
2174         if (old.link_status == link.link_status)
2175                 return -1;
2176
2177         /* changed */
2178         return 0;
2179 }
2180
2181 /*
2182  * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
2183  * For ASF and Pass Through versions of f/w this means
2184  * that the driver is loaded.
2185  */
2186 static void
2187 igb_hw_control_acquire(struct e1000_hw *hw)
2188 {
2189         uint32_t ctrl_ext;
2190
2191         /* Let firmware know the driver has taken over */
2192         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
2193         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
2194 }
2195
2196 /*
2197  * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
2198  * For ASF and Pass Through versions of f/w this means that the
2199  * driver is no longer loaded.
2200  */
2201 static void
2202 igb_hw_control_release(struct e1000_hw *hw)
2203 {
2204         uint32_t ctrl_ext;
2205
2206         /* Let firmware taken over control of h/w */
2207         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
2208         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
2209                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
2210 }
2211
2212 /*
2213  * Bit of a misnomer, what this really means is
2214  * to enable OS management of the system... aka
2215  * to disable special hardware management features.
2216  */
2217 static void
2218 igb_init_manageability(struct e1000_hw *hw)
2219 {
2220         if (e1000_enable_mng_pass_thru(hw)) {
2221                 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H);
2222                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
2223
2224                 /* disable hardware interception of ARP */
2225                 manc &= ~(E1000_MANC_ARP_EN);
2226
2227                 /* enable receiving management packets to the host */
2228                 manc |= E1000_MANC_EN_MNG2HOST;
2229                 manc2h |= 1 << 5;  /* Mng Port 623 */
2230                 manc2h |= 1 << 6;  /* Mng Port 664 */
2231                 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h);
2232                 E1000_WRITE_REG(hw, E1000_MANC, manc);
2233         }
2234 }
2235
2236 static void
2237 igb_release_manageability(struct e1000_hw *hw)
2238 {
2239         if (e1000_enable_mng_pass_thru(hw)) {
2240                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
2241
2242                 manc |= E1000_MANC_ARP_EN;
2243                 manc &= ~E1000_MANC_EN_MNG2HOST;
2244
2245                 E1000_WRITE_REG(hw, E1000_MANC, manc);
2246         }
2247 }
2248
2249 static void
2250 eth_igb_promiscuous_enable(struct rte_eth_dev *dev)
2251 {
2252         struct e1000_hw *hw =
2253                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2254         uint32_t rctl;
2255
2256         rctl = E1000_READ_REG(hw, E1000_RCTL);
2257         rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2258         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2259 }
2260
2261 static void
2262 eth_igb_promiscuous_disable(struct rte_eth_dev *dev)
2263 {
2264         struct e1000_hw *hw =
2265                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2266         uint32_t rctl;
2267
2268         rctl = E1000_READ_REG(hw, E1000_RCTL);
2269         rctl &= (~E1000_RCTL_UPE);
2270         if (dev->data->all_multicast == 1)
2271                 rctl |= E1000_RCTL_MPE;
2272         else
2273                 rctl &= (~E1000_RCTL_MPE);
2274         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2275 }
2276
2277 static void
2278 eth_igb_allmulticast_enable(struct rte_eth_dev *dev)
2279 {
2280         struct e1000_hw *hw =
2281                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2282         uint32_t rctl;
2283
2284         rctl = E1000_READ_REG(hw, E1000_RCTL);
2285         rctl |= E1000_RCTL_MPE;
2286         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2287 }
2288
2289 static void
2290 eth_igb_allmulticast_disable(struct rte_eth_dev *dev)
2291 {
2292         struct e1000_hw *hw =
2293                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2294         uint32_t rctl;
2295
2296         if (dev->data->promiscuous == 1)
2297                 return; /* must remain in all_multicast mode */
2298         rctl = E1000_READ_REG(hw, E1000_RCTL);
2299         rctl &= (~E1000_RCTL_MPE);
2300         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2301 }
2302
2303 static int
2304 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2305 {
2306         struct e1000_hw *hw =
2307                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2308         struct e1000_vfta * shadow_vfta =
2309                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2310         uint32_t vfta;
2311         uint32_t vid_idx;
2312         uint32_t vid_bit;
2313
2314         vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) &
2315                               E1000_VFTA_ENTRY_MASK);
2316         vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK));
2317         vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx);
2318         if (on)
2319                 vfta |= vid_bit;
2320         else
2321                 vfta &= ~vid_bit;
2322         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta);
2323
2324         /* update local VFTA copy */
2325         shadow_vfta->vfta[vid_idx] = vfta;
2326
2327         return 0;
2328 }
2329
2330 static int
2331 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev,
2332                       enum rte_vlan_type vlan_type,
2333                       uint16_t tpid)
2334 {
2335         struct e1000_hw *hw =
2336                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2337         uint32_t reg = ETHER_TYPE_VLAN;
2338         int ret = 0;
2339
2340         switch (vlan_type) {
2341         case ETH_VLAN_TYPE_INNER:
2342                 reg |= (tpid << 16);
2343                 E1000_WRITE_REG(hw, E1000_VET, reg);
2344                 break;
2345         default:
2346                 ret = -EINVAL;
2347                 PMD_DRV_LOG(ERR, "Unsupported vlan type %d\n", vlan_type);
2348                 break;
2349         }
2350
2351         return ret;
2352 }
2353
2354 static void
2355 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev)
2356 {
2357         struct e1000_hw *hw =
2358                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2359         uint32_t reg;
2360
2361         /* Filter Table Disable */
2362         reg = E1000_READ_REG(hw, E1000_RCTL);
2363         reg &= ~E1000_RCTL_CFIEN;
2364         reg &= ~E1000_RCTL_VFE;
2365         E1000_WRITE_REG(hw, E1000_RCTL, reg);
2366 }
2367
2368 static void
2369 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev)
2370 {
2371         struct e1000_hw *hw =
2372                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2373         struct e1000_vfta * shadow_vfta =
2374                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2375         uint32_t reg;
2376         int i;
2377
2378         /* Filter Table Enable, CFI not used for packet acceptance */
2379         reg = E1000_READ_REG(hw, E1000_RCTL);
2380         reg &= ~E1000_RCTL_CFIEN;
2381         reg |= E1000_RCTL_VFE;
2382         E1000_WRITE_REG(hw, E1000_RCTL, reg);
2383
2384         /* restore VFTA table */
2385         for (i = 0; i < IGB_VFTA_SIZE; i++)
2386                 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]);
2387 }
2388
2389 static void
2390 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev)
2391 {
2392         struct e1000_hw *hw =
2393                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2394         uint32_t reg;
2395
2396         /* VLAN Mode Disable */
2397         reg = E1000_READ_REG(hw, E1000_CTRL);
2398         reg &= ~E1000_CTRL_VME;
2399         E1000_WRITE_REG(hw, E1000_CTRL, reg);
2400 }
2401
2402 static void
2403 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev)
2404 {
2405         struct e1000_hw *hw =
2406                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2407         uint32_t reg;
2408
2409         /* VLAN Mode Enable */
2410         reg = E1000_READ_REG(hw, E1000_CTRL);
2411         reg |= E1000_CTRL_VME;
2412         E1000_WRITE_REG(hw, E1000_CTRL, reg);
2413 }
2414
2415 static void
2416 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev)
2417 {
2418         struct e1000_hw *hw =
2419                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2420         uint32_t reg;
2421
2422         /* CTRL_EXT: Extended VLAN */
2423         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
2424         reg &= ~E1000_CTRL_EXT_EXTEND_VLAN;
2425         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
2426
2427         /* Update maximum packet length */
2428         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
2429                 E1000_WRITE_REG(hw, E1000_RLPML,
2430                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
2431                                                 VLAN_TAG_SIZE);
2432 }
2433
2434 static void
2435 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev)
2436 {
2437         struct e1000_hw *hw =
2438                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2439         uint32_t reg;
2440
2441         /* CTRL_EXT: Extended VLAN */
2442         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
2443         reg |= E1000_CTRL_EXT_EXTEND_VLAN;
2444         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
2445
2446         /* Update maximum packet length */
2447         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
2448                 E1000_WRITE_REG(hw, E1000_RLPML,
2449                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
2450                                                 2 * VLAN_TAG_SIZE);
2451 }
2452
2453 static void
2454 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2455 {
2456         if(mask & ETH_VLAN_STRIP_MASK){
2457                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
2458                         igb_vlan_hw_strip_enable(dev);
2459                 else
2460                         igb_vlan_hw_strip_disable(dev);
2461         }
2462
2463         if(mask & ETH_VLAN_FILTER_MASK){
2464                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
2465                         igb_vlan_hw_filter_enable(dev);
2466                 else
2467                         igb_vlan_hw_filter_disable(dev);
2468         }
2469
2470         if(mask & ETH_VLAN_EXTEND_MASK){
2471                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
2472                         igb_vlan_hw_extend_enable(dev);
2473                 else
2474                         igb_vlan_hw_extend_disable(dev);
2475         }
2476 }
2477
2478
2479 /**
2480  * It enables the interrupt mask and then enable the interrupt.
2481  *
2482  * @param dev
2483  *  Pointer to struct rte_eth_dev.
2484  *
2485  * @return
2486  *  - On success, zero.
2487  *  - On failure, a negative value.
2488  */
2489 static int
2490 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev)
2491 {
2492         struct e1000_interrupt *intr =
2493                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2494
2495         intr->mask |= E1000_ICR_LSC;
2496
2497         return 0;
2498 }
2499
2500 /* It clears the interrupt causes and enables the interrupt.
2501  * It will be called once only during nic initialized.
2502  *
2503  * @param dev
2504  *  Pointer to struct rte_eth_dev.
2505  *
2506  * @return
2507  *  - On success, zero.
2508  *  - On failure, a negative value.
2509  */
2510 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev)
2511 {
2512         uint32_t mask, regval;
2513         struct e1000_hw *hw =
2514                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2515         struct rte_eth_dev_info dev_info;
2516
2517         memset(&dev_info, 0, sizeof(dev_info));
2518         eth_igb_infos_get(dev, &dev_info);
2519
2520         mask = 0xFFFFFFFF >> (32 - dev_info.max_rx_queues);
2521         regval = E1000_READ_REG(hw, E1000_EIMS);
2522         E1000_WRITE_REG(hw, E1000_EIMS, regval | mask);
2523
2524         return 0;
2525 }
2526
2527 /*
2528  * It reads ICR and gets interrupt causes, check it and set a bit flag
2529  * to update link status.
2530  *
2531  * @param dev
2532  *  Pointer to struct rte_eth_dev.
2533  *
2534  * @return
2535  *  - On success, zero.
2536  *  - On failure, a negative value.
2537  */
2538 static int
2539 eth_igb_interrupt_get_status(struct rte_eth_dev *dev)
2540 {
2541         uint32_t icr;
2542         struct e1000_hw *hw =
2543                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2544         struct e1000_interrupt *intr =
2545                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2546
2547         igb_intr_disable(hw);
2548
2549         /* read-on-clear nic registers here */
2550         icr = E1000_READ_REG(hw, E1000_ICR);
2551
2552         intr->flags = 0;
2553         if (icr & E1000_ICR_LSC) {
2554                 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
2555         }
2556
2557         if (icr & E1000_ICR_VMMB)
2558                 intr->flags |= E1000_FLAG_MAILBOX;
2559
2560         return 0;
2561 }
2562
2563 /*
2564  * It executes link_update after knowing an interrupt is prsent.
2565  *
2566  * @param dev
2567  *  Pointer to struct rte_eth_dev.
2568  *
2569  * @return
2570  *  - On success, zero.
2571  *  - On failure, a negative value.
2572  */
2573 static int
2574 eth_igb_interrupt_action(struct rte_eth_dev *dev)
2575 {
2576         struct e1000_hw *hw =
2577                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2578         struct e1000_interrupt *intr =
2579                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2580         uint32_t tctl, rctl;
2581         struct rte_eth_link link;
2582         int ret;
2583
2584         if (intr->flags & E1000_FLAG_MAILBOX) {
2585                 igb_pf_mbx_process(dev);
2586                 intr->flags &= ~E1000_FLAG_MAILBOX;
2587         }
2588
2589         igb_intr_enable(dev);
2590         rte_intr_enable(&(dev->pci_dev->intr_handle));
2591
2592         if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) {
2593                 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE;
2594
2595                 /* set get_link_status to check register later */
2596                 hw->mac.get_link_status = 1;
2597                 ret = eth_igb_link_update(dev, 0);
2598
2599                 /* check if link has changed */
2600                 if (ret < 0)
2601                         return 0;
2602
2603                 memset(&link, 0, sizeof(link));
2604                 rte_igb_dev_atomic_read_link_status(dev, &link);
2605                 if (link.link_status) {
2606                         PMD_INIT_LOG(INFO,
2607                                      " Port %d: Link Up - speed %u Mbps - %s",
2608                                      dev->data->port_id,
2609                                      (unsigned)link.link_speed,
2610                                      link.link_duplex == ETH_LINK_FULL_DUPLEX ?
2611                                      "full-duplex" : "half-duplex");
2612                 } else {
2613                         PMD_INIT_LOG(INFO, " Port %d: Link Down",
2614                                      dev->data->port_id);
2615                 }
2616
2617                 PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d",
2618                              dev->pci_dev->addr.domain,
2619                              dev->pci_dev->addr.bus,
2620                              dev->pci_dev->addr.devid,
2621                              dev->pci_dev->addr.function);
2622                 tctl = E1000_READ_REG(hw, E1000_TCTL);
2623                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2624                 if (link.link_status) {
2625                         /* enable Tx/Rx */
2626                         tctl |= E1000_TCTL_EN;
2627                         rctl |= E1000_RCTL_EN;
2628                 } else {
2629                         /* disable Tx/Rx */
2630                         tctl &= ~E1000_TCTL_EN;
2631                         rctl &= ~E1000_RCTL_EN;
2632                 }
2633                 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
2634                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2635                 E1000_WRITE_FLUSH(hw);
2636                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
2637         }
2638
2639         return 0;
2640 }
2641
2642 /**
2643  * Interrupt handler which shall be registered at first.
2644  *
2645  * @param handle
2646  *  Pointer to interrupt handle.
2647  * @param param
2648  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2649  *
2650  * @return
2651  *  void
2652  */
2653 static void
2654 eth_igb_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
2655                                                         void *param)
2656 {
2657         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2658
2659         eth_igb_interrupt_get_status(dev);
2660         eth_igb_interrupt_action(dev);
2661 }
2662
2663 static int
2664 eth_igbvf_interrupt_get_status(struct rte_eth_dev *dev)
2665 {
2666         uint32_t eicr;
2667         struct e1000_hw *hw =
2668                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2669         struct e1000_interrupt *intr =
2670                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2671
2672         igbvf_intr_disable(hw);
2673
2674         /* read-on-clear nic registers here */
2675         eicr = E1000_READ_REG(hw, E1000_EICR);
2676         intr->flags = 0;
2677
2678         if (eicr == E1000_VTIVAR_MISC_MAILBOX)
2679                 intr->flags |= E1000_FLAG_MAILBOX;
2680
2681         return 0;
2682 }
2683
2684 void igbvf_mbx_process(struct rte_eth_dev *dev)
2685 {
2686         struct e1000_hw *hw =
2687                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2688         struct e1000_mbx_info *mbx = &hw->mbx;
2689         u32 in_msg = 0;
2690
2691         if (mbx->ops.read(hw, &in_msg, 1, 0))
2692                 return;
2693
2694         /* PF reset VF event */
2695         if (in_msg == E1000_PF_CONTROL_MSG)
2696                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET);
2697 }
2698
2699 static int
2700 eth_igbvf_interrupt_action(struct rte_eth_dev *dev)
2701 {
2702         struct e1000_interrupt *intr =
2703                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2704
2705         if (intr->flags & E1000_FLAG_MAILBOX) {
2706                 igbvf_mbx_process(dev);
2707                 intr->flags &= ~E1000_FLAG_MAILBOX;
2708         }
2709
2710         igbvf_intr_enable(dev);
2711         rte_intr_enable(&dev->pci_dev->intr_handle);
2712
2713         return 0;
2714 }
2715
2716 static void
2717 eth_igbvf_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
2718                             void *param)
2719 {
2720         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2721
2722         eth_igbvf_interrupt_get_status(dev);
2723         eth_igbvf_interrupt_action(dev);
2724 }
2725
2726 static int
2727 eth_igb_led_on(struct rte_eth_dev *dev)
2728 {
2729         struct e1000_hw *hw;
2730
2731         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2732         return e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP;
2733 }
2734
2735 static int
2736 eth_igb_led_off(struct rte_eth_dev *dev)
2737 {
2738         struct e1000_hw *hw;
2739
2740         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2741         return e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP;
2742 }
2743
2744 static int
2745 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2746 {
2747         struct e1000_hw *hw;
2748         uint32_t ctrl;
2749         int tx_pause;
2750         int rx_pause;
2751
2752         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2753         fc_conf->pause_time = hw->fc.pause_time;
2754         fc_conf->high_water = hw->fc.high_water;
2755         fc_conf->low_water = hw->fc.low_water;
2756         fc_conf->send_xon = hw->fc.send_xon;
2757         fc_conf->autoneg = hw->mac.autoneg;
2758
2759         /*
2760          * Return rx_pause and tx_pause status according to actual setting of
2761          * the TFCE and RFCE bits in the CTRL register.
2762          */
2763         ctrl = E1000_READ_REG(hw, E1000_CTRL);
2764         if (ctrl & E1000_CTRL_TFCE)
2765                 tx_pause = 1;
2766         else
2767                 tx_pause = 0;
2768
2769         if (ctrl & E1000_CTRL_RFCE)
2770                 rx_pause = 1;
2771         else
2772                 rx_pause = 0;
2773
2774         if (rx_pause && tx_pause)
2775                 fc_conf->mode = RTE_FC_FULL;
2776         else if (rx_pause)
2777                 fc_conf->mode = RTE_FC_RX_PAUSE;
2778         else if (tx_pause)
2779                 fc_conf->mode = RTE_FC_TX_PAUSE;
2780         else
2781                 fc_conf->mode = RTE_FC_NONE;
2782
2783         return 0;
2784 }
2785
2786 static int
2787 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2788 {
2789         struct e1000_hw *hw;
2790         int err;
2791         enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = {
2792                 e1000_fc_none,
2793                 e1000_fc_rx_pause,
2794                 e1000_fc_tx_pause,
2795                 e1000_fc_full
2796         };
2797         uint32_t rx_buf_size;
2798         uint32_t max_high_water;
2799         uint32_t rctl;
2800
2801         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2802         if (fc_conf->autoneg != hw->mac.autoneg)
2803                 return -ENOTSUP;
2804         rx_buf_size = igb_get_rx_buffer_size(hw);
2805         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2806
2807         /* At least reserve one Ethernet frame for watermark */
2808         max_high_water = rx_buf_size - ETHER_MAX_LEN;
2809         if ((fc_conf->high_water > max_high_water) ||
2810             (fc_conf->high_water < fc_conf->low_water)) {
2811                 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value");
2812                 PMD_INIT_LOG(ERR, "high water must <=  0x%x", max_high_water);
2813                 return -EINVAL;
2814         }
2815
2816         hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode];
2817         hw->fc.pause_time     = fc_conf->pause_time;
2818         hw->fc.high_water     = fc_conf->high_water;
2819         hw->fc.low_water      = fc_conf->low_water;
2820         hw->fc.send_xon       = fc_conf->send_xon;
2821
2822         err = e1000_setup_link_generic(hw);
2823         if (err == E1000_SUCCESS) {
2824
2825                 /* check if we want to forward MAC frames - driver doesn't have native
2826                  * capability to do that, so we'll write the registers ourselves */
2827
2828                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2829
2830                 /* set or clear MFLCN.PMCF bit depending on configuration */
2831                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2832                         rctl |= E1000_RCTL_PMCF;
2833                 else
2834                         rctl &= ~E1000_RCTL_PMCF;
2835
2836                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2837                 E1000_WRITE_FLUSH(hw);
2838
2839                 return 0;
2840         }
2841
2842         PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err);
2843         return -EIO;
2844 }
2845
2846 #define E1000_RAH_POOLSEL_SHIFT      (18)
2847 static void
2848 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2849                 uint32_t index, __rte_unused uint32_t pool)
2850 {
2851         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2852         uint32_t rah;
2853
2854         e1000_rar_set(hw, mac_addr->addr_bytes, index);
2855         rah = E1000_READ_REG(hw, E1000_RAH(index));
2856         rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool));
2857         E1000_WRITE_REG(hw, E1000_RAH(index), rah);
2858 }
2859
2860 static void
2861 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index)
2862 {
2863         uint8_t addr[ETHER_ADDR_LEN];
2864         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2865
2866         memset(addr, 0, sizeof(addr));
2867
2868         e1000_rar_set(hw, addr, index);
2869 }
2870
2871 static void
2872 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev,
2873                                 struct ether_addr *addr)
2874 {
2875         eth_igb_rar_clear(dev, 0);
2876
2877         eth_igb_rar_set(dev, (void *)addr, 0, 0);
2878 }
2879 /*
2880  * Virtual Function operations
2881  */
2882 static void
2883 igbvf_intr_disable(struct e1000_hw *hw)
2884 {
2885         PMD_INIT_FUNC_TRACE();
2886
2887         /* Clear interrupt mask to stop from interrupts being generated */
2888         E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF);
2889
2890         E1000_WRITE_FLUSH(hw);
2891 }
2892
2893 static void
2894 igbvf_stop_adapter(struct rte_eth_dev *dev)
2895 {
2896         u32 reg_val;
2897         u16 i;
2898         struct rte_eth_dev_info dev_info;
2899         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2900
2901         memset(&dev_info, 0, sizeof(dev_info));
2902         eth_igbvf_infos_get(dev, &dev_info);
2903
2904         /* Clear interrupt mask to stop from interrupts being generated */
2905         igbvf_intr_disable(hw);
2906
2907         /* Clear any pending interrupts, flush previous writes */
2908         E1000_READ_REG(hw, E1000_EICR);
2909
2910         /* Disable the transmit unit.  Each queue must be disabled. */
2911         for (i = 0; i < dev_info.max_tx_queues; i++)
2912                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH);
2913
2914         /* Disable the receive unit by stopping each queue */
2915         for (i = 0; i < dev_info.max_rx_queues; i++) {
2916                 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i));
2917                 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE;
2918                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val);
2919                 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE)
2920                         ;
2921         }
2922
2923         /* flush all queues disables */
2924         E1000_WRITE_FLUSH(hw);
2925         msec_delay(2);
2926 }
2927
2928 static int eth_igbvf_link_update(struct e1000_hw *hw)
2929 {
2930         struct e1000_mbx_info *mbx = &hw->mbx;
2931         struct e1000_mac_info *mac = &hw->mac;
2932         int ret_val = E1000_SUCCESS;
2933
2934         PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf");
2935
2936         /*
2937          * We only want to run this if there has been a rst asserted.
2938          * in this case that could mean a link change, device reset,
2939          * or a virtual function reset
2940          */
2941
2942         /* If we were hit with a reset or timeout drop the link */
2943         if (!e1000_check_for_rst(hw, 0) || !mbx->timeout)
2944                 mac->get_link_status = TRUE;
2945
2946         if (!mac->get_link_status)
2947                 goto out;
2948
2949         /* if link status is down no point in checking to see if pf is up */
2950         if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU))
2951                 goto out;
2952
2953         /* if we passed all the tests above then the link is up and we no
2954          * longer need to check for link */
2955         mac->get_link_status = FALSE;
2956
2957 out:
2958         return ret_val;
2959 }
2960
2961
2962 static int
2963 igbvf_dev_configure(struct rte_eth_dev *dev)
2964 {
2965         struct rte_eth_conf* conf = &dev->data->dev_conf;
2966
2967         PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
2968                      dev->data->port_id);
2969
2970         /*
2971          * VF has no ability to enable/disable HW CRC
2972          * Keep the persistent behavior the same as Host PF
2973          */
2974 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC
2975         if (!conf->rxmode.hw_strip_crc) {
2976                 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip");
2977                 conf->rxmode.hw_strip_crc = 1;
2978         }
2979 #else
2980         if (conf->rxmode.hw_strip_crc) {
2981                 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip");
2982                 conf->rxmode.hw_strip_crc = 0;
2983         }
2984 #endif
2985
2986         return 0;
2987 }
2988
2989 static int
2990 igbvf_dev_start(struct rte_eth_dev *dev)
2991 {
2992         struct e1000_hw *hw =
2993                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2994         struct e1000_adapter *adapter =
2995                 E1000_DEV_PRIVATE(dev->data->dev_private);
2996         int ret;
2997         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2998         uint32_t intr_vector = 0;
2999
3000         PMD_INIT_FUNC_TRACE();
3001
3002         hw->mac.ops.reset_hw(hw);
3003         adapter->stopped = 0;
3004
3005         /* Set all vfta */
3006         igbvf_set_vfta_all(dev,1);
3007
3008         eth_igbvf_tx_init(dev);
3009
3010         /* This can fail when allocating mbufs for descriptor rings */
3011         ret = eth_igbvf_rx_init(dev);
3012         if (ret) {
3013                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
3014                 igb_dev_clear_queues(dev);
3015                 return ret;
3016         }
3017
3018         /* check and configure queue intr-vector mapping */
3019         if (dev->data->dev_conf.intr_conf.rxq != 0) {
3020                 intr_vector = dev->data->nb_rx_queues;
3021                 ret = rte_intr_efd_enable(intr_handle, intr_vector);
3022                 if (ret)
3023                         return ret;
3024         }
3025
3026         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
3027                 intr_handle->intr_vec =
3028                         rte_zmalloc("intr_vec",
3029                                     dev->data->nb_rx_queues * sizeof(int), 0);
3030                 if (!intr_handle->intr_vec) {
3031                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
3032                                      " intr_vec\n", dev->data->nb_rx_queues);
3033                         return -ENOMEM;
3034                 }
3035         }
3036
3037         eth_igbvf_configure_msix_intr(dev);
3038
3039         /* enable uio/vfio intr/eventfd mapping */
3040         rte_intr_enable(intr_handle);
3041
3042         /* resume enabled intr since hw reset */
3043         igbvf_intr_enable(dev);
3044
3045         return 0;
3046 }
3047
3048 static void
3049 igbvf_dev_stop(struct rte_eth_dev *dev)
3050 {
3051         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
3052
3053         PMD_INIT_FUNC_TRACE();
3054
3055         igbvf_stop_adapter(dev);
3056
3057         /*
3058           * Clear what we set, but we still keep shadow_vfta to
3059           * restore after device starts
3060           */
3061         igbvf_set_vfta_all(dev,0);
3062
3063         igb_dev_clear_queues(dev);
3064
3065         /* disable intr eventfd mapping */
3066         rte_intr_disable(intr_handle);
3067
3068         /* Clean datapath event and queue/vec mapping */
3069         rte_intr_efd_disable(intr_handle);
3070         if (intr_handle->intr_vec) {
3071                 rte_free(intr_handle->intr_vec);
3072                 intr_handle->intr_vec = NULL;
3073         }
3074 }
3075
3076 static void
3077 igbvf_dev_close(struct rte_eth_dev *dev)
3078 {
3079         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3080         struct e1000_adapter *adapter =
3081                 E1000_DEV_PRIVATE(dev->data->dev_private);
3082         struct ether_addr addr;
3083
3084         PMD_INIT_FUNC_TRACE();
3085
3086         e1000_reset_hw(hw);
3087
3088         igbvf_dev_stop(dev);
3089         adapter->stopped = 1;
3090         igb_dev_free_queues(dev);
3091
3092         /**
3093          * reprogram the RAR with a zero mac address,
3094          * to ensure that the VF traffic goes to the PF
3095          * after stop, close and detach of the VF.
3096          **/
3097
3098         memset(&addr, 0, sizeof(addr));
3099         igbvf_default_mac_addr_set(dev, &addr);
3100 }
3101
3102 static void
3103 igbvf_promiscuous_enable(struct rte_eth_dev *dev)
3104 {
3105         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3106
3107         /* Set both unicast and multicast promisc */
3108         e1000_promisc_set_vf(hw, e1000_promisc_enabled);
3109 }
3110
3111 static void
3112 igbvf_promiscuous_disable(struct rte_eth_dev *dev)
3113 {
3114         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3115
3116         /* If in allmulticast mode leave multicast promisc */
3117         if (dev->data->all_multicast == 1)
3118                 e1000_promisc_set_vf(hw, e1000_promisc_multicast);
3119         else
3120                 e1000_promisc_set_vf(hw, e1000_promisc_disabled);
3121 }
3122
3123 static void
3124 igbvf_allmulticast_enable(struct rte_eth_dev *dev)
3125 {
3126         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3127
3128         /* In promiscuous mode multicast promisc already set */
3129         if (dev->data->promiscuous == 0)
3130                 e1000_promisc_set_vf(hw, e1000_promisc_multicast);
3131 }
3132
3133 static void
3134 igbvf_allmulticast_disable(struct rte_eth_dev *dev)
3135 {
3136         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3137
3138         /* In promiscuous mode leave multicast promisc enabled */
3139         if (dev->data->promiscuous == 0)
3140                 e1000_promisc_set_vf(hw, e1000_promisc_disabled);
3141 }
3142
3143 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on)
3144 {
3145         struct e1000_mbx_info *mbx = &hw->mbx;
3146         uint32_t msgbuf[2];
3147         s32 err;
3148
3149         /* After set vlan, vlan strip will also be enabled in igb driver*/
3150         msgbuf[0] = E1000_VF_SET_VLAN;
3151         msgbuf[1] = vid;
3152         /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */
3153         if (on)
3154                 msgbuf[0] |= E1000_VF_SET_VLAN_ADD;
3155
3156         err = mbx->ops.write_posted(hw, msgbuf, 2, 0);
3157         if (err)
3158                 goto mbx_err;
3159
3160         err = mbx->ops.read_posted(hw, msgbuf, 2, 0);
3161         if (err)
3162                 goto mbx_err;
3163
3164         msgbuf[0] &= ~E1000_VT_MSGTYPE_CTS;
3165         if (msgbuf[0] == (E1000_VF_SET_VLAN | E1000_VT_MSGTYPE_NACK))
3166                 err = -EINVAL;
3167
3168 mbx_err:
3169         return err;
3170 }
3171
3172 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on)
3173 {
3174         struct e1000_hw *hw =
3175                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3176         struct e1000_vfta * shadow_vfta =
3177                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
3178         int i = 0, j = 0, vfta = 0, mask = 1;
3179
3180         for (i = 0; i < IGB_VFTA_SIZE; i++){
3181                 vfta = shadow_vfta->vfta[i];
3182                 if(vfta){
3183                         mask = 1;
3184                         for (j = 0; j < 32; j++){
3185                                 if(vfta & mask)
3186                                         igbvf_set_vfta(hw,
3187                                                 (uint16_t)((i<<5)+j), on);
3188                                 mask<<=1;
3189                         }
3190                 }
3191         }
3192
3193 }
3194
3195 static int
3196 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
3197 {
3198         struct e1000_hw *hw =
3199                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3200         struct e1000_vfta * shadow_vfta =
3201                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
3202         uint32_t vid_idx = 0;
3203         uint32_t vid_bit = 0;
3204         int ret = 0;
3205
3206         PMD_INIT_FUNC_TRACE();
3207
3208         /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/
3209         ret = igbvf_set_vfta(hw, vlan_id, !!on);
3210         if(ret){
3211                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
3212                 return ret;
3213         }
3214         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
3215         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
3216
3217         /*Save what we set and retore it after device reset*/
3218         if (on)
3219                 shadow_vfta->vfta[vid_idx] |= vid_bit;
3220         else
3221                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
3222
3223         return 0;
3224 }
3225
3226 static void
3227 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr)
3228 {
3229         struct e1000_hw *hw =
3230                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3231
3232         /* index is not used by rar_set() */
3233         hw->mac.ops.rar_set(hw, (void *)addr, 0);
3234 }
3235
3236
3237 static int
3238 eth_igb_rss_reta_update(struct rte_eth_dev *dev,
3239                         struct rte_eth_rss_reta_entry64 *reta_conf,
3240                         uint16_t reta_size)
3241 {
3242         uint8_t i, j, mask;
3243         uint32_t reta, r;
3244         uint16_t idx, shift;
3245         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3246
3247         if (reta_size != ETH_RSS_RETA_SIZE_128) {
3248                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3249                         "(%d) doesn't match the number hardware can supported "
3250                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
3251                 return -EINVAL;
3252         }
3253
3254         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
3255                 idx = i / RTE_RETA_GROUP_SIZE;
3256                 shift = i % RTE_RETA_GROUP_SIZE;
3257                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
3258                                                 IGB_4_BIT_MASK);
3259                 if (!mask)
3260                         continue;
3261                 if (mask == IGB_4_BIT_MASK)
3262                         r = 0;
3263                 else
3264                         r = E1000_READ_REG(hw, E1000_RETA(i >> 2));
3265                 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) {
3266                         if (mask & (0x1 << j))
3267                                 reta |= reta_conf[idx].reta[shift + j] <<
3268                                                         (CHAR_BIT * j);
3269                         else
3270                                 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j));
3271                 }
3272                 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta);
3273         }
3274
3275         return 0;
3276 }
3277
3278 static int
3279 eth_igb_rss_reta_query(struct rte_eth_dev *dev,
3280                        struct rte_eth_rss_reta_entry64 *reta_conf,
3281                        uint16_t reta_size)
3282 {
3283         uint8_t i, j, mask;
3284         uint32_t reta;
3285         uint16_t idx, shift;
3286         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3287
3288         if (reta_size != ETH_RSS_RETA_SIZE_128) {
3289                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3290                         "(%d) doesn't match the number hardware can supported "
3291                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
3292                 return -EINVAL;
3293         }
3294
3295         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
3296                 idx = i / RTE_RETA_GROUP_SIZE;
3297                 shift = i % RTE_RETA_GROUP_SIZE;
3298                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
3299                                                 IGB_4_BIT_MASK);
3300                 if (!mask)
3301                         continue;
3302                 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2));
3303                 for (j = 0; j < IGB_4_BIT_WIDTH; j++) {
3304                         if (mask & (0x1 << j))
3305                                 reta_conf[idx].reta[shift + j] =
3306                                         ((reta >> (CHAR_BIT * j)) &
3307                                                 IGB_8_BIT_MASK);
3308                 }
3309         }
3310
3311         return 0;
3312 }
3313
3314 #define MAC_TYPE_FILTER_SUP(type)    do {\
3315         if ((type) != e1000_82580 && (type) != e1000_i350 &&\
3316                 (type) != e1000_82576)\
3317                 return -ENOTSUP;\
3318 } while (0)
3319
3320 static int
3321 eth_igb_syn_filter_set(struct rte_eth_dev *dev,
3322                         struct rte_eth_syn_filter *filter,
3323                         bool add)
3324 {
3325         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3326         uint32_t synqf, rfctl;
3327
3328         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
3329                 return -EINVAL;
3330
3331         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
3332
3333         if (add) {
3334                 if (synqf & E1000_SYN_FILTER_ENABLE)
3335                         return -EINVAL;
3336
3337                 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) &
3338                         E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE);
3339
3340                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
3341                 if (filter->hig_pri)
3342                         rfctl |= E1000_RFCTL_SYNQFP;
3343                 else
3344                         rfctl &= ~E1000_RFCTL_SYNQFP;
3345
3346                 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl);
3347         } else {
3348                 if (!(synqf & E1000_SYN_FILTER_ENABLE))
3349                         return -ENOENT;
3350                 synqf = 0;
3351         }
3352
3353         E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf);
3354         E1000_WRITE_FLUSH(hw);
3355         return 0;
3356 }
3357
3358 static int
3359 eth_igb_syn_filter_get(struct rte_eth_dev *dev,
3360                         struct rte_eth_syn_filter *filter)
3361 {
3362         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3363         uint32_t synqf, rfctl;
3364
3365         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
3366         if (synqf & E1000_SYN_FILTER_ENABLE) {
3367                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
3368                 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0;
3369                 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >>
3370                                 E1000_SYN_FILTER_QUEUE_SHIFT);
3371                 return 0;
3372         }
3373
3374         return -ENOENT;
3375 }
3376
3377 static int
3378 eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
3379                         enum rte_filter_op filter_op,
3380                         void *arg)
3381 {
3382         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3383         int ret;
3384
3385         MAC_TYPE_FILTER_SUP(hw->mac.type);
3386
3387         if (filter_op == RTE_ETH_FILTER_NOP)
3388                 return 0;
3389
3390         if (arg == NULL) {
3391                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
3392                             filter_op);
3393                 return -EINVAL;
3394         }
3395
3396         switch (filter_op) {
3397         case RTE_ETH_FILTER_ADD:
3398                 ret = eth_igb_syn_filter_set(dev,
3399                                 (struct rte_eth_syn_filter *)arg,
3400                                 TRUE);
3401                 break;
3402         case RTE_ETH_FILTER_DELETE:
3403                 ret = eth_igb_syn_filter_set(dev,
3404                                 (struct rte_eth_syn_filter *)arg,
3405                                 FALSE);
3406                 break;
3407         case RTE_ETH_FILTER_GET:
3408                 ret = eth_igb_syn_filter_get(dev,
3409                                 (struct rte_eth_syn_filter *)arg);
3410                 break;
3411         default:
3412                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
3413                 ret = -EINVAL;
3414                 break;
3415         }
3416
3417         return ret;
3418 }
3419
3420 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
3421         if ((type) != e1000_82580 && (type) != e1000_i350)\
3422                 return -ENOSYS; \
3423 } while (0)
3424
3425 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/
3426 static inline int
3427 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter,
3428                         struct e1000_2tuple_filter_info *filter_info)
3429 {
3430         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
3431                 return -EINVAL;
3432         if (filter->priority > E1000_2TUPLE_MAX_PRI)
3433                 return -EINVAL;  /* filter index is out of range. */
3434         if (filter->tcp_flags > TCP_FLAG_ALL)
3435                 return -EINVAL;  /* flags is invalid. */
3436
3437         switch (filter->dst_port_mask) {
3438         case UINT16_MAX:
3439                 filter_info->dst_port_mask = 0;
3440                 filter_info->dst_port = filter->dst_port;
3441                 break;
3442         case 0:
3443                 filter_info->dst_port_mask = 1;
3444                 break;
3445         default:
3446                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
3447                 return -EINVAL;
3448         }
3449
3450         switch (filter->proto_mask) {
3451         case UINT8_MAX:
3452                 filter_info->proto_mask = 0;
3453                 filter_info->proto = filter->proto;
3454                 break;
3455         case 0:
3456                 filter_info->proto_mask = 1;
3457                 break;
3458         default:
3459                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
3460                 return -EINVAL;
3461         }
3462
3463         filter_info->priority = (uint8_t)filter->priority;
3464         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
3465                 filter_info->tcp_flags = filter->tcp_flags;
3466         else
3467                 filter_info->tcp_flags = 0;
3468
3469         return 0;
3470 }
3471
3472 static inline struct e1000_2tuple_filter *
3473 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list,
3474                         struct e1000_2tuple_filter_info *key)
3475 {
3476         struct e1000_2tuple_filter *it;
3477
3478         TAILQ_FOREACH(it, filter_list, entries) {
3479                 if (memcmp(key, &it->filter_info,
3480                         sizeof(struct e1000_2tuple_filter_info)) == 0) {
3481                         return it;
3482                 }
3483         }
3484         return NULL;
3485 }
3486
3487 /*
3488  * igb_add_2tuple_filter - add a 2tuple filter
3489  *
3490  * @param
3491  * dev: Pointer to struct rte_eth_dev.
3492  * ntuple_filter: ponter to the filter that will be added.
3493  *
3494  * @return
3495  *    - On success, zero.
3496  *    - On failure, a negative value.
3497  */
3498 static int
3499 igb_add_2tuple_filter(struct rte_eth_dev *dev,
3500                         struct rte_eth_ntuple_filter *ntuple_filter)
3501 {
3502         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3503         struct e1000_filter_info *filter_info =
3504                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3505         struct e1000_2tuple_filter *filter;
3506         uint32_t ttqf = E1000_TTQF_DISABLE_MASK;
3507         uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP;
3508         int i, ret;
3509
3510         filter = rte_zmalloc("e1000_2tuple_filter",
3511                         sizeof(struct e1000_2tuple_filter), 0);
3512         if (filter == NULL)
3513                 return -ENOMEM;
3514
3515         ret = ntuple_filter_to_2tuple(ntuple_filter,
3516                                       &filter->filter_info);
3517         if (ret < 0) {
3518                 rte_free(filter);
3519                 return ret;
3520         }
3521         if (igb_2tuple_filter_lookup(&filter_info->twotuple_list,
3522                                          &filter->filter_info) != NULL) {
3523                 PMD_DRV_LOG(ERR, "filter exists.");
3524                 rte_free(filter);
3525                 return -EEXIST;
3526         }
3527         filter->queue = ntuple_filter->queue;
3528
3529         /*
3530          * look for an unused 2tuple filter index,
3531          * and insert the filter to list.
3532          */
3533         for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) {
3534                 if (!(filter_info->twotuple_mask & (1 << i))) {
3535                         filter_info->twotuple_mask |= 1 << i;
3536                         filter->index = i;
3537                         TAILQ_INSERT_TAIL(&filter_info->twotuple_list,
3538                                           filter,
3539                                           entries);
3540                         break;
3541                 }
3542         }
3543         if (i >= E1000_MAX_TTQF_FILTERS) {
3544                 PMD_DRV_LOG(ERR, "2tuple filters are full.");
3545                 rte_free(filter);
3546                 return -ENOSYS;
3547         }
3548
3549         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
3550         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
3551                 imir |= E1000_IMIR_PORT_BP;
3552         else
3553                 imir &= ~E1000_IMIR_PORT_BP;
3554
3555         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
3556
3557         ttqf |= E1000_TTQF_QUEUE_ENABLE;
3558         ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT);
3559         ttqf |= (uint32_t)(filter->filter_info.proto & E1000_TTQF_PROTOCOL_MASK);
3560         if (filter->filter_info.proto_mask == 0)
3561                 ttqf &= ~E1000_TTQF_MASK_ENABLE;
3562
3563         /* tcp flags bits setting. */
3564         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
3565                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
3566                         imir_ext |= E1000_IMIREXT_CTRL_URG;
3567                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
3568                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
3569                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
3570                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
3571                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
3572                         imir_ext |= E1000_IMIREXT_CTRL_RST;
3573                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
3574                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
3575                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
3576                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
3577         } else
3578                 imir_ext |= E1000_IMIREXT_CTRL_BP;
3579         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
3580         E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf);
3581         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
3582         return 0;
3583 }
3584
3585 /*
3586  * igb_remove_2tuple_filter - remove a 2tuple filter
3587  *
3588  * @param
3589  * dev: Pointer to struct rte_eth_dev.
3590  * ntuple_filter: ponter to the filter that will be removed.
3591  *
3592  * @return
3593  *    - On success, zero.
3594  *    - On failure, a negative value.
3595  */
3596 static int
3597 igb_remove_2tuple_filter(struct rte_eth_dev *dev,
3598                         struct rte_eth_ntuple_filter *ntuple_filter)
3599 {
3600         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3601         struct e1000_filter_info *filter_info =
3602                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3603         struct e1000_2tuple_filter_info filter_2tuple;
3604         struct e1000_2tuple_filter *filter;
3605         int ret;
3606
3607         memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info));
3608         ret = ntuple_filter_to_2tuple(ntuple_filter,
3609                                       &filter_2tuple);
3610         if (ret < 0)
3611                 return ret;
3612
3613         filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list,
3614                                          &filter_2tuple);
3615         if (filter == NULL) {
3616                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3617                 return -ENOENT;
3618         }
3619
3620         filter_info->twotuple_mask &= ~(1 << filter->index);
3621         TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries);
3622         rte_free(filter);
3623
3624         E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK);
3625         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
3626         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
3627         return 0;
3628 }
3629
3630 static inline struct e1000_flex_filter *
3631 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list,
3632                         struct e1000_flex_filter_info *key)
3633 {
3634         struct e1000_flex_filter *it;
3635
3636         TAILQ_FOREACH(it, filter_list, entries) {
3637                 if (memcmp(key, &it->filter_info,
3638                         sizeof(struct e1000_flex_filter_info)) == 0)
3639                         return it;
3640         }
3641
3642         return NULL;
3643 }
3644
3645 static int
3646 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
3647                         struct rte_eth_flex_filter *filter,
3648                         bool add)
3649 {
3650         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3651         struct e1000_filter_info *filter_info =
3652                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3653         struct e1000_flex_filter *flex_filter, *it;
3654         uint32_t wufc, queueing, mask;
3655         uint32_t reg_off;
3656         uint8_t shift, i, j = 0;
3657
3658         flex_filter = rte_zmalloc("e1000_flex_filter",
3659                         sizeof(struct e1000_flex_filter), 0);
3660         if (flex_filter == NULL)
3661                 return -ENOMEM;
3662
3663         flex_filter->filter_info.len = filter->len;
3664         flex_filter->filter_info.priority = filter->priority;
3665         memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len);
3666         for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) {
3667                 mask = 0;
3668                 /* reverse bits in flex filter's mask*/
3669                 for (shift = 0; shift < CHAR_BIT; shift++) {
3670                         if (filter->mask[i] & (0x01 << shift))
3671                                 mask |= (0x80 >> shift);
3672                 }
3673                 flex_filter->filter_info.mask[i] = mask;
3674         }
3675
3676         wufc = E1000_READ_REG(hw, E1000_WUFC);
3677         if (flex_filter->index < E1000_MAX_FHFT)
3678                 reg_off = E1000_FHFT(flex_filter->index);
3679         else
3680                 reg_off = E1000_FHFT_EXT(flex_filter->index - E1000_MAX_FHFT);
3681
3682         if (add) {
3683                 if (eth_igb_flex_filter_lookup(&filter_info->flex_list,
3684                                 &flex_filter->filter_info) != NULL) {
3685                         PMD_DRV_LOG(ERR, "filter exists.");
3686                         rte_free(flex_filter);
3687                         return -EEXIST;
3688                 }
3689                 flex_filter->queue = filter->queue;
3690                 /*
3691                  * look for an unused flex filter index
3692                  * and insert the filter into the list.
3693                  */
3694                 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) {
3695                         if (!(filter_info->flex_mask & (1 << i))) {
3696                                 filter_info->flex_mask |= 1 << i;
3697                                 flex_filter->index = i;
3698                                 TAILQ_INSERT_TAIL(&filter_info->flex_list,
3699                                         flex_filter,
3700                                         entries);
3701                                 break;
3702                         }
3703                 }
3704                 if (i >= E1000_MAX_FLEX_FILTERS) {
3705                         PMD_DRV_LOG(ERR, "flex filters are full.");
3706                         rte_free(flex_filter);
3707                         return -ENOSYS;
3708                 }
3709
3710                 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ |
3711                                 (E1000_WUFC_FLX0 << flex_filter->index));
3712                 queueing = filter->len |
3713                         (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) |
3714                         (filter->priority << E1000_FHFT_QUEUEING_PRIO_SHIFT);
3715                 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET,
3716                                 queueing);
3717                 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) {
3718                         E1000_WRITE_REG(hw, reg_off,
3719                                         flex_filter->filter_info.dwords[j]);
3720                         reg_off += sizeof(uint32_t);
3721                         E1000_WRITE_REG(hw, reg_off,
3722                                         flex_filter->filter_info.dwords[++j]);
3723                         reg_off += sizeof(uint32_t);
3724                         E1000_WRITE_REG(hw, reg_off,
3725                                 (uint32_t)flex_filter->filter_info.mask[i]);
3726                         reg_off += sizeof(uint32_t) * 2;
3727                         ++j;
3728                 }
3729         } else {
3730                 it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
3731                                 &flex_filter->filter_info);
3732                 if (it == NULL) {
3733                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3734                         rte_free(flex_filter);
3735                         return -ENOENT;
3736                 }
3737
3738                 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++)
3739                         E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0);
3740                 E1000_WRITE_REG(hw, E1000_WUFC, wufc &
3741                         (~(E1000_WUFC_FLX0 << it->index)));
3742
3743                 filter_info->flex_mask &= ~(1 << it->index);
3744                 TAILQ_REMOVE(&filter_info->flex_list, it, entries);
3745                 rte_free(it);
3746                 rte_free(flex_filter);
3747         }
3748
3749         return 0;
3750 }
3751
3752 static int
3753 eth_igb_get_flex_filter(struct rte_eth_dev *dev,
3754                         struct rte_eth_flex_filter *filter)
3755 {
3756         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3757         struct e1000_filter_info *filter_info =
3758                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3759         struct e1000_flex_filter flex_filter, *it;
3760         uint32_t wufc, queueing, wufc_en = 0;
3761
3762         memset(&flex_filter, 0, sizeof(struct e1000_flex_filter));
3763         flex_filter.filter_info.len = filter->len;
3764         flex_filter.filter_info.priority = filter->priority;
3765         memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len);
3766         memcpy(flex_filter.filter_info.mask, filter->mask,
3767                         RTE_ALIGN(filter->len, sizeof(char)) / sizeof(char));
3768
3769         it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
3770                                 &flex_filter.filter_info);
3771         if (it == NULL) {
3772                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3773                 return -ENOENT;
3774         }
3775
3776         wufc = E1000_READ_REG(hw, E1000_WUFC);
3777         wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index);
3778
3779         if ((wufc & wufc_en) == wufc_en) {
3780                 uint32_t reg_off = 0;
3781                 if (it->index < E1000_MAX_FHFT)
3782                         reg_off = E1000_FHFT(it->index);
3783                 else
3784                         reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT);
3785
3786                 queueing = E1000_READ_REG(hw,
3787                                 reg_off + E1000_FHFT_QUEUEING_OFFSET);
3788                 filter->len = queueing & E1000_FHFT_QUEUEING_LEN;
3789                 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >>
3790                         E1000_FHFT_QUEUEING_PRIO_SHIFT;
3791                 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >>
3792                         E1000_FHFT_QUEUEING_QUEUE_SHIFT;
3793                 return 0;
3794         }
3795         return -ENOENT;
3796 }
3797
3798 static int
3799 eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
3800                         enum rte_filter_op filter_op,
3801                         void *arg)
3802 {
3803         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3804         struct rte_eth_flex_filter *filter;
3805         int ret = 0;
3806
3807         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
3808
3809         if (filter_op == RTE_ETH_FILTER_NOP)
3810                 return ret;
3811
3812         if (arg == NULL) {
3813                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
3814                             filter_op);
3815                 return -EINVAL;
3816         }
3817
3818         filter = (struct rte_eth_flex_filter *)arg;
3819         if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN
3820             || filter->len % sizeof(uint64_t) != 0) {
3821                 PMD_DRV_LOG(ERR, "filter's length is out of range");
3822                 return -EINVAL;
3823         }
3824         if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) {
3825                 PMD_DRV_LOG(ERR, "filter's priority is out of range");
3826                 return -EINVAL;
3827         }
3828
3829         switch (filter_op) {
3830         case RTE_ETH_FILTER_ADD:
3831                 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE);
3832                 break;
3833         case RTE_ETH_FILTER_DELETE:
3834                 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE);
3835                 break;
3836         case RTE_ETH_FILTER_GET:
3837                 ret = eth_igb_get_flex_filter(dev, filter);
3838                 break;
3839         default:
3840                 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op);
3841                 ret = -EINVAL;
3842                 break;
3843         }
3844
3845         return ret;
3846 }
3847
3848 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/
3849 static inline int
3850 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter,
3851                         struct e1000_5tuple_filter_info *filter_info)
3852 {
3853         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576)
3854                 return -EINVAL;
3855         if (filter->priority > E1000_2TUPLE_MAX_PRI)
3856                 return -EINVAL;  /* filter index is out of range. */
3857         if (filter->tcp_flags > TCP_FLAG_ALL)
3858                 return -EINVAL;  /* flags is invalid. */
3859
3860         switch (filter->dst_ip_mask) {
3861         case UINT32_MAX:
3862                 filter_info->dst_ip_mask = 0;
3863                 filter_info->dst_ip = filter->dst_ip;
3864                 break;
3865         case 0:
3866                 filter_info->dst_ip_mask = 1;
3867                 break;
3868         default:
3869                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
3870                 return -EINVAL;
3871         }
3872
3873         switch (filter->src_ip_mask) {
3874         case UINT32_MAX:
3875                 filter_info->src_ip_mask = 0;
3876                 filter_info->src_ip = filter->src_ip;
3877                 break;
3878         case 0:
3879                 filter_info->src_ip_mask = 1;
3880                 break;
3881         default:
3882                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
3883                 return -EINVAL;
3884         }
3885
3886         switch (filter->dst_port_mask) {
3887         case UINT16_MAX:
3888                 filter_info->dst_port_mask = 0;
3889                 filter_info->dst_port = filter->dst_port;
3890                 break;
3891         case 0:
3892                 filter_info->dst_port_mask = 1;
3893                 break;
3894         default:
3895                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
3896                 return -EINVAL;
3897         }
3898
3899         switch (filter->src_port_mask) {
3900         case UINT16_MAX:
3901                 filter_info->src_port_mask = 0;
3902                 filter_info->src_port = filter->src_port;
3903                 break;
3904         case 0:
3905                 filter_info->src_port_mask = 1;
3906                 break;
3907         default:
3908                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
3909                 return -EINVAL;
3910         }
3911
3912         switch (filter->proto_mask) {
3913         case UINT8_MAX:
3914                 filter_info->proto_mask = 0;
3915                 filter_info->proto = filter->proto;
3916                 break;
3917         case 0:
3918                 filter_info->proto_mask = 1;
3919                 break;
3920         default:
3921                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
3922                 return -EINVAL;
3923         }
3924
3925         filter_info->priority = (uint8_t)filter->priority;
3926         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
3927                 filter_info->tcp_flags = filter->tcp_flags;
3928         else
3929                 filter_info->tcp_flags = 0;
3930
3931         return 0;
3932 }
3933
3934 static inline struct e1000_5tuple_filter *
3935 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list,
3936                         struct e1000_5tuple_filter_info *key)
3937 {
3938         struct e1000_5tuple_filter *it;
3939
3940         TAILQ_FOREACH(it, filter_list, entries) {
3941                 if (memcmp(key, &it->filter_info,
3942                         sizeof(struct e1000_5tuple_filter_info)) == 0) {
3943                         return it;
3944                 }
3945         }
3946         return NULL;
3947 }
3948
3949 /*
3950  * igb_add_5tuple_filter_82576 - add a 5tuple filter
3951  *
3952  * @param
3953  * dev: Pointer to struct rte_eth_dev.
3954  * ntuple_filter: ponter to the filter that will be added.
3955  *
3956  * @return
3957  *    - On success, zero.
3958  *    - On failure, a negative value.
3959  */
3960 static int
3961 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
3962                         struct rte_eth_ntuple_filter *ntuple_filter)
3963 {
3964         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3965         struct e1000_filter_info *filter_info =
3966                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3967         struct e1000_5tuple_filter *filter;
3968         uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK;
3969         uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP;
3970         uint8_t i;
3971         int ret;
3972
3973         filter = rte_zmalloc("e1000_5tuple_filter",
3974                         sizeof(struct e1000_5tuple_filter), 0);
3975         if (filter == NULL)
3976                 return -ENOMEM;
3977
3978         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3979                                             &filter->filter_info);
3980         if (ret < 0) {
3981                 rte_free(filter);
3982                 return ret;
3983         }
3984
3985         if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
3986                                          &filter->filter_info) != NULL) {
3987                 PMD_DRV_LOG(ERR, "filter exists.");
3988                 rte_free(filter);
3989                 return -EEXIST;
3990         }
3991         filter->queue = ntuple_filter->queue;
3992
3993         /*
3994          * look for an unused 5tuple filter index,
3995          * and insert the filter to list.
3996          */
3997         for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) {
3998                 if (!(filter_info->fivetuple_mask & (1 << i))) {
3999                         filter_info->fivetuple_mask |= 1 << i;
4000                         filter->index = i;
4001                         TAILQ_INSERT_TAIL(&filter_info->fivetuple_list,
4002                                           filter,
4003                                           entries);
4004                         break;
4005                 }
4006         }
4007         if (i >= E1000_MAX_FTQF_FILTERS) {
4008                 PMD_DRV_LOG(ERR, "5tuple filters are full.");
4009                 rte_free(filter);
4010                 return -ENOSYS;
4011         }
4012
4013         ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK;
4014         if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */
4015                 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP;
4016         if (filter->filter_info.dst_ip_mask == 0)
4017                 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP;
4018         if (filter->filter_info.src_port_mask == 0)
4019                 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
4020         if (filter->filter_info.proto_mask == 0)
4021                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP;
4022         ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) &
4023                 E1000_FTQF_QUEUE_MASK;
4024         ftqf |= E1000_FTQF_QUEUE_ENABLE;
4025         E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf);
4026         E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip);
4027         E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip);
4028
4029         spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT;
4030         E1000_WRITE_REG(hw, E1000_SPQF(i), spqf);
4031
4032         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
4033         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
4034                 imir |= E1000_IMIR_PORT_BP;
4035         else
4036                 imir &= ~E1000_IMIR_PORT_BP;
4037         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
4038
4039         /* tcp flags bits setting. */
4040         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
4041                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
4042                         imir_ext |= E1000_IMIREXT_CTRL_URG;
4043                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
4044                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
4045                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
4046                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
4047                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
4048                         imir_ext |= E1000_IMIREXT_CTRL_RST;
4049                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
4050                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
4051                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
4052                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
4053         } else
4054                 imir_ext |= E1000_IMIREXT_CTRL_BP;
4055         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
4056         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
4057         return 0;
4058 }
4059
4060 /*
4061  * igb_remove_5tuple_filter_82576 - remove a 5tuple filter
4062  *
4063  * @param
4064  * dev: Pointer to struct rte_eth_dev.
4065  * ntuple_filter: ponter to the filter that will be removed.
4066  *
4067  * @return
4068  *    - On success, zero.
4069  *    - On failure, a negative value.
4070  */
4071 static int
4072 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
4073                                 struct rte_eth_ntuple_filter *ntuple_filter)
4074 {
4075         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4076         struct e1000_filter_info *filter_info =
4077                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4078         struct e1000_5tuple_filter_info filter_5tuple;
4079         struct e1000_5tuple_filter *filter;
4080         int ret;
4081
4082         memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info));
4083         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
4084                                             &filter_5tuple);
4085         if (ret < 0)
4086                 return ret;
4087
4088         filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
4089                                          &filter_5tuple);
4090         if (filter == NULL) {
4091                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
4092                 return -ENOENT;
4093         }
4094
4095         filter_info->fivetuple_mask &= ~(1 << filter->index);
4096         TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries);
4097         rte_free(filter);
4098
4099         E1000_WRITE_REG(hw, E1000_FTQF(filter->index),
4100                         E1000_FTQF_VF_BP | E1000_FTQF_MASK);
4101         E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0);
4102         E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0);
4103         E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0);
4104         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
4105         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
4106         return 0;
4107 }
4108
4109 static int
4110 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
4111 {
4112         uint32_t rctl;
4113         struct e1000_hw *hw;
4114         struct rte_eth_dev_info dev_info;
4115         uint32_t frame_size = mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN +
4116                                      VLAN_TAG_SIZE);
4117
4118         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4119
4120 #ifdef RTE_LIBRTE_82571_SUPPORT
4121         /* XXX: not bigger than max_rx_pktlen */
4122         if (hw->mac.type == e1000_82571)
4123                 return -ENOTSUP;
4124 #endif
4125         eth_igb_infos_get(dev, &dev_info);
4126
4127         /* check that mtu is within the allowed range */
4128         if ((mtu < ETHER_MIN_MTU) ||
4129             (frame_size > dev_info.max_rx_pktlen))
4130                 return -EINVAL;
4131
4132         /* refuse mtu that requires the support of scattered packets when this
4133          * feature has not been enabled before. */
4134         if (!dev->data->scattered_rx &&
4135             frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)
4136                 return -EINVAL;
4137
4138         rctl = E1000_READ_REG(hw, E1000_RCTL);
4139
4140         /* switch to jumbo mode if needed */
4141         if (frame_size > ETHER_MAX_LEN) {
4142                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
4143                 rctl |= E1000_RCTL_LPE;
4144         } else {
4145                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
4146                 rctl &= ~E1000_RCTL_LPE;
4147         }
4148         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4149
4150         /* update max frame size */
4151         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
4152
4153         E1000_WRITE_REG(hw, E1000_RLPML,
4154                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
4155
4156         return 0;
4157 }
4158
4159 /*
4160  * igb_add_del_ntuple_filter - add or delete a ntuple filter
4161  *
4162  * @param
4163  * dev: Pointer to struct rte_eth_dev.
4164  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
4165  * add: if true, add filter, if false, remove filter
4166  *
4167  * @return
4168  *    - On success, zero.
4169  *    - On failure, a negative value.
4170  */
4171 static int
4172 igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
4173                         struct rte_eth_ntuple_filter *ntuple_filter,
4174                         bool add)
4175 {
4176         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4177         int ret;
4178
4179         switch (ntuple_filter->flags) {
4180         case RTE_5TUPLE_FLAGS:
4181         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
4182                 if (hw->mac.type != e1000_82576)
4183                         return -ENOTSUP;
4184                 if (add)
4185                         ret = igb_add_5tuple_filter_82576(dev,
4186                                                           ntuple_filter);
4187                 else
4188                         ret = igb_remove_5tuple_filter_82576(dev,
4189                                                              ntuple_filter);
4190                 break;
4191         case RTE_2TUPLE_FLAGS:
4192         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
4193                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
4194                         return -ENOTSUP;
4195                 if (add)
4196                         ret = igb_add_2tuple_filter(dev, ntuple_filter);
4197                 else
4198                         ret = igb_remove_2tuple_filter(dev, ntuple_filter);
4199                 break;
4200         default:
4201                 ret = -EINVAL;
4202                 break;
4203         }
4204
4205         return ret;
4206 }
4207
4208 /*
4209  * igb_get_ntuple_filter - get a ntuple filter
4210  *
4211  * @param
4212  * dev: Pointer to struct rte_eth_dev.
4213  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
4214  *
4215  * @return
4216  *    - On success, zero.
4217  *    - On failure, a negative value.
4218  */
4219 static int
4220 igb_get_ntuple_filter(struct rte_eth_dev *dev,
4221                         struct rte_eth_ntuple_filter *ntuple_filter)
4222 {
4223         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4224         struct e1000_filter_info *filter_info =
4225                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4226         struct e1000_5tuple_filter_info filter_5tuple;
4227         struct e1000_2tuple_filter_info filter_2tuple;
4228         struct e1000_5tuple_filter *p_5tuple_filter;
4229         struct e1000_2tuple_filter *p_2tuple_filter;
4230         int ret;
4231
4232         switch (ntuple_filter->flags) {
4233         case RTE_5TUPLE_FLAGS:
4234         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
4235                 if (hw->mac.type != e1000_82576)
4236                         return -ENOTSUP;
4237                 memset(&filter_5tuple,
4238                         0,
4239                         sizeof(struct e1000_5tuple_filter_info));
4240                 ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
4241                                                     &filter_5tuple);
4242                 if (ret < 0)
4243                         return ret;
4244                 p_5tuple_filter = igb_5tuple_filter_lookup_82576(
4245                                         &filter_info->fivetuple_list,
4246                                         &filter_5tuple);
4247                 if (p_5tuple_filter == NULL) {
4248                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
4249                         return -ENOENT;
4250                 }
4251                 ntuple_filter->queue = p_5tuple_filter->queue;
4252                 break;
4253         case RTE_2TUPLE_FLAGS:
4254         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
4255                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
4256                         return -ENOTSUP;
4257                 memset(&filter_2tuple,
4258                         0,
4259                         sizeof(struct e1000_2tuple_filter_info));
4260                 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple);
4261                 if (ret < 0)
4262                         return ret;
4263                 p_2tuple_filter = igb_2tuple_filter_lookup(
4264                                         &filter_info->twotuple_list,
4265                                         &filter_2tuple);
4266                 if (p_2tuple_filter == NULL) {
4267                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
4268                         return -ENOENT;
4269                 }
4270                 ntuple_filter->queue = p_2tuple_filter->queue;
4271                 break;
4272         default:
4273                 ret = -EINVAL;
4274                 break;
4275         }
4276
4277         return 0;
4278 }
4279
4280 /*
4281  * igb_ntuple_filter_handle - Handle operations for ntuple filter.
4282  * @dev: pointer to rte_eth_dev structure
4283  * @filter_op:operation will be taken.
4284  * @arg: a pointer to specific structure corresponding to the filter_op
4285  */
4286 static int
4287 igb_ntuple_filter_handle(struct rte_eth_dev *dev,
4288                                 enum rte_filter_op filter_op,
4289                                 void *arg)
4290 {
4291         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4292         int ret;
4293
4294         MAC_TYPE_FILTER_SUP(hw->mac.type);
4295
4296         if (filter_op == RTE_ETH_FILTER_NOP)
4297                 return 0;
4298
4299         if (arg == NULL) {
4300                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
4301                             filter_op);
4302                 return -EINVAL;
4303         }
4304
4305         switch (filter_op) {
4306         case RTE_ETH_FILTER_ADD:
4307                 ret = igb_add_del_ntuple_filter(dev,
4308                         (struct rte_eth_ntuple_filter *)arg,
4309                         TRUE);
4310                 break;
4311         case RTE_ETH_FILTER_DELETE:
4312                 ret = igb_add_del_ntuple_filter(dev,
4313                         (struct rte_eth_ntuple_filter *)arg,
4314                         FALSE);
4315                 break;
4316         case RTE_ETH_FILTER_GET:
4317                 ret = igb_get_ntuple_filter(dev,
4318                         (struct rte_eth_ntuple_filter *)arg);
4319                 break;
4320         default:
4321                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
4322                 ret = -EINVAL;
4323                 break;
4324         }
4325         return ret;
4326 }
4327
4328 static inline int
4329 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info,
4330                         uint16_t ethertype)
4331 {
4332         int i;
4333
4334         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
4335                 if (filter_info->ethertype_filters[i] == ethertype &&
4336                     (filter_info->ethertype_mask & (1 << i)))
4337                         return i;
4338         }
4339         return -1;
4340 }
4341
4342 static inline int
4343 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info,
4344                         uint16_t ethertype)
4345 {
4346         int i;
4347
4348         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
4349                 if (!(filter_info->ethertype_mask & (1 << i))) {
4350                         filter_info->ethertype_mask |= 1 << i;
4351                         filter_info->ethertype_filters[i] = ethertype;
4352                         return i;
4353                 }
4354         }
4355         return -1;
4356 }
4357
4358 static inline int
4359 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info,
4360                         uint8_t idx)
4361 {
4362         if (idx >= E1000_MAX_ETQF_FILTERS)
4363                 return -1;
4364         filter_info->ethertype_mask &= ~(1 << idx);
4365         filter_info->ethertype_filters[idx] = 0;
4366         return idx;
4367 }
4368
4369
4370 static int
4371 igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
4372                         struct rte_eth_ethertype_filter *filter,
4373                         bool add)
4374 {
4375         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4376         struct e1000_filter_info *filter_info =
4377                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4378         uint32_t etqf = 0;
4379         int ret;
4380
4381         if (filter->ether_type == ETHER_TYPE_IPv4 ||
4382                 filter->ether_type == ETHER_TYPE_IPv6) {
4383                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
4384                         " ethertype filter.", filter->ether_type);
4385                 return -EINVAL;
4386         }
4387
4388         if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) {
4389                 PMD_DRV_LOG(ERR, "mac compare is unsupported.");
4390                 return -EINVAL;
4391         }
4392         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) {
4393                 PMD_DRV_LOG(ERR, "drop option is unsupported.");
4394                 return -EINVAL;
4395         }
4396
4397         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
4398         if (ret >= 0 && add) {
4399                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.",
4400                             filter->ether_type);
4401                 return -EEXIST;
4402         }
4403         if (ret < 0 && !add) {
4404                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
4405                             filter->ether_type);
4406                 return -ENOENT;
4407         }
4408
4409         if (add) {
4410                 ret = igb_ethertype_filter_insert(filter_info,
4411                         filter->ether_type);
4412                 if (ret < 0) {
4413                         PMD_DRV_LOG(ERR, "ethertype filters are full.");
4414                         return -ENOSYS;
4415                 }
4416
4417                 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE;
4418                 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE);
4419                 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT;
4420         } else {
4421                 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret);
4422                 if (ret < 0)
4423                         return -ENOSYS;
4424         }
4425         E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf);
4426         E1000_WRITE_FLUSH(hw);
4427
4428         return 0;
4429 }
4430
4431 static int
4432 igb_get_ethertype_filter(struct rte_eth_dev *dev,
4433                         struct rte_eth_ethertype_filter *filter)
4434 {
4435         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4436         struct e1000_filter_info *filter_info =
4437                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4438         uint32_t etqf;
4439         int ret;
4440
4441         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
4442         if (ret < 0) {
4443                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
4444                             filter->ether_type);
4445                 return -ENOENT;
4446         }
4447
4448         etqf = E1000_READ_REG(hw, E1000_ETQF(ret));
4449         if (etqf & E1000_ETQF_FILTER_ENABLE) {
4450                 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE;
4451                 filter->flags = 0;
4452                 filter->queue = (etqf & E1000_ETQF_QUEUE) >>
4453                                 E1000_ETQF_QUEUE_SHIFT;
4454                 return 0;
4455         }
4456
4457         return -ENOENT;
4458 }
4459
4460 /*
4461  * igb_ethertype_filter_handle - Handle operations for ethertype filter.
4462  * @dev: pointer to rte_eth_dev structure
4463  * @filter_op:operation will be taken.
4464  * @arg: a pointer to specific structure corresponding to the filter_op
4465  */
4466 static int
4467 igb_ethertype_filter_handle(struct rte_eth_dev *dev,
4468                                 enum rte_filter_op filter_op,
4469                                 void *arg)
4470 {
4471         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4472         int ret;
4473
4474         MAC_TYPE_FILTER_SUP(hw->mac.type);
4475
4476         if (filter_op == RTE_ETH_FILTER_NOP)
4477                 return 0;
4478
4479         if (arg == NULL) {
4480                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
4481                             filter_op);
4482                 return -EINVAL;
4483         }
4484
4485         switch (filter_op) {
4486         case RTE_ETH_FILTER_ADD:
4487                 ret = igb_add_del_ethertype_filter(dev,
4488                         (struct rte_eth_ethertype_filter *)arg,
4489                         TRUE);
4490                 break;
4491         case RTE_ETH_FILTER_DELETE:
4492                 ret = igb_add_del_ethertype_filter(dev,
4493                         (struct rte_eth_ethertype_filter *)arg,
4494                         FALSE);
4495                 break;
4496         case RTE_ETH_FILTER_GET:
4497                 ret = igb_get_ethertype_filter(dev,
4498                         (struct rte_eth_ethertype_filter *)arg);
4499                 break;
4500         default:
4501                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
4502                 ret = -EINVAL;
4503                 break;
4504         }
4505         return ret;
4506 }
4507
4508 static int
4509 eth_igb_filter_ctrl(struct rte_eth_dev *dev,
4510                      enum rte_filter_type filter_type,
4511                      enum rte_filter_op filter_op,
4512                      void *arg)
4513 {
4514         int ret = -EINVAL;
4515
4516         switch (filter_type) {
4517         case RTE_ETH_FILTER_NTUPLE:
4518                 ret = igb_ntuple_filter_handle(dev, filter_op, arg);
4519                 break;
4520         case RTE_ETH_FILTER_ETHERTYPE:
4521                 ret = igb_ethertype_filter_handle(dev, filter_op, arg);
4522                 break;
4523         case RTE_ETH_FILTER_SYN:
4524                 ret = eth_igb_syn_filter_handle(dev, filter_op, arg);
4525                 break;
4526         case RTE_ETH_FILTER_FLEXIBLE:
4527                 ret = eth_igb_flex_filter_handle(dev, filter_op, arg);
4528                 break;
4529         default:
4530                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
4531                                                         filter_type);
4532                 break;
4533         }
4534
4535         return ret;
4536 }
4537
4538 static int
4539 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
4540                          struct ether_addr *mc_addr_set,
4541                          uint32_t nb_mc_addr)
4542 {
4543         struct e1000_hw *hw;
4544
4545         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4546         e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr);
4547         return 0;
4548 }
4549
4550 static uint64_t
4551 igb_read_systime_cyclecounter(struct rte_eth_dev *dev)
4552 {
4553         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4554         uint64_t systime_cycles;
4555
4556         switch (hw->mac.type) {
4557         case e1000_i210:
4558         case e1000_i211:
4559                 /*
4560                  * Need to read System Time Residue Register to be able
4561                  * to read the other two registers.
4562                  */
4563                 E1000_READ_REG(hw, E1000_SYSTIMR);
4564                 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */
4565                 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML);
4566                 systime_cycles += (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH)
4567                                 * NSEC_PER_SEC;
4568                 break;
4569         case e1000_82580:
4570         case e1000_i350:
4571         case e1000_i354:
4572                 /*
4573                  * Need to read System Time Residue Register to be able
4574                  * to read the other two registers.
4575                  */
4576                 E1000_READ_REG(hw, E1000_SYSTIMR);
4577                 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML);
4578                 /* Only the 8 LSB are valid. */
4579                 systime_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_SYSTIMH)
4580                                 & 0xff) << 32;
4581                 break;
4582         default:
4583                 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML);
4584                 systime_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH)
4585                                 << 32;
4586                 break;
4587         }
4588
4589         return systime_cycles;
4590 }
4591
4592 static uint64_t
4593 igb_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev)
4594 {
4595         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4596         uint64_t rx_tstamp_cycles;
4597
4598         switch (hw->mac.type) {
4599         case e1000_i210:
4600         case e1000_i211:
4601                 /* RXSTMPL stores ns and RXSTMPH stores seconds. */
4602                 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL);
4603                 rx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH)
4604                                 * NSEC_PER_SEC;
4605                 break;
4606         case e1000_82580:
4607         case e1000_i350:
4608         case e1000_i354:
4609                 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL);
4610                 /* Only the 8 LSB are valid. */
4611                 rx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_RXSTMPH)
4612                                 & 0xff) << 32;
4613                 break;
4614         default:
4615                 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL);
4616                 rx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH)
4617                                 << 32;
4618                 break;
4619         }
4620
4621         return rx_tstamp_cycles;
4622 }
4623
4624 static uint64_t
4625 igb_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev)
4626 {
4627         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4628         uint64_t tx_tstamp_cycles;
4629
4630         switch (hw->mac.type) {
4631         case e1000_i210:
4632         case e1000_i211:
4633                 /* RXSTMPL stores ns and RXSTMPH stores seconds. */
4634                 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL);
4635                 tx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH)
4636                                 * NSEC_PER_SEC;
4637                 break;
4638         case e1000_82580:
4639         case e1000_i350:
4640         case e1000_i354:
4641                 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL);
4642                 /* Only the 8 LSB are valid. */
4643                 tx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_TXSTMPH)
4644                                 & 0xff) << 32;
4645                 break;
4646         default:
4647                 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL);
4648                 tx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH)
4649                                 << 32;
4650                 break;
4651         }
4652
4653         return tx_tstamp_cycles;
4654 }
4655
4656 static void
4657 igb_start_timecounters(struct rte_eth_dev *dev)
4658 {
4659         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4660         struct e1000_adapter *adapter =
4661                 (struct e1000_adapter *)dev->data->dev_private;
4662         uint32_t incval = 1;
4663         uint32_t shift = 0;
4664         uint64_t mask = E1000_CYCLECOUNTER_MASK;
4665
4666         switch (hw->mac.type) {
4667         case e1000_82580:
4668         case e1000_i350:
4669         case e1000_i354:
4670                 /* 32 LSB bits + 8 MSB bits = 40 bits */
4671                 mask = (1ULL << 40) - 1;
4672                 /* fall-through */
4673         case e1000_i210:
4674         case e1000_i211:
4675                 /*
4676                  * Start incrementing the register
4677                  * used to timestamp PTP packets.
4678                  */
4679                 E1000_WRITE_REG(hw, E1000_TIMINCA, incval);
4680                 break;
4681         case e1000_82576:
4682                 incval = E1000_INCVALUE_82576;
4683                 shift = IGB_82576_TSYNC_SHIFT;
4684                 E1000_WRITE_REG(hw, E1000_TIMINCA,
4685                                 E1000_INCPERIOD_82576 | incval);
4686                 break;
4687         default:
4688                 /* Not supported */
4689                 return;
4690         }
4691
4692         memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter));
4693         memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
4694         memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
4695
4696         adapter->systime_tc.cc_mask = mask;
4697         adapter->systime_tc.cc_shift = shift;
4698         adapter->systime_tc.nsec_mask = (1ULL << shift) - 1;
4699
4700         adapter->rx_tstamp_tc.cc_mask = mask;
4701         adapter->rx_tstamp_tc.cc_shift = shift;
4702         adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
4703
4704         adapter->tx_tstamp_tc.cc_mask = mask;
4705         adapter->tx_tstamp_tc.cc_shift = shift;
4706         adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
4707 }
4708
4709 static int
4710 igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
4711 {
4712         struct e1000_adapter *adapter =
4713                         (struct e1000_adapter *)dev->data->dev_private;
4714
4715         adapter->systime_tc.nsec += delta;
4716         adapter->rx_tstamp_tc.nsec += delta;
4717         adapter->tx_tstamp_tc.nsec += delta;
4718
4719         return 0;
4720 }
4721
4722 static int
4723 igb_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
4724 {
4725         uint64_t ns;
4726         struct e1000_adapter *adapter =
4727                         (struct e1000_adapter *)dev->data->dev_private;
4728
4729         ns = rte_timespec_to_ns(ts);
4730
4731         /* Set the timecounters to a new value. */
4732         adapter->systime_tc.nsec = ns;
4733         adapter->rx_tstamp_tc.nsec = ns;
4734         adapter->tx_tstamp_tc.nsec = ns;
4735
4736         return 0;
4737 }
4738
4739 static int
4740 igb_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
4741 {
4742         uint64_t ns, systime_cycles;
4743         struct e1000_adapter *adapter =
4744                         (struct e1000_adapter *)dev->data->dev_private;
4745
4746         systime_cycles = igb_read_systime_cyclecounter(dev);
4747         ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles);
4748         *ts = rte_ns_to_timespec(ns);
4749
4750         return 0;
4751 }
4752
4753 static int
4754 igb_timesync_enable(struct rte_eth_dev *dev)
4755 {
4756         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4757         uint32_t tsync_ctl;
4758         uint32_t tsauxc;
4759
4760         /* Stop the timesync system time. */
4761         E1000_WRITE_REG(hw, E1000_TIMINCA, 0x0);
4762         /* Reset the timesync system time value. */
4763         switch (hw->mac.type) {
4764         case e1000_82580:
4765         case e1000_i350:
4766         case e1000_i354:
4767         case e1000_i210:
4768         case e1000_i211:
4769                 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x0);
4770                 /* fall-through */
4771         case e1000_82576:
4772                 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x0);
4773                 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x0);
4774                 break;
4775         default:
4776                 /* Not supported. */
4777                 return -ENOTSUP;
4778         }
4779
4780         /* Enable system time for it isn't on by default. */
4781         tsauxc = E1000_READ_REG(hw, E1000_TSAUXC);
4782         tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME;
4783         E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc);
4784
4785         igb_start_timecounters(dev);
4786
4787         /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
4788         E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588),
4789                         (ETHER_TYPE_1588 |
4790                          E1000_ETQF_FILTER_ENABLE |
4791                          E1000_ETQF_1588));
4792
4793         /* Enable timestamping of received PTP packets. */
4794         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
4795         tsync_ctl |= E1000_TSYNCRXCTL_ENABLED;
4796         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl);
4797
4798         /* Enable Timestamping of transmitted PTP packets. */
4799         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
4800         tsync_ctl |= E1000_TSYNCTXCTL_ENABLED;
4801         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl);
4802
4803         return 0;
4804 }
4805
4806 static int
4807 igb_timesync_disable(struct rte_eth_dev *dev)
4808 {
4809         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4810         uint32_t tsync_ctl;
4811
4812         /* Disable timestamping of transmitted PTP packets. */
4813         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
4814         tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED;
4815         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl);
4816
4817         /* Disable timestamping of received PTP packets. */
4818         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
4819         tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED;
4820         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl);
4821
4822         /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
4823         E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0);
4824
4825         /* Stop incrementating the System Time registers. */
4826         E1000_WRITE_REG(hw, E1000_TIMINCA, 0);
4827
4828         return 0;
4829 }
4830
4831 static int
4832 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
4833                                struct timespec *timestamp,
4834                                uint32_t flags __rte_unused)
4835 {
4836         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4837         struct e1000_adapter *adapter =
4838                         (struct e1000_adapter *)dev->data->dev_private;
4839         uint32_t tsync_rxctl;
4840         uint64_t rx_tstamp_cycles;
4841         uint64_t ns;
4842
4843         tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
4844         if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0)
4845                 return -EINVAL;
4846
4847         rx_tstamp_cycles = igb_read_rx_tstamp_cyclecounter(dev);
4848         ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles);
4849         *timestamp = rte_ns_to_timespec(ns);
4850
4851         return  0;
4852 }
4853
4854 static int
4855 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
4856                                struct timespec *timestamp)
4857 {
4858         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4859         struct e1000_adapter *adapter =
4860                         (struct e1000_adapter *)dev->data->dev_private;
4861         uint32_t tsync_txctl;
4862         uint64_t tx_tstamp_cycles;
4863         uint64_t ns;
4864
4865         tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
4866         if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0)
4867                 return -EINVAL;
4868
4869         tx_tstamp_cycles = igb_read_tx_tstamp_cyclecounter(dev);
4870         ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles);
4871         *timestamp = rte_ns_to_timespec(ns);
4872
4873         return  0;
4874 }
4875
4876 static int
4877 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused)
4878 {
4879         int count = 0;
4880         int g_ind = 0;
4881         const struct reg_info *reg_group;
4882
4883         while ((reg_group = igb_regs[g_ind++]))
4884                 count += igb_reg_group_count(reg_group);
4885
4886         return count;
4887 }
4888
4889 static int
4890 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused)
4891 {
4892         int count = 0;
4893         int g_ind = 0;
4894         const struct reg_info *reg_group;
4895
4896         while ((reg_group = igbvf_regs[g_ind++]))
4897                 count += igb_reg_group_count(reg_group);
4898
4899         return count;
4900 }
4901
4902 static int
4903 eth_igb_get_regs(struct rte_eth_dev *dev,
4904         struct rte_dev_reg_info *regs)
4905 {
4906         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4907         uint32_t *data = regs->data;
4908         int g_ind = 0;
4909         int count = 0;
4910         const struct reg_info *reg_group;
4911
4912         /* Support only full register dump */
4913         if ((regs->length == 0) ||
4914             (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) {
4915                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4916                         hw->device_id;
4917                 while ((reg_group = igb_regs[g_ind++]))
4918                         count += igb_read_regs_group(dev, &data[count],
4919                                                         reg_group);
4920                 return 0;
4921         }
4922
4923         return -ENOTSUP;
4924 }
4925
4926 static int
4927 igbvf_get_regs(struct rte_eth_dev *dev,
4928         struct rte_dev_reg_info *regs)
4929 {
4930         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4931         uint32_t *data = regs->data;
4932         int g_ind = 0;
4933         int count = 0;
4934         const struct reg_info *reg_group;
4935
4936         /* Support only full register dump */
4937         if ((regs->length == 0) ||
4938             (regs->length == (uint32_t)igbvf_get_reg_length(dev))) {
4939                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4940                         hw->device_id;
4941                 while ((reg_group = igbvf_regs[g_ind++]))
4942                         count += igb_read_regs_group(dev, &data[count],
4943                                                         reg_group);
4944                 return 0;
4945         }
4946
4947         return -ENOTSUP;
4948 }
4949
4950 static int
4951 eth_igb_get_eeprom_length(struct rte_eth_dev *dev)
4952 {
4953         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4954
4955         /* Return unit is byte count */
4956         return hw->nvm.word_size * 2;
4957 }
4958
4959 static int
4960 eth_igb_get_eeprom(struct rte_eth_dev *dev,
4961         struct rte_dev_eeprom_info *in_eeprom)
4962 {
4963         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4964         struct e1000_nvm_info *nvm = &hw->nvm;
4965         uint16_t *data = in_eeprom->data;
4966         int first, length;
4967
4968         first = in_eeprom->offset >> 1;
4969         length = in_eeprom->length >> 1;
4970         if ((first >= hw->nvm.word_size) ||
4971             ((first + length) >= hw->nvm.word_size))
4972                 return -EINVAL;
4973
4974         in_eeprom->magic = hw->vendor_id |
4975                 ((uint32_t)hw->device_id << 16);
4976
4977         if ((nvm->ops.read) == NULL)
4978                 return -ENOTSUP;
4979
4980         return nvm->ops.read(hw, first, length, data);
4981 }
4982
4983 static int
4984 eth_igb_set_eeprom(struct rte_eth_dev *dev,
4985         struct rte_dev_eeprom_info *in_eeprom)
4986 {
4987         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4988         struct e1000_nvm_info *nvm = &hw->nvm;
4989         uint16_t *data = in_eeprom->data;
4990         int first, length;
4991
4992         first = in_eeprom->offset >> 1;
4993         length = in_eeprom->length >> 1;
4994         if ((first >= hw->nvm.word_size) ||
4995             ((first + length) >= hw->nvm.word_size))
4996                 return -EINVAL;
4997
4998         in_eeprom->magic = (uint32_t)hw->vendor_id |
4999                 ((uint32_t)hw->device_id << 16);
5000
5001         if ((nvm->ops.write) == NULL)
5002                 return -ENOTSUP;
5003         return nvm->ops.write(hw,  first, length, data);
5004 }
5005
5006 static struct rte_driver pmd_igb_drv = {
5007         .type = PMD_PDEV,
5008         .init = rte_igb_pmd_init,
5009 };
5010
5011 static struct rte_driver pmd_igbvf_drv = {
5012         .type = PMD_PDEV,
5013         .init = rte_igbvf_pmd_init,
5014 };
5015
5016 static int
5017 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
5018 {
5019         struct e1000_hw *hw =
5020                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5021         uint32_t mask = 1 << queue_id;
5022
5023         E1000_WRITE_REG(hw, E1000_EIMC, mask);
5024         E1000_WRITE_FLUSH(hw);
5025
5026         return 0;
5027 }
5028
5029 static int
5030 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
5031 {
5032         struct e1000_hw *hw =
5033                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5034         uint32_t mask = 1 << queue_id;
5035         uint32_t regval;
5036
5037         regval = E1000_READ_REG(hw, E1000_EIMS);
5038         E1000_WRITE_REG(hw, E1000_EIMS, regval | mask);
5039         E1000_WRITE_FLUSH(hw);
5040
5041         rte_intr_enable(&dev->pci_dev->intr_handle);
5042
5043         return 0;
5044 }
5045
5046 static void
5047 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t  msix_vector,
5048                    uint8_t index, uint8_t offset)
5049 {
5050         uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
5051
5052         /* clear bits */
5053         val &= ~((uint32_t)0xFF << offset);
5054
5055         /* write vector and valid bit */
5056         val |= (msix_vector | E1000_IVAR_VALID) << offset;
5057
5058         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val);
5059 }
5060
5061 static void
5062 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction,
5063                            uint8_t queue, uint8_t msix_vector)
5064 {
5065         uint32_t tmp = 0;
5066
5067         if (hw->mac.type == e1000_82575) {
5068                 if (direction == 0)
5069                         tmp = E1000_EICR_RX_QUEUE0 << queue;
5070                 else if (direction == 1)
5071                         tmp = E1000_EICR_TX_QUEUE0 << queue;
5072                 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp);
5073         } else if (hw->mac.type == e1000_82576) {
5074                 if ((direction == 0) || (direction == 1))
5075                         eth_igb_write_ivar(hw, msix_vector, queue & 0x7,
5076                                            ((queue & 0x8) << 1) +
5077                                            8 * direction);
5078         } else if ((hw->mac.type == e1000_82580) ||
5079                         (hw->mac.type == e1000_i350) ||
5080                         (hw->mac.type == e1000_i354) ||
5081                         (hw->mac.type == e1000_i210) ||
5082                         (hw->mac.type == e1000_i211)) {
5083                 if ((direction == 0) || (direction == 1))
5084                         eth_igb_write_ivar(hw, msix_vector,
5085                                            queue >> 1,
5086                                            ((queue & 0x1) << 4) +
5087                                            8 * direction);
5088         }
5089 }
5090
5091 /* Sets up the hardware to generate MSI-X interrupts properly
5092  * @hw
5093  *  board private structure
5094  */
5095 static void
5096 eth_igb_configure_msix_intr(struct rte_eth_dev *dev)
5097 {
5098         int queue_id;
5099         uint32_t tmpval, regval, intr_mask;
5100         struct e1000_hw *hw =
5101                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5102         uint32_t vec = E1000_MISC_VEC_ID;
5103         uint32_t base = E1000_MISC_VEC_ID;
5104         uint32_t misc_shift = 0;
5105
5106         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
5107
5108         /* won't configure msix register if no mapping is done
5109          * between intr vector and event fd
5110          */
5111         if (!rte_intr_dp_is_en(intr_handle))
5112                 return;
5113
5114         if (rte_intr_allow_others(intr_handle)) {
5115                 vec = base = E1000_RX_VEC_START;
5116                 misc_shift = 1;
5117         }
5118
5119         /* set interrupt vector for other causes */
5120         if (hw->mac.type == e1000_82575) {
5121                 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT);
5122                 /* enable MSI-X PBA support */
5123                 tmpval |= E1000_CTRL_EXT_PBA_CLR;
5124
5125                 /* Auto-Mask interrupts upon ICR read */
5126                 tmpval |= E1000_CTRL_EXT_EIAME;
5127                 tmpval |= E1000_CTRL_EXT_IRCA;
5128
5129                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval);
5130
5131                 /* enable msix_other interrupt */
5132                 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER);
5133                 regval = E1000_READ_REG(hw, E1000_EIAC);
5134                 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER);
5135                 regval = E1000_READ_REG(hw, E1000_EIAM);
5136                 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER);
5137         } else if ((hw->mac.type == e1000_82576) ||
5138                         (hw->mac.type == e1000_82580) ||
5139                         (hw->mac.type == e1000_i350) ||
5140                         (hw->mac.type == e1000_i354) ||
5141                         (hw->mac.type == e1000_i210) ||
5142                         (hw->mac.type == e1000_i211)) {
5143                 /* turn on MSI-X capability first */
5144                 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE |
5145                                         E1000_GPIE_PBA | E1000_GPIE_EIAME |
5146                                         E1000_GPIE_NSICR);
5147                 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) <<
5148                         misc_shift;
5149                 regval = E1000_READ_REG(hw, E1000_EIAC);
5150                 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask);
5151
5152                 /* enable msix_other interrupt */
5153                 regval = E1000_READ_REG(hw, E1000_EIMS);
5154                 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask);
5155                 tmpval = (dev->data->nb_rx_queues | E1000_IVAR_VALID) << 8;
5156                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval);
5157         }
5158
5159         /* use EIAM to auto-mask when MSI-X interrupt
5160          * is asserted, this saves a register write for every interrupt
5161          */
5162         intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) <<
5163                 misc_shift;
5164         regval = E1000_READ_REG(hw, E1000_EIAM);
5165         E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask);
5166
5167         for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) {
5168                 eth_igb_assign_msix_vector(hw, 0, queue_id, vec);
5169                 intr_handle->intr_vec[queue_id] = vec;
5170                 if (vec < base + intr_handle->nb_efd - 1)
5171                         vec++;
5172         }
5173
5174         E1000_WRITE_FLUSH(hw);
5175 }
5176
5177 PMD_REGISTER_DRIVER(pmd_igb_drv);
5178 PMD_REGISTER_DRIVER(pmd_igbvf_drv);