drivers/net/e1000/igb_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <stdarg.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_interrupts.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_log.h>
  44 #include <rte_debug.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_ethdev_pci.h>
  49 #include <rte_memory.h>
  50 #include <rte_memzone.h>
  51 #include <rte_eal.h>
  52 #include <rte_atomic.h>
  53 #include <rte_malloc.h>
  54 #include <rte_dev.h>
  55
  56 #include "e1000_logs.h"
  57 #include "base/e1000_api.h"
  58 #include "e1000_ethdev.h"
  59 #include "igb_regs.h"
  60
  61 /*
  62  * Default values for port configuration
  63  */
  64 #define IGB_DEFAULT_RX_FREE_THRESH  32
  65
  66 #define IGB_DEFAULT_RX_PTHRESH      ((hw->mac.type == e1000_i354) ? 12 : 8)
  67 #define IGB_DEFAULT_RX_HTHRESH      8
  68 #define IGB_DEFAULT_RX_WTHRESH      ((hw->mac.type == e1000_82576) ? 1 : 4)
  69
  70 #define IGB_DEFAULT_TX_PTHRESH      ((hw->mac.type == e1000_i354) ? 20 : 8)
  71 #define IGB_DEFAULT_TX_HTHRESH      1
  72 #define IGB_DEFAULT_TX_WTHRESH      ((hw->mac.type == e1000_82576) ? 1 : 16)
  73
  74 #define IGB_HKEY_MAX_INDEX 10
  75
  76 /* Bit shift and mask */
  77 #define IGB_4_BIT_WIDTH  (CHAR_BIT / 2)
  78 #define IGB_4_BIT_MASK   RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t)
  79 #define IGB_8_BIT_WIDTH  CHAR_BIT
  80 #define IGB_8_BIT_MASK   UINT8_MAX
  81
  82 /* Additional timesync values. */
  83 #define E1000_CYCLECOUNTER_MASK      0xffffffffffffffffULL
  84 #define E1000_ETQF_FILTER_1588       3
  85 #define IGB_82576_TSYNC_SHIFT        16
  86 #define E1000_INCPERIOD_82576        (1 << E1000_TIMINCA_16NS_SHIFT)
  87 #define E1000_INCVALUE_82576         (16 << IGB_82576_TSYNC_SHIFT)
  88 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000
  89
  90 #define E1000_VTIVAR_MISC                0x01740
  91 #define E1000_VTIVAR_MISC_MASK           0xFF
  92 #define E1000_VTIVAR_VALID               0x80
  93 #define E1000_VTIVAR_MISC_MAILBOX        0
  94 #define E1000_VTIVAR_MISC_INTR_MASK      0x3
  95
  96 /* External VLAN Enable bit mask */
  97 #define E1000_CTRL_EXT_EXT_VLAN      (1 << 26)
  98
  99 /* External VLAN Ether Type bit mask and shift */
 100 #define E1000_VET_VET_EXT            0xFFFF0000
 101 #define E1000_VET_VET_EXT_SHIFT      16
 102
 103 static int  eth_igb_configure(struct rte_eth_dev *dev);
 104 static int  eth_igb_start(struct rte_eth_dev *dev);
 105 static void eth_igb_stop(struct rte_eth_dev *dev);
 106 static int  eth_igb_dev_set_link_up(struct rte_eth_dev *dev);
 107 static int  eth_igb_dev_set_link_down(struct rte_eth_dev *dev);
 108 static void eth_igb_close(struct rte_eth_dev *dev);
 109 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev);
 110 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev);
 111 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev);
 112 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev);
 113 static int  eth_igb_link_update(struct rte_eth_dev *dev,
 114                                 int wait_to_complete);
 115 static void eth_igb_stats_get(struct rte_eth_dev *dev,
 116                                 struct rte_eth_stats *rte_stats);
 117 static int eth_igb_xstats_get(struct rte_eth_dev *dev,
 118                               struct rte_eth_xstat *xstats, unsigned n);
 119 static int eth_igb_xstats_get_by_ids(struct rte_eth_dev *dev, uint64_t *ids,
 120                 uint64_t *values, unsigned int n);
 121 static int eth_igb_xstats_get_names(struct rte_eth_dev *dev,
 122                 struct rte_eth_xstat_name *xstats_names, unsigned int limit);
 123 static int eth_igb_xstats_get_names_by_ids(struct rte_eth_dev *dev,
 124                 struct rte_eth_xstat_name *xstats_names, uint64_t *ids,
 125                 unsigned int limit);
 126 static void eth_igb_stats_reset(struct rte_eth_dev *dev);
 127 static void eth_igb_xstats_reset(struct rte_eth_dev *dev);
 128 static int eth_igb_fw_version_get(struct rte_eth_dev *dev,
 129                                    char *fw_version, size_t fw_size);
 130 static void eth_igb_infos_get(struct rte_eth_dev *dev,
 131                               struct rte_eth_dev_info *dev_info);
 132 static const uint32_t *eth_igb_supported_ptypes_get(struct rte_eth_dev *dev);
 133 static void eth_igbvf_infos_get(struct rte_eth_dev *dev,
 134                                 struct rte_eth_dev_info *dev_info);
 135 static int  eth_igb_flow_ctrl_get(struct rte_eth_dev *dev,
 136                                 struct rte_eth_fc_conf *fc_conf);
 137 static int  eth_igb_flow_ctrl_set(struct rte_eth_dev *dev,
 138                                 struct rte_eth_fc_conf *fc_conf);
 139 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev);
 140 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev);
 141 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev);
 142 static int eth_igb_interrupt_action(struct rte_eth_dev *dev,
 143                                     struct rte_intr_handle *handle);
 144 static void eth_igb_interrupt_handler(void *param);
 145 static int  igb_hardware_init(struct e1000_hw *hw);
 146 static void igb_hw_control_acquire(struct e1000_hw *hw);
 147 static void igb_hw_control_release(struct e1000_hw *hw);
 148 static void igb_init_manageability(struct e1000_hw *hw);
 149 static void igb_release_manageability(struct e1000_hw *hw);
 150
 151 static int  eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 152
 153 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev,
 154                 uint16_t vlan_id, int on);
 155 static int eth_igb_vlan_tpid_set(struct rte_eth_dev *dev,
 156                                  enum rte_vlan_type vlan_type,
 157                                  uint16_t tpid_id);
 158 static void eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 159
 160 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 161 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 162 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev);
 163 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev);
 164 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 165 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 166
 167 static int eth_igb_led_on(struct rte_eth_dev *dev);
 168 static int eth_igb_led_off(struct rte_eth_dev *dev);
 169
 170 static void igb_intr_disable(struct e1000_hw *hw);
 171 static int  igb_get_rx_buffer_size(struct e1000_hw *hw);
 172 static void eth_igb_rar_set(struct rte_eth_dev *dev,
 173                 struct ether_addr *mac_addr,
 174                 uint32_t index, uint32_t pool);
 175 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 176 static void eth_igb_default_mac_addr_set(struct rte_eth_dev *dev,
 177                 struct ether_addr *addr);
 178
 179 static void igbvf_intr_disable(struct e1000_hw *hw);
 180 static int igbvf_dev_configure(struct rte_eth_dev *dev);
 181 static int igbvf_dev_start(struct rte_eth_dev *dev);
 182 static void igbvf_dev_stop(struct rte_eth_dev *dev);
 183 static void igbvf_dev_close(struct rte_eth_dev *dev);
 184 static void igbvf_promiscuous_enable(struct rte_eth_dev *dev);
 185 static void igbvf_promiscuous_disable(struct rte_eth_dev *dev);
 186 static void igbvf_allmulticast_enable(struct rte_eth_dev *dev);
 187 static void igbvf_allmulticast_disable(struct rte_eth_dev *dev);
 188 static int eth_igbvf_link_update(struct e1000_hw *hw);
 189 static void eth_igbvf_stats_get(struct rte_eth_dev *dev,
 190                                 struct rte_eth_stats *rte_stats);
 191 static int eth_igbvf_xstats_get(struct rte_eth_dev *dev,
 192                                 struct rte_eth_xstat *xstats, unsigned n);
 193 static int eth_igbvf_xstats_get_names(struct rte_eth_dev *dev,
 194                                       struct rte_eth_xstat_name *xstats_names,
 195                                       unsigned limit);
 196 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev);
 197 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev,
 198                 uint16_t vlan_id, int on);
 199 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on);
 200 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 201 static void igbvf_default_mac_addr_set(struct rte_eth_dev *dev,
 202                 struct ether_addr *addr);
 203 static int igbvf_get_reg_length(struct rte_eth_dev *dev);
 204 static int igbvf_get_regs(struct rte_eth_dev *dev,
 205                 struct rte_dev_reg_info *regs);
 206
 207 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev,
 208                                    struct rte_eth_rss_reta_entry64 *reta_conf,
 209                                    uint16_t reta_size);
 210 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev,
 211                                   struct rte_eth_rss_reta_entry64 *reta_conf,
 212                                   uint16_t reta_size);
 213
 214 static int eth_igb_syn_filter_set(struct rte_eth_dev *dev,
 215                         struct rte_eth_syn_filter *filter,
 216                         bool add);
 217 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev,
 218                         struct rte_eth_syn_filter *filter);
 219 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
 220                         enum rte_filter_op filter_op,
 221                         void *arg);
 222 static int igb_add_2tuple_filter(struct rte_eth_dev *dev,
 223                         struct rte_eth_ntuple_filter *ntuple_filter);
 224 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev,
 225                         struct rte_eth_ntuple_filter *ntuple_filter);
 226 static int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
 227                         struct rte_eth_flex_filter *filter,
 228                         bool add);
 229 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev,
 230                         struct rte_eth_flex_filter *filter);
 231 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
 232                         enum rte_filter_op filter_op,
 233                         void *arg);
 234 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
 235                         struct rte_eth_ntuple_filter *ntuple_filter);
 236 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
 237                         struct rte_eth_ntuple_filter *ntuple_filter);
 238 static int igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
 239                         struct rte_eth_ntuple_filter *filter,
 240                         bool add);
 241 static int igb_get_ntuple_filter(struct rte_eth_dev *dev,
 242                         struct rte_eth_ntuple_filter *filter);
 243 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev,
 244                                 enum rte_filter_op filter_op,
 245                                 void *arg);
 246 static int igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
 247                         struct rte_eth_ethertype_filter *filter,
 248                         bool add);
 249 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev,
 250                                 enum rte_filter_op filter_op,
 251                                 void *arg);
 252 static int igb_get_ethertype_filter(struct rte_eth_dev *dev,
 253                         struct rte_eth_ethertype_filter *filter);
 254 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev,
 255                      enum rte_filter_type filter_type,
 256                      enum rte_filter_op filter_op,
 257                      void *arg);
 258 static int eth_igb_get_reg_length(struct rte_eth_dev *dev);
 259 static int eth_igb_get_regs(struct rte_eth_dev *dev,
 260                 struct rte_dev_reg_info *regs);
 261 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev);
 262 static int eth_igb_get_eeprom(struct rte_eth_dev *dev,
 263                 struct rte_dev_eeprom_info *eeprom);
 264 static int eth_igb_set_eeprom(struct rte_eth_dev *dev,
 265                 struct rte_dev_eeprom_info *eeprom);
 266 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
 267                                     struct ether_addr *mc_addr_set,
 268                                     uint32_t nb_mc_addr);
 269 static int igb_timesync_enable(struct rte_eth_dev *dev);
 270 static int igb_timesync_disable(struct rte_eth_dev *dev);
 271 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 272                                           struct timespec *timestamp,
 273                                           uint32_t flags);
 274 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 275                                           struct timespec *timestamp);
 276 static int igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta);
 277 static int igb_timesync_read_time(struct rte_eth_dev *dev,
 278                                   struct timespec *timestamp);
 279 static int igb_timesync_write_time(struct rte_eth_dev *dev,
 280                                    const struct timespec *timestamp);
 281 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev,
 282                                         uint16_t queue_id);
 283 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev,
 284                                          uint16_t queue_id);
 285 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction,
 286                                        uint8_t queue, uint8_t msix_vector);
 287 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector,
 288                                uint8_t index, uint8_t offset);
 289 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev);
 290 static void eth_igbvf_interrupt_handler(void *param);
 291 static void igbvf_mbx_process(struct rte_eth_dev *dev);
 292
 293 /*
 294  * Define VF Stats MACRO for Non "cleared on read" register
 295  */
 296 #define UPDATE_VF_STAT(reg, last, cur)            \
 297 {                                                 \
 298         u32 latest = E1000_READ_REG(hw, reg);     \
 299         cur += (latest - last) & UINT_MAX;        \
 300         last = latest;                            \
 301 }
 302
 303 #define IGB_FC_PAUSE_TIME 0x0680
 304 #define IGB_LINK_UPDATE_CHECK_TIMEOUT  90  /* 9s */
 305 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */
 306
 307 #define IGBVF_PMD_NAME "rte_igbvf_pmd"     /* PMD name */
 308
 309 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full;
 310
 311 /*
 312  * The set of PCI devices this driver supports
 313  */
 314 static const struct rte_pci_id pci_id_igb_map[] = {
 315         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576) },
 316         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_FIBER) },
 317         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES) },
 318         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER) },
 319         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_QUAD_COPPER_ET2) },
 320         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS) },
 321         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_NS_SERDES) },
 322         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_SERDES_QUAD) },
 323
 324         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_COPPER) },
 325         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575EB_FIBER_SERDES) },
 326         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82575GB_QUAD_COPPER) },
 327
 328         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER) },
 329         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_FIBER) },
 330         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SERDES) },
 331         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_SGMII) },
 332         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_COPPER_DUAL) },
 333         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82580_QUAD_FIBER) },
 334
 335         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_COPPER) },
 336         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_FIBER) },
 337         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SERDES) },
 338         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_SGMII) },
 339         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_DA4) },
 340         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER) },
 341         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_OEM1) },
 342         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_COPPER_IT) },
 343         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_FIBER) },
 344         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SERDES) },
 345         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I210_SGMII) },
 346         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I211_COPPER) },
 347         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_1GBPS) },
 348         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_SGMII) },
 349         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) },
 350         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SGMII) },
 351         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SERDES) },
 352         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_BACKPLANE) },
 353         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_DH89XXCC_SFP) },
 354         { .vendor_id = 0, /* sentinel */ },
 355 };
 356
 357 /*
 358  * The set of PCI devices this driver supports (for 82576&I350 VF)
 359  */
 360 static const struct rte_pci_id pci_id_igbvf_map[] = {
 361         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF) },
 362         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_82576_VF_HV) },
 363         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF) },
 364         { RTE_PCI_DEVICE(E1000_INTEL_VENDOR_ID, E1000_DEV_ID_I350_VF_HV) },
 365         { .vendor_id = 0, /* sentinel */ },
 366 };
 367
 368 static const struct rte_eth_desc_lim rx_desc_lim = {
 369         .nb_max = E1000_MAX_RING_DESC,
 370         .nb_min = E1000_MIN_RING_DESC,
 371         .nb_align = IGB_RXD_ALIGN,
 372 };
 373
 374 static const struct rte_eth_desc_lim tx_desc_lim = {
 375         .nb_max = E1000_MAX_RING_DESC,
 376         .nb_min = E1000_MIN_RING_DESC,
 377         .nb_align = IGB_RXD_ALIGN,
 378         .nb_seg_max = IGB_TX_MAX_SEG,
 379         .nb_mtu_seg_max = IGB_TX_MAX_MTU_SEG,
 380 };
 381
 382 static const struct eth_dev_ops eth_igb_ops = {
 383         .dev_configure        = eth_igb_configure,
 384         .dev_start            = eth_igb_start,
 385         .dev_stop             = eth_igb_stop,
 386         .dev_set_link_up      = eth_igb_dev_set_link_up,
 387         .dev_set_link_down    = eth_igb_dev_set_link_down,
 388         .dev_close            = eth_igb_close,
 389         .promiscuous_enable   = eth_igb_promiscuous_enable,
 390         .promiscuous_disable  = eth_igb_promiscuous_disable,
 391         .allmulticast_enable  = eth_igb_allmulticast_enable,
 392         .allmulticast_disable = eth_igb_allmulticast_disable,
 393         .link_update          = eth_igb_link_update,
 394         .stats_get            = eth_igb_stats_get,
 395         .xstats_get           = eth_igb_xstats_get,
 396         .xstats_get_by_ids    = eth_igb_xstats_get_by_ids,
 397         .xstats_get_names_by_ids = eth_igb_xstats_get_names_by_ids,
 398         .xstats_get_names     = eth_igb_xstats_get_names,
 399         .stats_reset          = eth_igb_stats_reset,
 400         .xstats_reset         = eth_igb_xstats_reset,
 401         .fw_version_get       = eth_igb_fw_version_get,
 402         .dev_infos_get        = eth_igb_infos_get,
 403         .dev_supported_ptypes_get = eth_igb_supported_ptypes_get,
 404         .mtu_set              = eth_igb_mtu_set,
 405         .vlan_filter_set      = eth_igb_vlan_filter_set,
 406         .vlan_tpid_set        = eth_igb_vlan_tpid_set,
 407         .vlan_offload_set     = eth_igb_vlan_offload_set,
 408         .rx_queue_setup       = eth_igb_rx_queue_setup,
 409         .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable,
 410         .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable,
 411         .rx_queue_release     = eth_igb_rx_queue_release,
 412         .rx_queue_count       = eth_igb_rx_queue_count,
 413         .rx_descriptor_done   = eth_igb_rx_descriptor_done,
 414         .rx_descriptor_status = eth_igb_rx_descriptor_status,
 415         .tx_descriptor_status = eth_igb_tx_descriptor_status,
 416         .tx_queue_setup       = eth_igb_tx_queue_setup,
 417         .tx_queue_release     = eth_igb_tx_queue_release,
 418         .tx_done_cleanup      = eth_igb_tx_done_cleanup,
 419         .dev_led_on           = eth_igb_led_on,
 420         .dev_led_off          = eth_igb_led_off,
 421         .flow_ctrl_get        = eth_igb_flow_ctrl_get,
 422         .flow_ctrl_set        = eth_igb_flow_ctrl_set,
 423         .mac_addr_add         = eth_igb_rar_set,
 424         .mac_addr_remove      = eth_igb_rar_clear,
 425         .mac_addr_set         = eth_igb_default_mac_addr_set,
 426         .reta_update          = eth_igb_rss_reta_update,
 427         .reta_query           = eth_igb_rss_reta_query,
 428         .rss_hash_update      = eth_igb_rss_hash_update,
 429         .rss_hash_conf_get    = eth_igb_rss_hash_conf_get,
 430         .filter_ctrl          = eth_igb_filter_ctrl,
 431         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 432         .rxq_info_get         = igb_rxq_info_get,
 433         .txq_info_get         = igb_txq_info_get,
 434         .timesync_enable      = igb_timesync_enable,
 435         .timesync_disable     = igb_timesync_disable,
 436         .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp,
 437         .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp,
 438         .get_reg              = eth_igb_get_regs,
 439         .get_eeprom_length    = eth_igb_get_eeprom_length,
 440         .get_eeprom           = eth_igb_get_eeprom,
 441         .set_eeprom           = eth_igb_set_eeprom,
 442         .timesync_adjust_time = igb_timesync_adjust_time,
 443         .timesync_read_time   = igb_timesync_read_time,
 444         .timesync_write_time  = igb_timesync_write_time,
 445 };
 446
 447 /*
 448  * dev_ops for virtual function, bare necessities for basic vf
 449  * operation have been implemented
 450  */
 451 static const struct eth_dev_ops igbvf_eth_dev_ops = {
 452         .dev_configure        = igbvf_dev_configure,
 453         .dev_start            = igbvf_dev_start,
 454         .dev_stop             = igbvf_dev_stop,
 455         .dev_close            = igbvf_dev_close,
 456         .promiscuous_enable   = igbvf_promiscuous_enable,
 457         .promiscuous_disable  = igbvf_promiscuous_disable,
 458         .allmulticast_enable  = igbvf_allmulticast_enable,
 459         .allmulticast_disable = igbvf_allmulticast_disable,
 460         .link_update          = eth_igb_link_update,
 461         .stats_get            = eth_igbvf_stats_get,
 462         .xstats_get           = eth_igbvf_xstats_get,
 463         .xstats_get_names     = eth_igbvf_xstats_get_names,
 464         .stats_reset          = eth_igbvf_stats_reset,
 465         .xstats_reset         = eth_igbvf_stats_reset,
 466         .vlan_filter_set      = igbvf_vlan_filter_set,
 467         .dev_infos_get        = eth_igbvf_infos_get,
 468         .dev_supported_ptypes_get = eth_igb_supported_ptypes_get,
 469         .rx_queue_setup       = eth_igb_rx_queue_setup,
 470         .rx_queue_release     = eth_igb_rx_queue_release,
 471         .tx_queue_setup       = eth_igb_tx_queue_setup,
 472         .tx_queue_release     = eth_igb_tx_queue_release,
 473         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 474         .rxq_info_get         = igb_rxq_info_get,
 475         .txq_info_get         = igb_txq_info_get,
 476         .mac_addr_set         = igbvf_default_mac_addr_set,
 477         .get_reg              = igbvf_get_regs,
 478 };
 479
 480 /* store statistics names and its offset in stats structure */
 481 struct rte_igb_xstats_name_off {
 482         char name[RTE_ETH_XSTATS_NAME_SIZE];
 483         unsigned offset;
 484 };
 485
 486 static const struct rte_igb_xstats_name_off rte_igb_stats_strings[] = {
 487         {"rx_crc_errors", offsetof(struct e1000_hw_stats, crcerrs)},
 488         {"rx_align_errors", offsetof(struct e1000_hw_stats, algnerrc)},
 489         {"rx_symbol_errors", offsetof(struct e1000_hw_stats, symerrs)},
 490         {"rx_missed_packets", offsetof(struct e1000_hw_stats, mpc)},
 491         {"tx_single_collision_packets", offsetof(struct e1000_hw_stats, scc)},
 492         {"tx_multiple_collision_packets", offsetof(struct e1000_hw_stats, mcc)},
 493         {"tx_excessive_collision_packets", offsetof(struct e1000_hw_stats,
 494                 ecol)},
 495         {"tx_late_collisions", offsetof(struct e1000_hw_stats, latecol)},
 496         {"tx_total_collisions", offsetof(struct e1000_hw_stats, colc)},
 497         {"tx_deferred_packets", offsetof(struct e1000_hw_stats, dc)},
 498         {"tx_no_carrier_sense_packets", offsetof(struct e1000_hw_stats, tncrs)},
 499         {"rx_carrier_ext_errors", offsetof(struct e1000_hw_stats, cexterr)},
 500         {"rx_length_errors", offsetof(struct e1000_hw_stats, rlec)},
 501         {"rx_xon_packets", offsetof(struct e1000_hw_stats, xonrxc)},
 502         {"tx_xon_packets", offsetof(struct e1000_hw_stats, xontxc)},
 503         {"rx_xoff_packets", offsetof(struct e1000_hw_stats, xoffrxc)},
 504         {"tx_xoff_packets", offsetof(struct e1000_hw_stats, xofftxc)},
 505         {"rx_flow_control_unsupported_packets", offsetof(struct e1000_hw_stats,
 506                 fcruc)},
 507         {"rx_size_64_packets", offsetof(struct e1000_hw_stats, prc64)},
 508         {"rx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, prc127)},
 509         {"rx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, prc255)},
 510         {"rx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, prc511)},
 511         {"rx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats,
 512                 prc1023)},
 513         {"rx_size_1024_to_max_packets", offsetof(struct e1000_hw_stats,
 514                 prc1522)},
 515         {"rx_broadcast_packets", offsetof(struct e1000_hw_stats, bprc)},
 516         {"rx_multicast_packets", offsetof(struct e1000_hw_stats, mprc)},
 517         {"rx_undersize_errors", offsetof(struct e1000_hw_stats, ruc)},
 518         {"rx_fragment_errors", offsetof(struct e1000_hw_stats, rfc)},
 519         {"rx_oversize_errors", offsetof(struct e1000_hw_stats, roc)},
 520         {"rx_jabber_errors", offsetof(struct e1000_hw_stats, rjc)},
 521         {"rx_management_packets", offsetof(struct e1000_hw_stats, mgprc)},
 522         {"rx_management_dropped", offsetof(struct e1000_hw_stats, mgpdc)},
 523         {"tx_management_packets", offsetof(struct e1000_hw_stats, mgptc)},
 524         {"rx_total_packets", offsetof(struct e1000_hw_stats, tpr)},
 525         {"tx_total_packets", offsetof(struct e1000_hw_stats, tpt)},
 526         {"rx_total_bytes", offsetof(struct e1000_hw_stats, tor)},
 527         {"tx_total_bytes", offsetof(struct e1000_hw_stats, tot)},
 528         {"tx_size_64_packets", offsetof(struct e1000_hw_stats, ptc64)},
 529         {"tx_size_65_to_127_packets", offsetof(struct e1000_hw_stats, ptc127)},
 530         {"tx_size_128_to_255_packets", offsetof(struct e1000_hw_stats, ptc255)},
 531         {"tx_size_256_to_511_packets", offsetof(struct e1000_hw_stats, ptc511)},
 532         {"tx_size_512_to_1023_packets", offsetof(struct e1000_hw_stats,
 533                 ptc1023)},
 534         {"tx_size_1023_to_max_packets", offsetof(struct e1000_hw_stats,
 535                 ptc1522)},
 536         {"tx_multicast_packets", offsetof(struct e1000_hw_stats, mptc)},
 537         {"tx_broadcast_packets", offsetof(struct e1000_hw_stats, bptc)},
 538         {"tx_tso_packets", offsetof(struct e1000_hw_stats, tsctc)},
 539         {"tx_tso_errors", offsetof(struct e1000_hw_stats, tsctfc)},
 540         {"rx_sent_to_host_packets", offsetof(struct e1000_hw_stats, rpthc)},
 541         {"tx_sent_by_host_packets", offsetof(struct e1000_hw_stats, hgptc)},
 542         {"rx_code_violation_packets", offsetof(struct e1000_hw_stats, scvpc)},
 543
 544         {"interrupt_assert_count", offsetof(struct e1000_hw_stats, iac)},
 545 };
 546
 547 #define IGB_NB_XSTATS (sizeof(rte_igb_stats_strings) / \
 548                 sizeof(rte_igb_stats_strings[0]))
 549
 550 static const struct rte_igb_xstats_name_off rte_igbvf_stats_strings[] = {
 551         {"rx_multicast_packets", offsetof(struct e1000_vf_stats, mprc)},
 552         {"rx_good_loopback_packets", offsetof(struct e1000_vf_stats, gprlbc)},
 553         {"tx_good_loopback_packets", offsetof(struct e1000_vf_stats, gptlbc)},
 554         {"rx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gorlbc)},
 555         {"tx_good_loopback_bytes", offsetof(struct e1000_vf_stats, gotlbc)},
 556 };
 557
 558 #define IGBVF_NB_XSTATS (sizeof(rte_igbvf_stats_strings) / \
 559                 sizeof(rte_igbvf_stats_strings[0]))
 560
 561 /**
 562  * Atomically reads the link status information from global
 563  * structure rte_eth_dev.
 564  *
 565  * @param dev
 566  *   - Pointer to the structure rte_eth_dev to read from.
 567  *   - Pointer to the buffer to be saved with the link status.
 568  *
 569  * @return
 570  *   - On success, zero.
 571  *   - On failure, negative value.
 572  */
 573 static inline int
 574 rte_igb_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 575                                 struct rte_eth_link *link)
 576 {
 577         struct rte_eth_link *dst = link;
 578         struct rte_eth_link *src = &(dev->data->dev_link);
 579
 580         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 581                                         *(uint64_t *)src) == 0)
 582                 return -1;
 583
 584         return 0;
 585 }
 586
 587 /**
 588  * Atomically writes the link status information into global
 589  * structure rte_eth_dev.
 590  *
 591  * @param dev
 592  *   - Pointer to the structure rte_eth_dev to read from.
 593  *   - Pointer to the buffer to be saved with the link status.
 594  *
 595  * @return
 596  *   - On success, zero.
 597  *   - On failure, negative value.
 598  */
 599 static inline int
 600 rte_igb_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 601                                 struct rte_eth_link *link)
 602 {
 603         struct rte_eth_link *dst = &(dev->data->dev_link);
 604         struct rte_eth_link *src = link;
 605
 606         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 607                                         *(uint64_t *)src) == 0)
 608                 return -1;
 609
 610         return 0;
 611 }
 612
 613 static inline void
 614 igb_intr_enable(struct rte_eth_dev *dev)
 615 {
 616         struct e1000_interrupt *intr =
 617                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 618         struct e1000_hw *hw =
 619                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 620
 621         E1000_WRITE_REG(hw, E1000_IMS, intr->mask);
 622         E1000_WRITE_FLUSH(hw);
 623 }
 624
 625 static void
 626 igb_intr_disable(struct e1000_hw *hw)
 627 {
 628         E1000_WRITE_REG(hw, E1000_IMC, ~0);
 629         E1000_WRITE_FLUSH(hw);
 630 }
 631
 632 static inline void
 633 igbvf_intr_enable(struct rte_eth_dev *dev)
 634 {
 635         struct e1000_hw *hw =
 636                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 637
 638         /* only for mailbox */
 639         E1000_WRITE_REG(hw, E1000_EIAM, 1 << E1000_VTIVAR_MISC_MAILBOX);
 640         E1000_WRITE_REG(hw, E1000_EIAC, 1 << E1000_VTIVAR_MISC_MAILBOX);
 641         E1000_WRITE_REG(hw, E1000_EIMS, 1 << E1000_VTIVAR_MISC_MAILBOX);
 642         E1000_WRITE_FLUSH(hw);
 643 }
 644
 645 /* only for mailbox now. If RX/TX needed, should extend this function.  */
 646 static void
 647 igbvf_set_ivar_map(struct e1000_hw *hw, uint8_t msix_vector)
 648 {
 649         uint32_t tmp = 0;
 650
 651         /* mailbox */
 652         tmp |= (msix_vector & E1000_VTIVAR_MISC_INTR_MASK);
 653         tmp |= E1000_VTIVAR_VALID;
 654         E1000_WRITE_REG(hw, E1000_VTIVAR_MISC, tmp);
 655 }
 656
 657 static void
 658 eth_igbvf_configure_msix_intr(struct rte_eth_dev *dev)
 659 {
 660         struct e1000_hw *hw =
 661                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 662
 663         /* Configure VF other cause ivar */
 664         igbvf_set_ivar_map(hw, E1000_VTIVAR_MISC_MAILBOX);
 665 }
 666
 667 static inline int32_t
 668 igb_pf_reset_hw(struct e1000_hw *hw)
 669 {
 670         uint32_t ctrl_ext;
 671         int32_t status;
 672
 673         status = e1000_reset_hw(hw);
 674
 675         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 676         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 677         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 678         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 679         E1000_WRITE_FLUSH(hw);
 680
 681         return status;
 682 }
 683
 684 static void
 685 igb_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev)
 686 {
 687         struct e1000_hw *hw =
 688                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 689
 690
 691         hw->vendor_id = pci_dev->id.vendor_id;
 692         hw->device_id = pci_dev->id.device_id;
 693         hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
 694         hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
 695
 696         e1000_set_mac_type(hw);
 697
 698         /* need to check if it is a vf device below */
 699 }
 700
 701 static int
 702 igb_reset_swfw_lock(struct e1000_hw *hw)
 703 {
 704         int ret_val;
 705
 706         /*
 707          * Do mac ops initialization manually here, since we will need
 708          * some function pointers set by this call.
 709          */
 710         ret_val = e1000_init_mac_params(hw);
 711         if (ret_val)
 712                 return ret_val;
 713
 714         /*
 715          * SMBI lock should not fail in this early stage. If this is the case,
 716          * it is due to an improper exit of the application.
 717          * So force the release of the faulty lock.
 718          */
 719         if (e1000_get_hw_semaphore_generic(hw) < 0) {
 720                 PMD_DRV_LOG(DEBUG, "SMBI lock released");
 721         }
 722         e1000_put_hw_semaphore_generic(hw);
 723
 724         if (hw->mac.ops.acquire_swfw_sync != NULL) {
 725                 uint16_t mask;
 726
 727                 /*
 728                  * Phy lock should not fail in this early stage. If this is the case,
 729                  * it is due to an improper exit of the application.
 730                  * So force the release of the faulty lock.
 731                  */
 732                 mask = E1000_SWFW_PHY0_SM << hw->bus.func;
 733                 if (hw->bus.func > E1000_FUNC_1)
 734                         mask <<= 2;
 735                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 736                         PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released",
 737                                     hw->bus.func);
 738                 }
 739                 hw->mac.ops.release_swfw_sync(hw, mask);
 740
 741                 /*
 742                  * This one is more tricky since it is common to all ports; but
 743                  * swfw_sync retries last long enough (1s) to be almost sure that if
 744                  * lock can not be taken it is due to an improper lock of the
 745                  * semaphore.
 746                  */
 747                 mask = E1000_SWFW_EEP_SM;
 748                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 749                         PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 750                 }
 751                 hw->mac.ops.release_swfw_sync(hw, mask);
 752         }
 753
 754         return E1000_SUCCESS;
 755 }
 756
 757 static int
 758 eth_igb_dev_init(struct rte_eth_dev *eth_dev)
 759 {
 760         int error = 0;
 761         struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(eth_dev);
 762         struct e1000_hw *hw =
 763                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 764         struct e1000_vfta * shadow_vfta =
 765                 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 766         struct e1000_filter_info *filter_info =
 767                 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
 768         struct e1000_adapter *adapter =
 769                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 770
 771         uint32_t ctrl_ext;
 772
 773         eth_dev->dev_ops = &eth_igb_ops;
 774         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 775         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 776         eth_dev->tx_pkt_prepare = &eth_igb_prep_pkts;
 777
 778         /* for secondary processes, we don't initialise any further as primary
 779          * has already done this work. Only check we don't need a different
 780          * RX function */
 781         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 782                 if (eth_dev->data->scattered_rx)
 783                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 784                 return 0;
 785         }
 786
 787         rte_eth_copy_pci_info(eth_dev, pci_dev);
 788         eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE;
 789
 790         hw->hw_addr= (void *)pci_dev->mem_resource[0].addr;
 791
 792         igb_identify_hardware(eth_dev, pci_dev);
 793         if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) {
 794                 error = -EIO;
 795                 goto err_late;
 796         }
 797
 798         e1000_get_bus_info(hw);
 799
 800         /* Reset any pending lock */
 801         if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) {
 802                 error = -EIO;
 803                 goto err_late;
 804         }
 805
 806         /* Finish initialization */
 807         if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) {
 808                 error = -EIO;
 809                 goto err_late;
 810         }
 811
 812         hw->mac.autoneg = 1;
 813         hw->phy.autoneg_wait_to_complete = 0;
 814         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 815
 816         /* Copper options */
 817         if (hw->phy.media_type == e1000_media_type_copper) {
 818                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
 819                 hw->phy.disable_polarity_correction = 0;
 820                 hw->phy.ms_type = e1000_ms_hw_default;
 821         }
 822
 823         /*
 824          * Start from a known state, this is important in reading the nvm
 825          * and mac from that.
 826          */
 827         igb_pf_reset_hw(hw);
 828
 829         /* Make sure we have a good EEPROM before we read from it */
 830         if (e1000_validate_nvm_checksum(hw) < 0) {
 831                 /*
 832                  * Some PCI-E parts fail the first check due to
 833                  * the link being in sleep state, call it again,
 834                  * if it fails a second time its a real issue.
 835                  */
 836                 if (e1000_validate_nvm_checksum(hw) < 0) {
 837                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
 838                         error = -EIO;
 839                         goto err_late;
 840                 }
 841         }
 842
 843         /* Read the permanent MAC address out of the EEPROM */
 844         if (e1000_read_mac_addr(hw) != 0) {
 845                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
 846                 error = -EIO;
 847                 goto err_late;
 848         }
 849
 850         /* Allocate memory for storing MAC addresses */
 851         eth_dev->data->mac_addrs = rte_zmalloc("e1000",
 852                 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
 853         if (eth_dev->data->mac_addrs == NULL) {
 854                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to "
 855                                                 "store MAC addresses",
 856                                 ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 857                 error = -ENOMEM;
 858                 goto err_late;
 859         }
 860
 861         /* Copy the permanent MAC address */
 862         ether_addr_copy((struct ether_addr *)hw->mac.addr, &eth_dev->data->mac_addrs[0]);
 863
 864         /* initialize the vfta */
 865         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 866
 867         /* Now initialize the hardware */
 868         if (igb_hardware_init(hw) != 0) {
 869                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
 870                 rte_free(eth_dev->data->mac_addrs);
 871                 eth_dev->data->mac_addrs = NULL;
 872                 error = -ENODEV;
 873                 goto err_late;
 874         }
 875         hw->mac.get_link_status = 1;
 876         adapter->stopped = 0;
 877
 878         /* Indicate SOL/IDER usage */
 879         if (e1000_check_reset_block(hw) < 0) {
 880                 PMD_INIT_LOG(ERR, "PHY reset is blocked due to"
 881                                         "SOL/IDER session");
 882         }
 883
 884         /* initialize PF if max_vfs not zero */
 885         igb_pf_host_init(eth_dev);
 886
 887         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 888         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 889         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 890         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 891         E1000_WRITE_FLUSH(hw);
 892
 893         PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x",
 894                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 895                      pci_dev->id.device_id);
 896
 897         rte_intr_callback_register(&pci_dev->intr_handle,
 898                                    eth_igb_interrupt_handler,
 899                                    (void *)eth_dev);
 900
 901         /* enable uio/vfio intr/eventfd mapping */
 902         rte_intr_enable(&pci_dev->intr_handle);
 903
 904         /* enable support intr */
 905         igb_intr_enable(eth_dev);
 906
 907         TAILQ_INIT(&filter_info->flex_list);
 908         filter_info->flex_mask = 0;
 909         TAILQ_INIT(&filter_info->twotuple_list);
 910         filter_info->twotuple_mask = 0;
 911         TAILQ_INIT(&filter_info->fivetuple_list);
 912         filter_info->fivetuple_mask = 0;
 913
 914         return 0;
 915
 916 err_late:
 917         igb_hw_control_release(hw);
 918
 919         return error;
 920 }
 921
 922 static int
 923 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev)
 924 {
 925         struct rte_pci_device *pci_dev;
 926         struct rte_intr_handle *intr_handle;
 927         struct e1000_hw *hw;
 928         struct e1000_adapter *adapter =
 929                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 930
 931         PMD_INIT_FUNC_TRACE();
 932
 933         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 934                 return -EPERM;
 935
 936         hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 937         pci_dev = E1000_DEV_TO_PCI(eth_dev);
 938         intr_handle = &pci_dev->intr_handle;
 939
 940         if (adapter->stopped == 0)
 941                 eth_igb_close(eth_dev);
 942
 943         eth_dev->dev_ops = NULL;
 944         eth_dev->rx_pkt_burst = NULL;
 945         eth_dev->tx_pkt_burst = NULL;
 946
 947         /* Reset any pending lock */
 948         igb_reset_swfw_lock(hw);
 949
 950         rte_free(eth_dev->data->mac_addrs);
 951         eth_dev->data->mac_addrs = NULL;
 952
 953         /* uninitialize PF if max_vfs not zero */
 954         igb_pf_host_uninit(eth_dev);
 955
 956         /* disable uio intr before callback unregister */
 957         rte_intr_disable(intr_handle);
 958         rte_intr_callback_unregister(intr_handle,
 959                                      eth_igb_interrupt_handler, eth_dev);
 960
 961         return 0;
 962 }
 963
 964 /*
 965  * Virtual Function device init
 966  */
 967 static int
 968 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev)
 969 {
 970         struct rte_pci_device *pci_dev;
 971         struct rte_intr_handle *intr_handle;
 972         struct e1000_adapter *adapter =
 973                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 974         struct e1000_hw *hw =
 975                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 976         int diag;
 977         struct ether_addr *perm_addr = (struct ether_addr *)hw->mac.perm_addr;
 978
 979         PMD_INIT_FUNC_TRACE();
 980
 981         eth_dev->dev_ops = &igbvf_eth_dev_ops;
 982         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 983         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 984         eth_dev->tx_pkt_prepare = &eth_igb_prep_pkts;
 985
 986         /* for secondary processes, we don't initialise any further as primary
 987          * has already done this work. Only check we don't need a different
 988          * RX function */
 989         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 990                 if (eth_dev->data->scattered_rx)
 991                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 992                 return 0;
 993         }
 994
 995         pci_dev = E1000_DEV_TO_PCI(eth_dev);
 996         rte_eth_copy_pci_info(eth_dev, pci_dev);
 997         eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE;
 998
 999         hw->device_id = pci_dev->id.device_id;
1000         hw->vendor_id = pci_dev->id.vendor_id;
1001         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1002         adapter->stopped = 0;
1003
1004         /* Initialize the shared code (base driver) */
1005         diag = e1000_setup_init_funcs(hw, TRUE);
1006         if (diag != 0) {
1007                 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d",
1008                         diag);
1009                 return -EIO;
1010         }
1011
1012         /* init_mailbox_params */
1013         hw->mbx.ops.init_params(hw);
1014
1015         /* Disable the interrupts for VF */
1016         igbvf_intr_disable(hw);
1017
1018         diag = hw->mac.ops.reset_hw(hw);
1019
1020         /* Allocate memory for storing MAC addresses */
1021         eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN *
1022                 hw->mac.rar_entry_count, 0);
1023         if (eth_dev->data->mac_addrs == NULL) {
1024                 PMD_INIT_LOG(ERR,
1025                         "Failed to allocate %d bytes needed to store MAC "
1026                         "addresses",
1027                         ETHER_ADDR_LEN * hw->mac.rar_entry_count);
1028                 return -ENOMEM;
1029         }
1030
1031         /* Generate a random MAC address, if none was assigned by PF. */
1032         if (is_zero_ether_addr(perm_addr)) {
1033                 eth_random_addr(perm_addr->addr_bytes);
1034                 diag = e1000_rar_set(hw, perm_addr->addr_bytes, 0);
1035                 if (diag) {
1036                         rte_free(eth_dev->data->mac_addrs);
1037                         eth_dev->data->mac_addrs = NULL;
1038                         return diag;
1039                 }
1040                 PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF");
1041                 PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address "
1042                              "%02x:%02x:%02x:%02x:%02x:%02x",
1043                              perm_addr->addr_bytes[0],
1044                              perm_addr->addr_bytes[1],
1045                              perm_addr->addr_bytes[2],
1046                              perm_addr->addr_bytes[3],
1047                              perm_addr->addr_bytes[4],
1048                              perm_addr->addr_bytes[5]);
1049         }
1050
1051         /* Copy the permanent MAC address */
1052         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
1053                         &eth_dev->data->mac_addrs[0]);
1054
1055         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x "
1056                      "mac.type=%s",
1057                      eth_dev->data->port_id, pci_dev->id.vendor_id,
1058                      pci_dev->id.device_id, "igb_mac_82576_vf");
1059
1060         intr_handle = &pci_dev->intr_handle;
1061         rte_intr_callback_register(intr_handle,
1062                                    eth_igbvf_interrupt_handler, eth_dev);
1063
1064         return 0;
1065 }
1066
1067 static int
1068 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev)
1069 {
1070         struct e1000_adapter *adapter =
1071                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
1072         struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(eth_dev);
1073
1074         PMD_INIT_FUNC_TRACE();
1075
1076         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1077                 return -EPERM;
1078
1079         if (adapter->stopped == 0)
1080                 igbvf_dev_close(eth_dev);
1081
1082         eth_dev->dev_ops = NULL;
1083         eth_dev->rx_pkt_burst = NULL;
1084         eth_dev->tx_pkt_burst = NULL;
1085
1086         rte_free(eth_dev->data->mac_addrs);
1087         eth_dev->data->mac_addrs = NULL;
1088
1089         /* disable uio intr before callback unregister */
1090         rte_intr_disable(&pci_dev->intr_handle);
1091         rte_intr_callback_unregister(&pci_dev->intr_handle,
1092                                      eth_igbvf_interrupt_handler,
1093                                      (void *)eth_dev);
1094
1095         return 0;
1096 }
1097
1098 static int eth_igb_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1099         struct rte_pci_device *pci_dev)
1100 {
1101         return rte_eth_dev_pci_generic_probe(pci_dev,
1102                 sizeof(struct e1000_adapter), eth_igb_dev_init);
1103 }
1104
1105 static int eth_igb_pci_remove(struct rte_pci_device *pci_dev)
1106 {
1107         return rte_eth_dev_pci_generic_remove(pci_dev, eth_igb_dev_uninit);
1108 }
1109
1110 static struct rte_pci_driver rte_igb_pmd = {
1111         .id_table = pci_id_igb_map,
1112         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
1113         .probe = eth_igb_pci_probe,
1114         .remove = eth_igb_pci_remove,
1115 };
1116
1117
1118 static int eth_igbvf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1119         struct rte_pci_device *pci_dev)
1120 {
1121         return rte_eth_dev_pci_generic_probe(pci_dev,
1122                 sizeof(struct e1000_adapter), eth_igbvf_dev_init);
1123 }
1124
1125 static int eth_igbvf_pci_remove(struct rte_pci_device *pci_dev)
1126 {
1127         return rte_eth_dev_pci_generic_remove(pci_dev, eth_igbvf_dev_uninit);
1128 }
1129
1130 /*
1131  * virtual function driver struct
1132  */
1133 static struct rte_pci_driver rte_igbvf_pmd = {
1134         .id_table = pci_id_igbvf_map,
1135         .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
1136         .probe = eth_igbvf_pci_probe,
1137         .remove = eth_igbvf_pci_remove,
1138 };
1139
1140 static void
1141 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1142 {
1143         struct e1000_hw *hw =
1144                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1145         /* RCTL: enable VLAN filter since VMDq always use VLAN filter */
1146         uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL);
1147         rctl |= E1000_RCTL_VFE;
1148         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1149 }
1150
1151 static int
1152 igb_check_mq_mode(struct rte_eth_dev *dev)
1153 {
1154         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
1155         enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode;
1156         uint16_t nb_rx_q = dev->data->nb_rx_queues;
1157         uint16_t nb_tx_q = dev->data->nb_rx_queues;
1158
1159         if ((rx_mq_mode & ETH_MQ_RX_DCB_FLAG) ||
1160             tx_mq_mode == ETH_MQ_TX_DCB ||
1161             tx_mq_mode == ETH_MQ_TX_VMDQ_DCB) {
1162                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
1163                 return -EINVAL;
1164         }
1165         if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
1166                 /* Check multi-queue mode.
1167                  * To no break software we accept ETH_MQ_RX_NONE as this might
1168                  * be used to turn off VLAN filter.
1169                  */
1170
1171                 if (rx_mq_mode == ETH_MQ_RX_NONE ||
1172                     rx_mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1173                         dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY;
1174                         RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1;
1175                 } else {
1176                         /* Only support one queue on VFs.
1177                          * RSS together with SRIOV is not supported.
1178                          */
1179                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1180                                         " wrong mq_mode rx %d.",
1181                                         rx_mq_mode);
1182                         return -EINVAL;
1183                 }
1184                 /* TX mode is not used here, so mode might be ignored.*/
1185                 if (tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) {
1186                         /* SRIOV only works in VMDq enable mode */
1187                         PMD_INIT_LOG(WARNING, "SRIOV is active,"
1188                                         " TX mode %d is not supported. "
1189                                         " Driver will behave as %d mode.",
1190                                         tx_mq_mode, ETH_MQ_TX_VMDQ_ONLY);
1191                 }
1192
1193                 /* check valid queue number */
1194                 if ((nb_rx_q > 1) || (nb_tx_q > 1)) {
1195                         PMD_INIT_LOG(ERR, "SRIOV is active,"
1196                                         " only support one queue on VFs.");
1197                         return -EINVAL;
1198                 }
1199         } else {
1200                 /* To no break software that set invalid mode, only display
1201                  * warning if invalid mode is used.
1202                  */
1203                 if (rx_mq_mode != ETH_MQ_RX_NONE &&
1204                     rx_mq_mode != ETH_MQ_RX_VMDQ_ONLY &&
1205                     rx_mq_mode != ETH_MQ_RX_RSS) {
1206                         /* RSS together with VMDq not supported*/
1207                         PMD_INIT_LOG(ERR, "RX mode %d is not supported.",
1208                                      rx_mq_mode);
1209                         return -EINVAL;
1210                 }
1211
1212                 if (tx_mq_mode != ETH_MQ_TX_NONE &&
1213                     tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) {
1214                         PMD_INIT_LOG(WARNING, "TX mode %d is not supported."
1215                                         " Due to txmode is meaningless in this"
1216                                         " driver, just ignore.",
1217                                         tx_mq_mode);
1218                 }
1219         }
1220         return 0;
1221 }
1222
1223 static int
1224 eth_igb_configure(struct rte_eth_dev *dev)
1225 {
1226         struct e1000_interrupt *intr =
1227                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1228         int ret;
1229
1230         PMD_INIT_FUNC_TRACE();
1231
1232         /* multipe queue mode checking */
1233         ret  = igb_check_mq_mode(dev);
1234         if (ret != 0) {
1235                 PMD_DRV_LOG(ERR, "igb_check_mq_mode fails with %d.",
1236                             ret);
1237                 return ret;
1238         }
1239
1240         intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
1241         PMD_INIT_FUNC_TRACE();
1242
1243         return 0;
1244 }
1245
1246 static int
1247 eth_igb_start(struct rte_eth_dev *dev)
1248 {
1249         struct e1000_hw *hw =
1250                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1251         struct e1000_adapter *adapter =
1252                 E1000_DEV_PRIVATE(dev->data->dev_private);
1253         struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev);
1254         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1255         int ret, mask;
1256         uint32_t intr_vector = 0;
1257         uint32_t ctrl_ext;
1258         uint32_t *speeds;
1259         int num_speeds;
1260         bool autoneg;
1261
1262         PMD_INIT_FUNC_TRACE();
1263
1264         /* disable uio/vfio intr/eventfd mapping */
1265         rte_intr_disable(intr_handle);
1266
1267         /* Power up the phy. Needed to make the link go Up */
1268         eth_igb_dev_set_link_up(dev);
1269
1270         /*
1271          * Packet Buffer Allocation (PBA)
1272          * Writing PBA sets the receive portion of the buffer
1273          * the remainder is used for the transmit buffer.
1274          */
1275         if (hw->mac.type == e1000_82575) {
1276                 uint32_t pba;
1277
1278                 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
1279                 E1000_WRITE_REG(hw, E1000_PBA, pba);
1280         }
1281
1282         /* Put the address into the Receive Address Array */
1283         e1000_rar_set(hw, hw->mac.addr, 0);
1284
1285         /* Initialize the hardware */
1286         if (igb_hardware_init(hw)) {
1287                 PMD_INIT_LOG(ERR, "Unable to initialize the hardware");
1288                 return -EIO;
1289         }
1290         adapter->stopped = 0;
1291
1292         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1293
1294         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1295         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
1296         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
1297         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
1298         E1000_WRITE_FLUSH(hw);
1299
1300         /* configure PF module if SRIOV enabled */
1301         igb_pf_host_configure(dev);
1302
1303         /* check and configure queue intr-vector mapping */
1304         if ((rte_intr_cap_multiple(intr_handle) ||
1305              !RTE_ETH_DEV_SRIOV(dev).active) &&
1306             dev->data->dev_conf.intr_conf.rxq != 0) {
1307                 intr_vector = dev->data->nb_rx_queues;
1308                 if (rte_intr_efd_enable(intr_handle, intr_vector))
1309                         return -1;
1310         }
1311
1312         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
1313                 intr_handle->intr_vec =
1314                         rte_zmalloc("intr_vec",
1315                                     dev->data->nb_rx_queues * sizeof(int), 0);
1316                 if (intr_handle->intr_vec == NULL) {
1317                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
1318                                      " intr_vec", dev->data->nb_rx_queues);
1319                         return -ENOMEM;
1320                 }
1321         }
1322
1323         /* confiugre msix for rx interrupt */
1324         eth_igb_configure_msix_intr(dev);
1325
1326         /* Configure for OS presence */
1327         igb_init_manageability(hw);
1328
1329         eth_igb_tx_init(dev);
1330
1331         /* This can fail when allocating mbufs for descriptor rings */
1332         ret = eth_igb_rx_init(dev);
1333         if (ret) {
1334                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
1335                 igb_dev_clear_queues(dev);
1336                 return ret;
1337         }
1338
1339         e1000_clear_hw_cntrs_base_generic(hw);
1340
1341         /*
1342          * VLAN Offload Settings
1343          */
1344         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1345                         ETH_VLAN_EXTEND_MASK;
1346         eth_igb_vlan_offload_set(dev, mask);
1347
1348         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1349                 /* Enable VLAN filter since VMDq always use VLAN filter */
1350                 igb_vmdq_vlan_hw_filter_enable(dev);
1351         }
1352
1353         if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) ||
1354                 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) ||
1355                 (hw->mac.type == e1000_i211)) {
1356                 /* Configure EITR with the maximum possible value (0xFFFF) */
1357                 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF);
1358         }
1359
1360         /* Setup link speed and duplex */
1361         speeds = &dev->data->dev_conf.link_speeds;
1362         if (*speeds == ETH_LINK_SPEED_AUTONEG) {
1363                 hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
1364                 hw->mac.autoneg = 1;
1365         } else {
1366                 num_speeds = 0;
1367                 autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0;
1368
1369                 /* Reset */
1370                 hw->phy.autoneg_advertised = 0;
1371
1372                 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M |
1373                                 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M |
1374                                 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_FIXED)) {
1375                         num_speeds = -1;
1376                         goto error_invalid_config;
1377                 }
1378                 if (*speeds & ETH_LINK_SPEED_10M_HD) {
1379                         hw->phy.autoneg_advertised |= ADVERTISE_10_HALF;
1380                         num_speeds++;
1381                 }
1382                 if (*speeds & ETH_LINK_SPEED_10M) {
1383                         hw->phy.autoneg_advertised |= ADVERTISE_10_FULL;
1384                         num_speeds++;
1385                 }
1386                 if (*speeds & ETH_LINK_SPEED_100M_HD) {
1387                         hw->phy.autoneg_advertised |= ADVERTISE_100_HALF;
1388                         num_speeds++;
1389                 }
1390                 if (*speeds & ETH_LINK_SPEED_100M) {
1391                         hw->phy.autoneg_advertised |= ADVERTISE_100_FULL;
1392                         num_speeds++;
1393                 }
1394                 if (*speeds & ETH_LINK_SPEED_1G) {
1395                         hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL;
1396                         num_speeds++;
1397                 }
1398                 if (num_speeds == 0 || (!autoneg && (num_speeds > 1)))
1399                         goto error_invalid_config;
1400
1401                 /* Set/reset the mac.autoneg based on the link speed,
1402                  * fixed or not
1403                  */
1404                 if (!autoneg) {
1405                         hw->mac.autoneg = 0;
1406                         hw->mac.forced_speed_duplex =
1407                                         hw->phy.autoneg_advertised;
1408                 } else {
1409                         hw->mac.autoneg = 1;
1410                 }
1411         }
1412
1413         e1000_setup_link(hw);
1414
1415         if (rte_intr_allow_others(intr_handle)) {
1416                 /* check if lsc interrupt is enabled */
1417                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1418                         eth_igb_lsc_interrupt_setup(dev);
1419         } else {
1420                 rte_intr_callback_unregister(intr_handle,
1421                                              eth_igb_interrupt_handler,
1422                                              (void *)dev);
1423                 if (dev->data->dev_conf.intr_conf.lsc != 0)
1424                         PMD_INIT_LOG(INFO, "lsc won't enable because of"
1425                                      " no intr multiplex");
1426         }
1427
1428         /* check if rxq interrupt is enabled */
1429         if (dev->data->dev_conf.intr_conf.rxq != 0 &&
1430             rte_intr_dp_is_en(intr_handle))
1431                 eth_igb_rxq_interrupt_setup(dev);
1432
1433         /* enable uio/vfio intr/eventfd mapping */
1434         rte_intr_enable(intr_handle);
1435
1436         /* resume enabled intr since hw reset */
1437         igb_intr_enable(dev);
1438
1439         PMD_INIT_LOG(DEBUG, "<<");
1440
1441         return 0;
1442
1443 error_invalid_config:
1444         PMD_INIT_LOG(ERR, "Invalid advertised speeds (%u) for port %u",
1445                      dev->data->dev_conf.link_speeds, dev->data->port_id);
1446         igb_dev_clear_queues(dev);
1447         return -EINVAL;
1448 }
1449
1450 /*********************************************************************
1451  *
1452  *  This routine disables all traffic on the adapter by issuing a
1453  *  global reset on the MAC.
1454  *
1455  **********************************************************************/
1456 static void
1457 eth_igb_stop(struct rte_eth_dev *dev)
1458 {
1459         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1460         struct e1000_filter_info *filter_info =
1461                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
1462         struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev);
1463         struct rte_eth_link link;
1464         struct e1000_flex_filter *p_flex;
1465         struct e1000_5tuple_filter *p_5tuple, *p_5tuple_next;
1466         struct e1000_2tuple_filter *p_2tuple, *p_2tuple_next;
1467         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1468
1469         igb_intr_disable(hw);
1470
1471         /* disable intr eventfd mapping */
1472         rte_intr_disable(intr_handle);
1473
1474         igb_pf_reset_hw(hw);
1475         E1000_WRITE_REG(hw, E1000_WUC, 0);
1476
1477         /* Set bit for Go Link disconnect */
1478         if (hw->mac.type >= e1000_82580) {
1479                 uint32_t phpm_reg;
1480
1481                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1482                 phpm_reg |= E1000_82580_PM_GO_LINKD;
1483                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1484         }
1485
1486         /* Power down the phy. Needed to make the link go Down */
1487         eth_igb_dev_set_link_down(dev);
1488
1489         igb_dev_clear_queues(dev);
1490
1491         /* clear the recorded link status */
1492         memset(&link, 0, sizeof(link));
1493         rte_igb_dev_atomic_write_link_status(dev, &link);
1494
1495         /* Remove all flex filters of the device */
1496         while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) {
1497                 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries);
1498                 rte_free(p_flex);
1499         }
1500         filter_info->flex_mask = 0;
1501
1502         /* Remove all ntuple filters of the device */
1503         for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list);
1504              p_5tuple != NULL; p_5tuple = p_5tuple_next) {
1505                 p_5tuple_next = TAILQ_NEXT(p_5tuple, entries);
1506                 TAILQ_REMOVE(&filter_info->fivetuple_list,
1507                              p_5tuple, entries);
1508                 rte_free(p_5tuple);
1509         }
1510         filter_info->fivetuple_mask = 0;
1511         for (p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list);
1512              p_2tuple != NULL; p_2tuple = p_2tuple_next) {
1513                 p_2tuple_next = TAILQ_NEXT(p_2tuple, entries);
1514                 TAILQ_REMOVE(&filter_info->twotuple_list,
1515                              p_2tuple, entries);
1516                 rte_free(p_2tuple);
1517         }
1518         filter_info->twotuple_mask = 0;
1519
1520         if (!rte_intr_allow_others(intr_handle))
1521                 /* resume to the default handler */
1522                 rte_intr_callback_register(intr_handle,
1523                                            eth_igb_interrupt_handler,
1524                                            (void *)dev);
1525
1526         /* Clean datapath event and queue/vec mapping */
1527         rte_intr_efd_disable(intr_handle);
1528         if (intr_handle->intr_vec != NULL) {
1529                 rte_free(intr_handle->intr_vec);
1530                 intr_handle->intr_vec = NULL;
1531         }
1532 }
1533
1534 static int
1535 eth_igb_dev_set_link_up(struct rte_eth_dev *dev)
1536 {
1537         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1538
1539         if (hw->phy.media_type == e1000_media_type_copper)
1540                 e1000_power_up_phy(hw);
1541         else
1542                 e1000_power_up_fiber_serdes_link(hw);
1543
1544         return 0;
1545 }
1546
1547 static int
1548 eth_igb_dev_set_link_down(struct rte_eth_dev *dev)
1549 {
1550         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1551
1552         if (hw->phy.media_type == e1000_media_type_copper)
1553                 e1000_power_down_phy(hw);
1554         else
1555                 e1000_shutdown_fiber_serdes_link(hw);
1556
1557         return 0;
1558 }
1559
1560 static void
1561 eth_igb_close(struct rte_eth_dev *dev)
1562 {
1563         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1564         struct e1000_adapter *adapter =
1565                 E1000_DEV_PRIVATE(dev->data->dev_private);
1566         struct rte_eth_link link;
1567         struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev);
1568         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1569
1570         eth_igb_stop(dev);
1571         adapter->stopped = 1;
1572
1573         e1000_phy_hw_reset(hw);
1574         igb_release_manageability(hw);
1575         igb_hw_control_release(hw);
1576
1577         /* Clear bit for Go Link disconnect */
1578         if (hw->mac.type >= e1000_82580) {
1579                 uint32_t phpm_reg;
1580
1581                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1582                 phpm_reg &= ~E1000_82580_PM_GO_LINKD;
1583                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1584         }
1585
1586         igb_dev_free_queues(dev);
1587
1588         if (intr_handle->intr_vec) {
1589                 rte_free(intr_handle->intr_vec);
1590                 intr_handle->intr_vec = NULL;
1591         }
1592
1593         memset(&link, 0, sizeof(link));
1594         rte_igb_dev_atomic_write_link_status(dev, &link);
1595 }
1596
1597 static int
1598 igb_get_rx_buffer_size(struct e1000_hw *hw)
1599 {
1600         uint32_t rx_buf_size;
1601         if (hw->mac.type == e1000_82576) {
1602                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10;
1603         } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) {
1604                 /* PBS needs to be translated according to a lookup table */
1605                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf);
1606                 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size);
1607                 rx_buf_size = (rx_buf_size << 10);
1608         } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) {
1609                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10;
1610         } else {
1611                 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10;
1612         }
1613
1614         return rx_buf_size;
1615 }
1616
1617 /*********************************************************************
1618  *
1619  *  Initialize the hardware
1620  *
1621  **********************************************************************/
1622 static int
1623 igb_hardware_init(struct e1000_hw *hw)
1624 {
1625         uint32_t rx_buf_size;
1626         int diag;
1627
1628         /* Let the firmware know the OS is in control */
1629         igb_hw_control_acquire(hw);
1630
1631         /*
1632          * These parameters control the automatic generation (Tx) and
1633          * response (Rx) to Ethernet PAUSE frames.
1634          * - High water mark should allow for at least two standard size (1518)
1635          *   frames to be received after sending an XOFF.
1636          * - Low water mark works best when it is very near the high water mark.
1637          *   This allows the receiver to restart by sending XON when it has
1638          *   drained a bit. Here we use an arbitrary value of 1500 which will
1639          *   restart after one full frame is pulled from the buffer. There
1640          *   could be several smaller frames in the buffer and if so they will
1641          *   not trigger the XON until their total number reduces the buffer
1642          *   by 1500.
1643          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
1644          */
1645         rx_buf_size = igb_get_rx_buffer_size(hw);
1646
1647         hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2);
1648         hw->fc.low_water = hw->fc.high_water - 1500;
1649         hw->fc.pause_time = IGB_FC_PAUSE_TIME;
1650         hw->fc.send_xon = 1;
1651
1652         /* Set Flow control, use the tunable location if sane */
1653         if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4))
1654                 hw->fc.requested_mode = igb_fc_setting;
1655         else
1656                 hw->fc.requested_mode = e1000_fc_none;
1657
1658         /* Issue a global reset */
1659         igb_pf_reset_hw(hw);
1660         E1000_WRITE_REG(hw, E1000_WUC, 0);
1661
1662         diag = e1000_init_hw(hw);
1663         if (diag < 0)
1664                 return diag;
1665
1666         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1667         e1000_get_phy_info(hw);
1668         e1000_check_for_link(hw);
1669
1670         return 0;
1671 }
1672
1673 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */
1674 static void
1675 igb_read_stats_registers(struct e1000_hw *hw, struct e1000_hw_stats *stats)
1676 {
1677         int pause_frames;
1678
1679         uint64_t old_gprc  = stats->gprc;
1680         uint64_t old_gptc  = stats->gptc;
1681         uint64_t old_tpr   = stats->tpr;
1682         uint64_t old_tpt   = stats->tpt;
1683         uint64_t old_rpthc = stats->rpthc;
1684         uint64_t old_hgptc = stats->hgptc;
1685
1686         if(hw->phy.media_type == e1000_media_type_copper ||
1687             (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
1688                 stats->symerrs +=
1689                     E1000_READ_REG(hw,E1000_SYMERRS);
1690                 stats->sec += E1000_READ_REG(hw, E1000_SEC);
1691         }
1692
1693         stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
1694         stats->mpc += E1000_READ_REG(hw, E1000_MPC);
1695         stats->scc += E1000_READ_REG(hw, E1000_SCC);
1696         stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
1697
1698         stats->mcc += E1000_READ_REG(hw, E1000_MCC);
1699         stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
1700         stats->colc += E1000_READ_REG(hw, E1000_COLC);
1701         stats->dc += E1000_READ_REG(hw, E1000_DC);
1702         stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
1703         stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
1704         stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
1705         /*
1706         ** For watchdog management we need to know if we have been
1707         ** paused during the last interval, so capture that here.
1708         */
1709         pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC);
1710         stats->xoffrxc += pause_frames;
1711         stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
1712         stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
1713         stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
1714         stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
1715         stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
1716         stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
1717         stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
1718         stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
1719         stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
1720         stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
1721         stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
1722         stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
1723
1724         /* For the 64-bit byte counters the low dword must be read first. */
1725         /* Both registers clear on the read of the high dword */
1726
1727         /* Workaround CRC bytes included in size, take away 4 bytes/packet */
1728         stats->gorc += E1000_READ_REG(hw, E1000_GORCL);
1729         stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32);
1730         stats->gorc -= (stats->gprc - old_gprc) * ETHER_CRC_LEN;
1731         stats->gotc += E1000_READ_REG(hw, E1000_GOTCL);
1732         stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32);
1733         stats->gotc -= (stats->gptc - old_gptc) * ETHER_CRC_LEN;
1734
1735         stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
1736         stats->ruc += E1000_READ_REG(hw, E1000_RUC);
1737         stats->rfc += E1000_READ_REG(hw, E1000_RFC);
1738         stats->roc += E1000_READ_REG(hw, E1000_ROC);
1739         stats->rjc += E1000_READ_REG(hw, E1000_RJC);
1740
1741         stats->tpr += E1000_READ_REG(hw, E1000_TPR);
1742         stats->tpt += E1000_READ_REG(hw, E1000_TPT);
1743
1744         stats->tor += E1000_READ_REG(hw, E1000_TORL);
1745         stats->tor += ((uint64_t)E1000_READ_REG(hw, E1000_TORH) << 32);
1746         stats->tor -= (stats->tpr - old_tpr) * ETHER_CRC_LEN;
1747         stats->tot += E1000_READ_REG(hw, E1000_TOTL);
1748         stats->tot += ((uint64_t)E1000_READ_REG(hw, E1000_TOTH) << 32);
1749         stats->tot -= (stats->tpt - old_tpt) * ETHER_CRC_LEN;
1750
1751         stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
1752         stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
1753         stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
1754         stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
1755         stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
1756         stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
1757         stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
1758         stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
1759
1760         /* Interrupt Counts */
1761
1762         stats->iac += E1000_READ_REG(hw, E1000_IAC);
1763         stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
1764         stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
1765         stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
1766         stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
1767         stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
1768         stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
1769         stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
1770         stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
1771
1772         /* Host to Card Statistics */
1773
1774         stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
1775         stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
1776         stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
1777         stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
1778         stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
1779         stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
1780         stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
1781         stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL);
1782         stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32);
1783         stats->hgorc -= (stats->rpthc - old_rpthc) * ETHER_CRC_LEN;
1784         stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL);
1785         stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32);
1786         stats->hgotc -= (stats->hgptc - old_hgptc) * ETHER_CRC_LEN;
1787         stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
1788         stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
1789         stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
1790
1791         stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
1792         stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
1793         stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
1794         stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
1795         stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
1796         stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
1797 }
1798
1799 static void
1800 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1801 {
1802         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1803         struct e1000_hw_stats *stats =
1804                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1805
1806         igb_read_stats_registers(hw, stats);
1807
1808         if (rte_stats == NULL)
1809                 return;
1810
1811         /* Rx Errors */
1812         rte_stats->imissed = stats->mpc;
1813         rte_stats->ierrors = stats->crcerrs +
1814                              stats->rlec + stats->ruc + stats->roc +
1815                              stats->rxerrc + stats->algnerrc + stats->cexterr;
1816
1817         /* Tx Errors */
1818         rte_stats->oerrors = stats->ecol + stats->latecol;
1819
1820         rte_stats->ipackets = stats->gprc;
1821         rte_stats->opackets = stats->gptc;
1822         rte_stats->ibytes   = stats->gorc;
1823         rte_stats->obytes   = stats->gotc;
1824 }
1825
1826 static void
1827 eth_igb_stats_reset(struct rte_eth_dev *dev)
1828 {
1829         struct e1000_hw_stats *hw_stats =
1830                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1831
1832         /* HW registers are cleared on read */
1833         eth_igb_stats_get(dev, NULL);
1834
1835         /* Reset software totals */
1836         memset(hw_stats, 0, sizeof(*hw_stats));
1837 }
1838
1839 static void
1840 eth_igb_xstats_reset(struct rte_eth_dev *dev)
1841 {
1842         struct e1000_hw_stats *stats =
1843                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1844
1845         /* HW registers are cleared on read */
1846         eth_igb_xstats_get(dev, NULL, IGB_NB_XSTATS);
1847
1848         /* Reset software totals */
1849         memset(stats, 0, sizeof(*stats));
1850 }
1851
1852 static int eth_igb_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1853         struct rte_eth_xstat_name *xstats_names,
1854         __rte_unused unsigned limit)
1855 {
1856         unsigned i;
1857
1858         if (xstats_names == NULL)
1859                 return IGB_NB_XSTATS;
1860
1861         /* Note: limit checked in rte_eth_xstats_names() */
1862
1863         for (i = 0; i < IGB_NB_XSTATS; i++) {
1864                 snprintf(xstats_names[i].name, sizeof(xstats_names[i].name),
1865                          "%s", rte_igb_stats_strings[i].name);
1866         }
1867
1868         return IGB_NB_XSTATS;
1869 }
1870
1871 static int eth_igb_xstats_get_names_by_ids(struct rte_eth_dev *dev,
1872                 struct rte_eth_xstat_name *xstats_names, uint64_t *ids,
1873                 unsigned int limit)
1874 {
1875         unsigned int i;
1876
1877         if (!ids) {
1878                 if (xstats_names == NULL)
1879                         return IGB_NB_XSTATS;
1880
1881                 for (i = 0; i < IGB_NB_XSTATS; i++)
1882                         snprintf(xstats_names[i].name,
1883                                         sizeof(xstats_names[i].name),
1884                                         "%s", rte_igb_stats_strings[i].name);
1885
1886                 return IGB_NB_XSTATS;
1887
1888         } else {
1889                 struct rte_eth_xstat_name xstats_names_copy[IGB_NB_XSTATS];
1890
1891                 eth_igb_xstats_get_names_by_ids(dev, xstats_names_copy, NULL,
1892                                 IGB_NB_XSTATS);
1893
1894                 for (i = 0; i < limit; i++) {
1895                         if (ids[i] >= IGB_NB_XSTATS) {
1896                                 PMD_INIT_LOG(ERR, "id value isn't valid");
1897                                 return -1;
1898                         }
1899                         strcpy(xstats_names[i].name,
1900                                         xstats_names_copy[ids[i]].name);
1901                 }
1902                 return limit;
1903         }
1904 }
1905
1906 static int
1907 eth_igb_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1908                    unsigned n)
1909 {
1910         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1911         struct e1000_hw_stats *hw_stats =
1912                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1913         unsigned i;
1914
1915         if (n < IGB_NB_XSTATS)
1916                 return IGB_NB_XSTATS;
1917
1918         igb_read_stats_registers(hw, hw_stats);
1919
1920         /* If this is a reset xstats is NULL, and we have cleared the
1921          * registers by reading them.
1922          */
1923         if (!xstats)
1924                 return 0;
1925
1926         /* Extended stats */
1927         for (i = 0; i < IGB_NB_XSTATS; i++) {
1928                 xstats[i].id = i;
1929                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
1930                         rte_igb_stats_strings[i].offset);
1931         }
1932
1933         return IGB_NB_XSTATS;
1934 }
1935
1936 static int
1937 eth_igb_xstats_get_by_ids(struct rte_eth_dev *dev, uint64_t *ids,
1938                 uint64_t *values, unsigned int n)
1939 {
1940         unsigned int i;
1941
1942         if (!ids) {
1943                 struct e1000_hw *hw =
1944                         E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1945                 struct e1000_hw_stats *hw_stats =
1946                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1947
1948                 if (n < IGB_NB_XSTATS)
1949                         return IGB_NB_XSTATS;
1950
1951                 igb_read_stats_registers(hw, hw_stats);
1952
1953                 /* If this is a reset xstats is NULL, and we have cleared the
1954                  * registers by reading them.
1955                  */
1956                 if (!values)
1957                         return 0;
1958
1959                 /* Extended stats */
1960                 for (i = 0; i < IGB_NB_XSTATS; i++)
1961                         values[i] = *(uint64_t *)(((char *)hw_stats) +
1962                                         rte_igb_stats_strings[i].offset);
1963
1964                 return IGB_NB_XSTATS;
1965
1966         } else {
1967                 uint64_t values_copy[IGB_NB_XSTATS];
1968
1969                 eth_igb_xstats_get_by_ids(dev, NULL, values_copy,
1970                                 IGB_NB_XSTATS);
1971
1972                 for (i = 0; i < n; i++) {
1973                         if (ids[i] >= IGB_NB_XSTATS) {
1974                                 PMD_INIT_LOG(ERR, "id value isn't valid");
1975                                 return -1;
1976                         }
1977                         values[i] = values_copy[ids[i]];
1978                 }
1979                 return n;
1980         }
1981 }
1982
1983 static void
1984 igbvf_read_stats_registers(struct e1000_hw *hw, struct e1000_vf_stats *hw_stats)
1985 {
1986         /* Good Rx packets, include VF loopback */
1987         UPDATE_VF_STAT(E1000_VFGPRC,
1988             hw_stats->last_gprc, hw_stats->gprc);
1989
1990         /* Good Rx octets, include VF loopback */
1991         UPDATE_VF_STAT(E1000_VFGORC,
1992             hw_stats->last_gorc, hw_stats->gorc);
1993
1994         /* Good Tx packets, include VF loopback */
1995         UPDATE_VF_STAT(E1000_VFGPTC,
1996             hw_stats->last_gptc, hw_stats->gptc);
1997
1998         /* Good Tx octets, include VF loopback */
1999         UPDATE_VF_STAT(E1000_VFGOTC,
2000             hw_stats->last_gotc, hw_stats->gotc);
2001
2002         /* Rx Multicst packets */
2003         UPDATE_VF_STAT(E1000_VFMPRC,
2004             hw_stats->last_mprc, hw_stats->mprc);
2005
2006         /* Good Rx loopback packets */
2007         UPDATE_VF_STAT(E1000_VFGPRLBC,
2008             hw_stats->last_gprlbc, hw_stats->gprlbc);
2009
2010         /* Good Rx loopback octets */
2011         UPDATE_VF_STAT(E1000_VFGORLBC,
2012             hw_stats->last_gorlbc, hw_stats->gorlbc);
2013
2014         /* Good Tx loopback packets */
2015         UPDATE_VF_STAT(E1000_VFGPTLBC,
2016             hw_stats->last_gptlbc, hw_stats->gptlbc);
2017
2018         /* Good Tx loopback octets */
2019         UPDATE_VF_STAT(E1000_VFGOTLBC,
2020             hw_stats->last_gotlbc, hw_stats->gotlbc);
2021 }
2022
2023 static int eth_igbvf_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
2024                                      struct rte_eth_xstat_name *xstats_names,
2025                                      __rte_unused unsigned limit)
2026 {
2027         unsigned i;
2028
2029         if (xstats_names != NULL)
2030                 for (i = 0; i < IGBVF_NB_XSTATS; i++) {
2031                         snprintf(xstats_names[i].name,
2032                                 sizeof(xstats_names[i].name), "%s",
2033                                 rte_igbvf_stats_strings[i].name);
2034                 }
2035         return IGBVF_NB_XSTATS;
2036 }
2037
2038 static int
2039 eth_igbvf_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
2040                      unsigned n)
2041 {
2042         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2043         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *)
2044                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2045         unsigned i;
2046
2047         if (n < IGBVF_NB_XSTATS)
2048                 return IGBVF_NB_XSTATS;
2049
2050         igbvf_read_stats_registers(hw, hw_stats);
2051
2052         if (!xstats)
2053                 return 0;
2054
2055         for (i = 0; i < IGBVF_NB_XSTATS; i++) {
2056                 xstats[i].id = i;
2057                 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
2058                         rte_igbvf_stats_strings[i].offset);
2059         }
2060
2061         return IGBVF_NB_XSTATS;
2062 }
2063
2064 static void
2065 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
2066 {
2067         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2068         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats *)
2069                           E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2070
2071         igbvf_read_stats_registers(hw, hw_stats);
2072
2073         if (rte_stats == NULL)
2074                 return;
2075
2076         rte_stats->ipackets = hw_stats->gprc;
2077         rte_stats->ibytes = hw_stats->gorc;
2078         rte_stats->opackets = hw_stats->gptc;
2079         rte_stats->obytes = hw_stats->gotc;
2080 }
2081
2082 static void
2083 eth_igbvf_stats_reset(struct rte_eth_dev *dev)
2084 {
2085         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
2086                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2087
2088         /* Sync HW register to the last stats */
2089         eth_igbvf_stats_get(dev, NULL);
2090
2091         /* reset HW current stats*/
2092         memset(&hw_stats->gprc, 0, sizeof(*hw_stats) -
2093                offsetof(struct e1000_vf_stats, gprc));
2094 }
2095
2096 static int
2097 eth_igb_fw_version_get(struct rte_eth_dev *dev, char *fw_version,
2098                        size_t fw_size)
2099 {
2100         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2101         struct e1000_fw_version fw;
2102         int ret;
2103
2104         e1000_get_fw_version(hw, &fw);
2105
2106         switch (hw->mac.type) {
2107         case e1000_i210:
2108         case e1000_i211:
2109                 if (!(e1000_get_flash_presence_i210(hw))) {
2110                         ret = snprintf(fw_version, fw_size,
2111                                  "%2d.%2d-%d",
2112                                  fw.invm_major, fw.invm_minor,
2113                                  fw.invm_img_type);
2114                         break;
2115                 }
2116                 /* fall through */
2117         default:
2118                 /* if option rom is valid, display its version too */
2119                 if (fw.or_valid) {
2120                         ret = snprintf(fw_version, fw_size,
2121                                  "%d.%d, 0x%08x, %d.%d.%d",
2122                                  fw.eep_major, fw.eep_minor, fw.etrack_id,
2123                                  fw.or_major, fw.or_build, fw.or_patch);
2124                 /* no option rom */
2125                 } else {
2126                         if (fw.etrack_id != 0X0000) {
2127                                 ret = snprintf(fw_version, fw_size,
2128                                          "%d.%d, 0x%08x",
2129                                          fw.eep_major, fw.eep_minor,
2130                                          fw.etrack_id);
2131                         } else {
2132                                 ret = snprintf(fw_version, fw_size,
2133                                          "%d.%d.%d",
2134                                          fw.eep_major, fw.eep_minor,
2135                                          fw.eep_build);
2136                         }
2137                 }
2138                 break;
2139         }
2140
2141         ret += 1; /* add the size of '\0' */
2142         if (fw_size < (u32)ret)
2143                 return ret;
2144         else
2145                 return 0;
2146 }
2147
2148 static void
2149 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2150 {
2151         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2152
2153         dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device);
2154         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
2155         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
2156         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
2157         dev_info->rx_offload_capa =
2158                 DEV_RX_OFFLOAD_VLAN_STRIP |
2159                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2160                 DEV_RX_OFFLOAD_UDP_CKSUM  |
2161                 DEV_RX_OFFLOAD_TCP_CKSUM;
2162         dev_info->tx_offload_capa =
2163                 DEV_TX_OFFLOAD_VLAN_INSERT |
2164                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
2165                 DEV_TX_OFFLOAD_UDP_CKSUM   |
2166                 DEV_TX_OFFLOAD_TCP_CKSUM   |
2167                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
2168                 DEV_TX_OFFLOAD_TCP_TSO;
2169
2170         switch (hw->mac.type) {
2171         case e1000_82575:
2172                 dev_info->max_rx_queues = 4;
2173                 dev_info->max_tx_queues = 4;
2174                 dev_info->max_vmdq_pools = 0;
2175                 break;
2176
2177         case e1000_82576:
2178                 dev_info->max_rx_queues = 16;
2179                 dev_info->max_tx_queues = 16;
2180                 dev_info->max_vmdq_pools = ETH_8_POOLS;
2181                 dev_info->vmdq_queue_num = 16;
2182                 break;
2183
2184         case e1000_82580:
2185                 dev_info->max_rx_queues = 8;
2186                 dev_info->max_tx_queues = 8;
2187                 dev_info->max_vmdq_pools = ETH_8_POOLS;
2188                 dev_info->vmdq_queue_num = 8;
2189                 break;
2190
2191         case e1000_i350:
2192                 dev_info->max_rx_queues = 8;
2193                 dev_info->max_tx_queues = 8;
2194                 dev_info->max_vmdq_pools = ETH_8_POOLS;
2195                 dev_info->vmdq_queue_num = 8;
2196                 break;
2197
2198         case e1000_i354:
2199                 dev_info->max_rx_queues = 8;
2200                 dev_info->max_tx_queues = 8;
2201                 break;
2202
2203         case e1000_i210:
2204                 dev_info->max_rx_queues = 4;
2205                 dev_info->max_tx_queues = 4;
2206                 dev_info->max_vmdq_pools = 0;
2207                 break;
2208
2209         case e1000_i211:
2210                 dev_info->max_rx_queues = 2;
2211                 dev_info->max_tx_queues = 2;
2212                 dev_info->max_vmdq_pools = 0;
2213                 break;
2214
2215         default:
2216                 /* Should not happen */
2217                 break;
2218         }
2219         dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t);
2220         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
2221         dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL;
2222
2223         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2224                 .rx_thresh = {
2225                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
2226                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
2227                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
2228                 },
2229                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
2230                 .rx_drop_en = 0,
2231         };
2232
2233         dev_info->default_txconf = (struct rte_eth_txconf) {
2234                 .tx_thresh = {
2235                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
2236                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
2237                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
2238                 },
2239                 .txq_flags = 0,
2240         };
2241
2242         dev_info->rx_desc_lim = rx_desc_lim;
2243         dev_info->tx_desc_lim = tx_desc_lim;
2244
2245         dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M |
2246                         ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M |
2247                         ETH_LINK_SPEED_1G;
2248 }
2249
2250 static const uint32_t *
2251 eth_igb_supported_ptypes_get(struct rte_eth_dev *dev)
2252 {
2253         static const uint32_t ptypes[] = {
2254                 /* refers to igb_rxd_pkt_info_to_pkt_type() */
2255                 RTE_PTYPE_L2_ETHER,
2256                 RTE_PTYPE_L3_IPV4,
2257                 RTE_PTYPE_L3_IPV4_EXT,
2258                 RTE_PTYPE_L3_IPV6,
2259                 RTE_PTYPE_L3_IPV6_EXT,
2260                 RTE_PTYPE_L4_TCP,
2261                 RTE_PTYPE_L4_UDP,
2262                 RTE_PTYPE_L4_SCTP,
2263                 RTE_PTYPE_TUNNEL_IP,
2264                 RTE_PTYPE_INNER_L3_IPV6,
2265                 RTE_PTYPE_INNER_L3_IPV6_EXT,
2266                 RTE_PTYPE_INNER_L4_TCP,
2267                 RTE_PTYPE_INNER_L4_UDP,
2268                 RTE_PTYPE_UNKNOWN
2269         };
2270
2271         if (dev->rx_pkt_burst == eth_igb_recv_pkts ||
2272             dev->rx_pkt_burst == eth_igb_recv_scattered_pkts)
2273                 return ptypes;
2274         return NULL;
2275 }
2276
2277 static void
2278 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2279 {
2280         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2281
2282         dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device);
2283         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
2284         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
2285         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
2286         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
2287                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
2288                                 DEV_RX_OFFLOAD_UDP_CKSUM  |
2289                                 DEV_RX_OFFLOAD_TCP_CKSUM;
2290         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
2291                                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
2292                                 DEV_TX_OFFLOAD_UDP_CKSUM   |
2293                                 DEV_TX_OFFLOAD_TCP_CKSUM   |
2294                                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
2295                                 DEV_TX_OFFLOAD_TCP_TSO;
2296         switch (hw->mac.type) {
2297         case e1000_vfadapt:
2298                 dev_info->max_rx_queues = 2;
2299                 dev_info->max_tx_queues = 2;
2300                 break;
2301         case e1000_vfadapt_i350:
2302                 dev_info->max_rx_queues = 1;
2303                 dev_info->max_tx_queues = 1;
2304                 break;
2305         default:
2306                 /* Should not happen */
2307                 break;
2308         }
2309
2310         dev_info->default_rxconf = (struct rte_eth_rxconf) {
2311                 .rx_thresh = {
2312                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
2313                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
2314                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
2315                 },
2316                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
2317                 .rx_drop_en = 0,
2318         };
2319
2320         dev_info->default_txconf = (struct rte_eth_txconf) {
2321                 .tx_thresh = {
2322                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
2323                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
2324                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
2325                 },
2326                 .txq_flags = 0,
2327         };
2328
2329         dev_info->rx_desc_lim = rx_desc_lim;
2330         dev_info->tx_desc_lim = tx_desc_lim;
2331 }
2332
2333 /* return 0 means link status changed, -1 means not changed */
2334 static int
2335 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete)
2336 {
2337         struct e1000_hw *hw =
2338                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2339         struct rte_eth_link link, old;
2340         int link_check, count;
2341
2342         link_check = 0;
2343         hw->mac.get_link_status = 1;
2344
2345         /* possible wait-to-complete in up to 9 seconds */
2346         for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) {
2347                 /* Read the real link status */
2348                 switch (hw->phy.media_type) {
2349                 case e1000_media_type_copper:
2350                         /* Do the work to read phy */
2351                         e1000_check_for_link(hw);
2352                         link_check = !hw->mac.get_link_status;
2353                         break;
2354
2355                 case e1000_media_type_fiber:
2356                         e1000_check_for_link(hw);
2357                         link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2358                                       E1000_STATUS_LU);
2359                         break;
2360
2361                 case e1000_media_type_internal_serdes:
2362                         e1000_check_for_link(hw);
2363                         link_check = hw->mac.serdes_has_link;
2364                         break;
2365
2366                 /* VF device is type_unknown */
2367                 case e1000_media_type_unknown:
2368                         eth_igbvf_link_update(hw);
2369                         link_check = !hw->mac.get_link_status;
2370                         break;
2371
2372                 default:
2373                         break;
2374                 }
2375                 if (link_check || wait_to_complete == 0)
2376                         break;
2377                 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL);
2378         }
2379         memset(&link, 0, sizeof(link));
2380         rte_igb_dev_atomic_read_link_status(dev, &link);
2381         old = link;
2382
2383         /* Now we check if a transition has happened */
2384         if (link_check) {
2385                 uint16_t duplex, speed;
2386                 hw->mac.ops.get_link_up_info(hw, &speed, &duplex);
2387                 link.link_duplex = (duplex == FULL_DUPLEX) ?
2388                                 ETH_LINK_FULL_DUPLEX :
2389                                 ETH_LINK_HALF_DUPLEX;
2390                 link.link_speed = speed;
2391                 link.link_status = ETH_LINK_UP;
2392                 link.link_autoneg = !(dev->data->dev_conf.link_speeds &
2393                                 ETH_LINK_SPEED_FIXED);
2394         } else if (!link_check) {
2395                 link.link_speed = 0;
2396                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
2397                 link.link_status = ETH_LINK_DOWN;
2398                 link.link_autoneg = ETH_LINK_SPEED_FIXED;
2399         }
2400         rte_igb_dev_atomic_write_link_status(dev, &link);
2401
2402         /* not changed */
2403         if (old.link_status == link.link_status)
2404                 return -1;
2405
2406         /* changed */
2407         return 0;
2408 }
2409
2410 /*
2411  * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
2412  * For ASF and Pass Through versions of f/w this means
2413  * that the driver is loaded.
2414  */
2415 static void
2416 igb_hw_control_acquire(struct e1000_hw *hw)
2417 {
2418         uint32_t ctrl_ext;
2419
2420         /* Let firmware know the driver has taken over */
2421         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
2422         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
2423 }
2424
2425 /*
2426  * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
2427  * For ASF and Pass Through versions of f/w this means that the
2428  * driver is no longer loaded.
2429  */
2430 static void
2431 igb_hw_control_release(struct e1000_hw *hw)
2432 {
2433         uint32_t ctrl_ext;
2434
2435         /* Let firmware taken over control of h/w */
2436         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
2437         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
2438                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
2439 }
2440
2441 /*
2442  * Bit of a misnomer, what this really means is
2443  * to enable OS management of the system... aka
2444  * to disable special hardware management features.
2445  */
2446 static void
2447 igb_init_manageability(struct e1000_hw *hw)
2448 {
2449         if (e1000_enable_mng_pass_thru(hw)) {
2450                 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H);
2451                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
2452
2453                 /* disable hardware interception of ARP */
2454                 manc &= ~(E1000_MANC_ARP_EN);
2455
2456                 /* enable receiving management packets to the host */
2457                 manc |= E1000_MANC_EN_MNG2HOST;
2458                 manc2h |= 1 << 5;  /* Mng Port 623 */
2459                 manc2h |= 1 << 6;  /* Mng Port 664 */
2460                 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h);
2461                 E1000_WRITE_REG(hw, E1000_MANC, manc);
2462         }
2463 }
2464
2465 static void
2466 igb_release_manageability(struct e1000_hw *hw)
2467 {
2468         if (e1000_enable_mng_pass_thru(hw)) {
2469                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
2470
2471                 manc |= E1000_MANC_ARP_EN;
2472                 manc &= ~E1000_MANC_EN_MNG2HOST;
2473
2474                 E1000_WRITE_REG(hw, E1000_MANC, manc);
2475         }
2476 }
2477
2478 static void
2479 eth_igb_promiscuous_enable(struct rte_eth_dev *dev)
2480 {
2481         struct e1000_hw *hw =
2482                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2483         uint32_t rctl;
2484
2485         rctl = E1000_READ_REG(hw, E1000_RCTL);
2486         rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2487         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2488 }
2489
2490 static void
2491 eth_igb_promiscuous_disable(struct rte_eth_dev *dev)
2492 {
2493         struct e1000_hw *hw =
2494                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2495         uint32_t rctl;
2496
2497         rctl = E1000_READ_REG(hw, E1000_RCTL);
2498         rctl &= (~E1000_RCTL_UPE);
2499         if (dev->data->all_multicast == 1)
2500                 rctl |= E1000_RCTL_MPE;
2501         else
2502                 rctl &= (~E1000_RCTL_MPE);
2503         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2504 }
2505
2506 static void
2507 eth_igb_allmulticast_enable(struct rte_eth_dev *dev)
2508 {
2509         struct e1000_hw *hw =
2510                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2511         uint32_t rctl;
2512
2513         rctl = E1000_READ_REG(hw, E1000_RCTL);
2514         rctl |= E1000_RCTL_MPE;
2515         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2516 }
2517
2518 static void
2519 eth_igb_allmulticast_disable(struct rte_eth_dev *dev)
2520 {
2521         struct e1000_hw *hw =
2522                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2523         uint32_t rctl;
2524
2525         if (dev->data->promiscuous == 1)
2526                 return; /* must remain in all_multicast mode */
2527         rctl = E1000_READ_REG(hw, E1000_RCTL);
2528         rctl &= (~E1000_RCTL_MPE);
2529         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2530 }
2531
2532 static int
2533 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2534 {
2535         struct e1000_hw *hw =
2536                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2537         struct e1000_vfta * shadow_vfta =
2538                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2539         uint32_t vfta;
2540         uint32_t vid_idx;
2541         uint32_t vid_bit;
2542
2543         vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) &
2544                               E1000_VFTA_ENTRY_MASK);
2545         vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK));
2546         vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx);
2547         if (on)
2548                 vfta |= vid_bit;
2549         else
2550                 vfta &= ~vid_bit;
2551         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta);
2552
2553         /* update local VFTA copy */
2554         shadow_vfta->vfta[vid_idx] = vfta;
2555
2556         return 0;
2557 }
2558
2559 static int
2560 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev,
2561                       enum rte_vlan_type vlan_type,
2562                       uint16_t tpid)
2563 {
2564         struct e1000_hw *hw =
2565                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2566         uint32_t reg, qinq;
2567
2568         qinq = E1000_READ_REG(hw, E1000_CTRL_EXT);
2569         qinq &= E1000_CTRL_EXT_EXT_VLAN;
2570
2571         /* only outer TPID of double VLAN can be configured*/
2572         if (qinq && vlan_type == ETH_VLAN_TYPE_OUTER) {
2573                 reg = E1000_READ_REG(hw, E1000_VET);
2574                 reg = (reg & (~E1000_VET_VET_EXT)) |
2575                         ((uint32_t)tpid << E1000_VET_VET_EXT_SHIFT);
2576                 E1000_WRITE_REG(hw, E1000_VET, reg);
2577
2578                 return 0;
2579         }
2580
2581         /* all other TPID values are read-only*/
2582         PMD_DRV_LOG(ERR, "Not supported");
2583
2584         return -ENOTSUP;
2585 }
2586
2587 static void
2588 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev)
2589 {
2590         struct e1000_hw *hw =
2591                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2592         uint32_t reg;
2593
2594         /* Filter Table Disable */
2595         reg = E1000_READ_REG(hw, E1000_RCTL);
2596         reg &= ~E1000_RCTL_CFIEN;
2597         reg &= ~E1000_RCTL_VFE;
2598         E1000_WRITE_REG(hw, E1000_RCTL, reg);
2599 }
2600
2601 static void
2602 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev)
2603 {
2604         struct e1000_hw *hw =
2605                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2606         struct e1000_vfta * shadow_vfta =
2607                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2608         uint32_t reg;
2609         int i;
2610
2611         /* Filter Table Enable, CFI not used for packet acceptance */
2612         reg = E1000_READ_REG(hw, E1000_RCTL);
2613         reg &= ~E1000_RCTL_CFIEN;
2614         reg |= E1000_RCTL_VFE;
2615         E1000_WRITE_REG(hw, E1000_RCTL, reg);
2616
2617         /* restore VFTA table */
2618         for (i = 0; i < IGB_VFTA_SIZE; i++)
2619                 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]);
2620 }
2621
2622 static void
2623 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev)
2624 {
2625         struct e1000_hw *hw =
2626                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2627         uint32_t reg;
2628
2629         /* VLAN Mode Disable */
2630         reg = E1000_READ_REG(hw, E1000_CTRL);
2631         reg &= ~E1000_CTRL_VME;
2632         E1000_WRITE_REG(hw, E1000_CTRL, reg);
2633 }
2634
2635 static void
2636 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev)
2637 {
2638         struct e1000_hw *hw =
2639                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2640         uint32_t reg;
2641
2642         /* VLAN Mode Enable */
2643         reg = E1000_READ_REG(hw, E1000_CTRL);
2644         reg |= E1000_CTRL_VME;
2645         E1000_WRITE_REG(hw, E1000_CTRL, reg);
2646 }
2647
2648 static void
2649 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev)
2650 {
2651         struct e1000_hw *hw =
2652                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2653         uint32_t reg;
2654
2655         /* CTRL_EXT: Extended VLAN */
2656         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
2657         reg &= ~E1000_CTRL_EXT_EXTEND_VLAN;
2658         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
2659
2660         /* Update maximum packet length */
2661         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
2662                 E1000_WRITE_REG(hw, E1000_RLPML,
2663                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
2664                                                 VLAN_TAG_SIZE);
2665 }
2666
2667 static void
2668 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev)
2669 {
2670         struct e1000_hw *hw =
2671                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2672         uint32_t reg;
2673
2674         /* CTRL_EXT: Extended VLAN */
2675         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
2676         reg |= E1000_CTRL_EXT_EXTEND_VLAN;
2677         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
2678
2679         /* Update maximum packet length */
2680         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
2681                 E1000_WRITE_REG(hw, E1000_RLPML,
2682                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
2683                                                 2 * VLAN_TAG_SIZE);
2684 }
2685
2686 static void
2687 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2688 {
2689         if(mask & ETH_VLAN_STRIP_MASK){
2690                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
2691                         igb_vlan_hw_strip_enable(dev);
2692                 else
2693                         igb_vlan_hw_strip_disable(dev);
2694         }
2695
2696         if(mask & ETH_VLAN_FILTER_MASK){
2697                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
2698                         igb_vlan_hw_filter_enable(dev);
2699                 else
2700                         igb_vlan_hw_filter_disable(dev);
2701         }
2702
2703         if(mask & ETH_VLAN_EXTEND_MASK){
2704                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
2705                         igb_vlan_hw_extend_enable(dev);
2706                 else
2707                         igb_vlan_hw_extend_disable(dev);
2708         }
2709 }
2710
2711
2712 /**
2713  * It enables the interrupt mask and then enable the interrupt.
2714  *
2715  * @param dev
2716  *  Pointer to struct rte_eth_dev.
2717  *
2718  * @return
2719  *  - On success, zero.
2720  *  - On failure, a negative value.
2721  */
2722 static int
2723 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev)
2724 {
2725         struct e1000_interrupt *intr =
2726                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2727
2728         intr->mask |= E1000_ICR_LSC;
2729
2730         return 0;
2731 }
2732
2733 /* It clears the interrupt causes and enables the interrupt.
2734  * It will be called once only during nic initialized.
2735  *
2736  * @param dev
2737  *  Pointer to struct rte_eth_dev.
2738  *
2739  * @return
2740  *  - On success, zero.
2741  *  - On failure, a negative value.
2742  */
2743 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev)
2744 {
2745         uint32_t mask, regval;
2746         struct e1000_hw *hw =
2747                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2748         struct rte_eth_dev_info dev_info;
2749
2750         memset(&dev_info, 0, sizeof(dev_info));
2751         eth_igb_infos_get(dev, &dev_info);
2752
2753         mask = 0xFFFFFFFF >> (32 - dev_info.max_rx_queues);
2754         regval = E1000_READ_REG(hw, E1000_EIMS);
2755         E1000_WRITE_REG(hw, E1000_EIMS, regval | mask);
2756
2757         return 0;
2758 }
2759
2760 /*
2761  * It reads ICR and gets interrupt causes, check it and set a bit flag
2762  * to update link status.
2763  *
2764  * @param dev
2765  *  Pointer to struct rte_eth_dev.
2766  *
2767  * @return
2768  *  - On success, zero.
2769  *  - On failure, a negative value.
2770  */
2771 static int
2772 eth_igb_interrupt_get_status(struct rte_eth_dev *dev)
2773 {
2774         uint32_t icr;
2775         struct e1000_hw *hw =
2776                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2777         struct e1000_interrupt *intr =
2778                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2779
2780         igb_intr_disable(hw);
2781
2782         /* read-on-clear nic registers here */
2783         icr = E1000_READ_REG(hw, E1000_ICR);
2784
2785         intr->flags = 0;
2786         if (icr & E1000_ICR_LSC) {
2787                 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
2788         }
2789
2790         if (icr & E1000_ICR_VMMB)
2791                 intr->flags |= E1000_FLAG_MAILBOX;
2792
2793         return 0;
2794 }
2795
2796 /*
2797  * It executes link_update after knowing an interrupt is prsent.
2798  *
2799  * @param dev
2800  *  Pointer to struct rte_eth_dev.
2801  *
2802  * @return
2803  *  - On success, zero.
2804  *  - On failure, a negative value.
2805  */
2806 static int
2807 eth_igb_interrupt_action(struct rte_eth_dev *dev,
2808                          struct rte_intr_handle *intr_handle)
2809 {
2810         struct e1000_hw *hw =
2811                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2812         struct e1000_interrupt *intr =
2813                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2814         struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev);
2815         uint32_t tctl, rctl;
2816         struct rte_eth_link link;
2817         int ret;
2818
2819         if (intr->flags & E1000_FLAG_MAILBOX) {
2820                 igb_pf_mbx_process(dev);
2821                 intr->flags &= ~E1000_FLAG_MAILBOX;
2822         }
2823
2824         igb_intr_enable(dev);
2825         rte_intr_enable(intr_handle);
2826
2827         if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) {
2828                 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE;
2829
2830                 /* set get_link_status to check register later */
2831                 hw->mac.get_link_status = 1;
2832                 ret = eth_igb_link_update(dev, 0);
2833
2834                 /* check if link has changed */
2835                 if (ret < 0)
2836                         return 0;
2837
2838                 memset(&link, 0, sizeof(link));
2839                 rte_igb_dev_atomic_read_link_status(dev, &link);
2840                 if (link.link_status) {
2841                         PMD_INIT_LOG(INFO,
2842                                      " Port %d: Link Up - speed %u Mbps - %s",
2843                                      dev->data->port_id,
2844                                      (unsigned)link.link_speed,
2845                                      link.link_duplex == ETH_LINK_FULL_DUPLEX ?
2846                                      "full-duplex" : "half-duplex");
2847                 } else {
2848                         PMD_INIT_LOG(INFO, " Port %d: Link Down",
2849                                      dev->data->port_id);
2850                 }
2851
2852                 PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d",
2853                              pci_dev->addr.domain,
2854                              pci_dev->addr.bus,
2855                              pci_dev->addr.devid,
2856                              pci_dev->addr.function);
2857                 tctl = E1000_READ_REG(hw, E1000_TCTL);
2858                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2859                 if (link.link_status) {
2860                         /* enable Tx/Rx */
2861                         tctl |= E1000_TCTL_EN;
2862                         rctl |= E1000_RCTL_EN;
2863                 } else {
2864                         /* disable Tx/Rx */
2865                         tctl &= ~E1000_TCTL_EN;
2866                         rctl &= ~E1000_RCTL_EN;
2867                 }
2868                 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
2869                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2870                 E1000_WRITE_FLUSH(hw);
2871                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
2872         }
2873
2874         return 0;
2875 }
2876
2877 /**
2878  * Interrupt handler which shall be registered at first.
2879  *
2880  * @param handle
2881  *  Pointer to interrupt handle.
2882  * @param param
2883  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2884  *
2885  * @return
2886  *  void
2887  */
2888 static void
2889 eth_igb_interrupt_handler(void *param)
2890 {
2891         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2892
2893         eth_igb_interrupt_get_status(dev);
2894         eth_igb_interrupt_action(dev, dev->intr_handle);
2895 }
2896
2897 static int
2898 eth_igbvf_interrupt_get_status(struct rte_eth_dev *dev)
2899 {
2900         uint32_t eicr;
2901         struct e1000_hw *hw =
2902                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2903         struct e1000_interrupt *intr =
2904                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2905
2906         igbvf_intr_disable(hw);
2907
2908         /* read-on-clear nic registers here */
2909         eicr = E1000_READ_REG(hw, E1000_EICR);
2910         intr->flags = 0;
2911
2912         if (eicr == E1000_VTIVAR_MISC_MAILBOX)
2913                 intr->flags |= E1000_FLAG_MAILBOX;
2914
2915         return 0;
2916 }
2917
2918 void igbvf_mbx_process(struct rte_eth_dev *dev)
2919 {
2920         struct e1000_hw *hw =
2921                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2922         struct e1000_mbx_info *mbx = &hw->mbx;
2923         u32 in_msg = 0;
2924
2925         if (mbx->ops.read(hw, &in_msg, 1, 0))
2926                 return;
2927
2928         /* PF reset VF event */
2929         if (in_msg == E1000_PF_CONTROL_MSG)
2930                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, NULL);
2931 }
2932
2933 static int
2934 eth_igbvf_interrupt_action(struct rte_eth_dev *dev, struct rte_intr_handle *intr_handle)
2935 {
2936         struct e1000_interrupt *intr =
2937                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2938
2939         if (intr->flags & E1000_FLAG_MAILBOX) {
2940                 igbvf_mbx_process(dev);
2941                 intr->flags &= ~E1000_FLAG_MAILBOX;
2942         }
2943
2944         igbvf_intr_enable(dev);
2945         rte_intr_enable(intr_handle);
2946
2947         return 0;
2948 }
2949
2950 static void
2951 eth_igbvf_interrupt_handler(void *param)
2952 {
2953         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2954
2955         eth_igbvf_interrupt_get_status(dev);
2956         eth_igbvf_interrupt_action(dev, dev->intr_handle);
2957 }
2958
2959 static int
2960 eth_igb_led_on(struct rte_eth_dev *dev)
2961 {
2962         struct e1000_hw *hw;
2963
2964         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2965         return e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP;
2966 }
2967
2968 static int
2969 eth_igb_led_off(struct rte_eth_dev *dev)
2970 {
2971         struct e1000_hw *hw;
2972
2973         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2974         return e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP;
2975 }
2976
2977 static int
2978 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2979 {
2980         struct e1000_hw *hw;
2981         uint32_t ctrl;
2982         int tx_pause;
2983         int rx_pause;
2984
2985         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2986         fc_conf->pause_time = hw->fc.pause_time;
2987         fc_conf->high_water = hw->fc.high_water;
2988         fc_conf->low_water = hw->fc.low_water;
2989         fc_conf->send_xon = hw->fc.send_xon;
2990         fc_conf->autoneg = hw->mac.autoneg;
2991
2992         /*
2993          * Return rx_pause and tx_pause status according to actual setting of
2994          * the TFCE and RFCE bits in the CTRL register.
2995          */
2996         ctrl = E1000_READ_REG(hw, E1000_CTRL);
2997         if (ctrl & E1000_CTRL_TFCE)
2998                 tx_pause = 1;
2999         else
3000                 tx_pause = 0;
3001
3002         if (ctrl & E1000_CTRL_RFCE)
3003                 rx_pause = 1;
3004         else
3005                 rx_pause = 0;
3006
3007         if (rx_pause && tx_pause)
3008                 fc_conf->mode = RTE_FC_FULL;
3009         else if (rx_pause)
3010                 fc_conf->mode = RTE_FC_RX_PAUSE;
3011         else if (tx_pause)
3012                 fc_conf->mode = RTE_FC_TX_PAUSE;
3013         else
3014                 fc_conf->mode = RTE_FC_NONE;
3015
3016         return 0;
3017 }
3018
3019 static int
3020 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
3021 {
3022         struct e1000_hw *hw;
3023         int err;
3024         enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = {
3025                 e1000_fc_none,
3026                 e1000_fc_rx_pause,
3027                 e1000_fc_tx_pause,
3028                 e1000_fc_full
3029         };
3030         uint32_t rx_buf_size;
3031         uint32_t max_high_water;
3032         uint32_t rctl;
3033
3034         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3035         if (fc_conf->autoneg != hw->mac.autoneg)
3036                 return -ENOTSUP;
3037         rx_buf_size = igb_get_rx_buffer_size(hw);
3038         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
3039
3040         /* At least reserve one Ethernet frame for watermark */
3041         max_high_water = rx_buf_size - ETHER_MAX_LEN;
3042         if ((fc_conf->high_water > max_high_water) ||
3043             (fc_conf->high_water < fc_conf->low_water)) {
3044                 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value");
3045                 PMD_INIT_LOG(ERR, "high water must <=  0x%x", max_high_water);
3046                 return -EINVAL;
3047         }
3048
3049         hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode];
3050         hw->fc.pause_time     = fc_conf->pause_time;
3051         hw->fc.high_water     = fc_conf->high_water;
3052         hw->fc.low_water      = fc_conf->low_water;
3053         hw->fc.send_xon       = fc_conf->send_xon;
3054
3055         err = e1000_setup_link_generic(hw);
3056         if (err == E1000_SUCCESS) {
3057
3058                 /* check if we want to forward MAC frames - driver doesn't have native
3059                  * capability to do that, so we'll write the registers ourselves */
3060
3061                 rctl = E1000_READ_REG(hw, E1000_RCTL);
3062
3063                 /* set or clear MFLCN.PMCF bit depending on configuration */
3064                 if (fc_conf->mac_ctrl_frame_fwd != 0)
3065                         rctl |= E1000_RCTL_PMCF;
3066                 else
3067                         rctl &= ~E1000_RCTL_PMCF;
3068
3069                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3070                 E1000_WRITE_FLUSH(hw);
3071
3072                 return 0;
3073         }
3074
3075         PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err);
3076         return -EIO;
3077 }
3078
3079 #define E1000_RAH_POOLSEL_SHIFT      (18)
3080 static void
3081 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
3082                 uint32_t index, __rte_unused uint32_t pool)
3083 {
3084         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3085         uint32_t rah;
3086
3087         e1000_rar_set(hw, mac_addr->addr_bytes, index);
3088         rah = E1000_READ_REG(hw, E1000_RAH(index));
3089         rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool));
3090         E1000_WRITE_REG(hw, E1000_RAH(index), rah);
3091 }
3092
3093 static void
3094 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index)
3095 {
3096         uint8_t addr[ETHER_ADDR_LEN];
3097         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3098
3099         memset(addr, 0, sizeof(addr));
3100
3101         e1000_rar_set(hw, addr, index);
3102 }
3103
3104 static void
3105 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev,
3106                                 struct ether_addr *addr)
3107 {
3108         eth_igb_rar_clear(dev, 0);
3109
3110         eth_igb_rar_set(dev, (void *)addr, 0, 0);
3111 }
3112 /*
3113  * Virtual Function operations
3114  */
3115 static void
3116 igbvf_intr_disable(struct e1000_hw *hw)
3117 {
3118         PMD_INIT_FUNC_TRACE();
3119
3120         /* Clear interrupt mask to stop from interrupts being generated */
3121         E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF);
3122
3123         E1000_WRITE_FLUSH(hw);
3124 }
3125
3126 static void
3127 igbvf_stop_adapter(struct rte_eth_dev *dev)
3128 {
3129         u32 reg_val;
3130         u16 i;
3131         struct rte_eth_dev_info dev_info;
3132         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3133
3134         memset(&dev_info, 0, sizeof(dev_info));
3135         eth_igbvf_infos_get(dev, &dev_info);
3136
3137         /* Clear interrupt mask to stop from interrupts being generated */
3138         igbvf_intr_disable(hw);
3139
3140         /* Clear any pending interrupts, flush previous writes */
3141         E1000_READ_REG(hw, E1000_EICR);
3142
3143         /* Disable the transmit unit.  Each queue must be disabled. */
3144         for (i = 0; i < dev_info.max_tx_queues; i++)
3145                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH);
3146
3147         /* Disable the receive unit by stopping each queue */
3148         for (i = 0; i < dev_info.max_rx_queues; i++) {
3149                 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i));
3150                 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE;
3151                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val);
3152                 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE)
3153                         ;
3154         }
3155
3156         /* flush all queues disables */
3157         E1000_WRITE_FLUSH(hw);
3158         msec_delay(2);
3159 }
3160
3161 static int eth_igbvf_link_update(struct e1000_hw *hw)
3162 {
3163         struct e1000_mbx_info *mbx = &hw->mbx;
3164         struct e1000_mac_info *mac = &hw->mac;
3165         int ret_val = E1000_SUCCESS;
3166
3167         PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf");
3168
3169         /*
3170          * We only want to run this if there has been a rst asserted.
3171          * in this case that could mean a link change, device reset,
3172          * or a virtual function reset
3173          */
3174
3175         /* If we were hit with a reset or timeout drop the link */
3176         if (!e1000_check_for_rst(hw, 0) || !mbx->timeout)
3177                 mac->get_link_status = TRUE;
3178
3179         if (!mac->get_link_status)
3180                 goto out;
3181
3182         /* if link status is down no point in checking to see if pf is up */
3183         if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU))
3184                 goto out;
3185
3186         /* if we passed all the tests above then the link is up and we no
3187          * longer need to check for link */
3188         mac->get_link_status = FALSE;
3189
3190 out:
3191         return ret_val;
3192 }
3193
3194
3195 static int
3196 igbvf_dev_configure(struct rte_eth_dev *dev)
3197 {
3198         struct rte_eth_conf* conf = &dev->data->dev_conf;
3199
3200         PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
3201                      dev->data->port_id);
3202
3203         /*
3204          * VF has no ability to enable/disable HW CRC
3205          * Keep the persistent behavior the same as Host PF
3206          */
3207 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC
3208         if (!conf->rxmode.hw_strip_crc) {
3209                 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip");
3210                 conf->rxmode.hw_strip_crc = 1;
3211         }
3212 #else
3213         if (conf->rxmode.hw_strip_crc) {
3214                 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip");
3215                 conf->rxmode.hw_strip_crc = 0;
3216         }
3217 #endif
3218
3219         return 0;
3220 }
3221
3222 static int
3223 igbvf_dev_start(struct rte_eth_dev *dev)
3224 {
3225         struct e1000_hw *hw =
3226                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3227         struct e1000_adapter *adapter =
3228                 E1000_DEV_PRIVATE(dev->data->dev_private);
3229         struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev);
3230         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
3231         int ret;
3232         uint32_t intr_vector = 0;
3233
3234         PMD_INIT_FUNC_TRACE();
3235
3236         hw->mac.ops.reset_hw(hw);
3237         adapter->stopped = 0;
3238
3239         /* Set all vfta */
3240         igbvf_set_vfta_all(dev,1);
3241
3242         eth_igbvf_tx_init(dev);
3243
3244         /* This can fail when allocating mbufs for descriptor rings */
3245         ret = eth_igbvf_rx_init(dev);
3246         if (ret) {
3247                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
3248                 igb_dev_clear_queues(dev);
3249                 return ret;
3250         }
3251
3252         /* check and configure queue intr-vector mapping */
3253         if (dev->data->dev_conf.intr_conf.rxq != 0) {
3254                 intr_vector = dev->data->nb_rx_queues;
3255                 ret = rte_intr_efd_enable(intr_handle, intr_vector);
3256                 if (ret)
3257                         return ret;
3258         }
3259
3260         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
3261                 intr_handle->intr_vec =
3262                         rte_zmalloc("intr_vec",
3263                                     dev->data->nb_rx_queues * sizeof(int), 0);
3264                 if (!intr_handle->intr_vec) {
3265                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
3266                                      " intr_vec", dev->data->nb_rx_queues);
3267                         return -ENOMEM;
3268                 }
3269         }
3270
3271         eth_igbvf_configure_msix_intr(dev);
3272
3273         /* enable uio/vfio intr/eventfd mapping */
3274         rte_intr_enable(intr_handle);
3275
3276         /* resume enabled intr since hw reset */
3277         igbvf_intr_enable(dev);
3278
3279         return 0;
3280 }
3281
3282 static void
3283 igbvf_dev_stop(struct rte_eth_dev *dev)
3284 {
3285         struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev);
3286         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
3287
3288         PMD_INIT_FUNC_TRACE();
3289
3290         igbvf_stop_adapter(dev);
3291
3292         /*
3293           * Clear what we set, but we still keep shadow_vfta to
3294           * restore after device starts
3295           */
3296         igbvf_set_vfta_all(dev,0);
3297
3298         igb_dev_clear_queues(dev);
3299
3300         /* disable intr eventfd mapping */
3301         rte_intr_disable(intr_handle);
3302
3303         /* Clean datapath event and queue/vec mapping */
3304         rte_intr_efd_disable(intr_handle);
3305         if (intr_handle->intr_vec) {
3306                 rte_free(intr_handle->intr_vec);
3307                 intr_handle->intr_vec = NULL;
3308         }
3309 }
3310
3311 static void
3312 igbvf_dev_close(struct rte_eth_dev *dev)
3313 {
3314         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3315         struct e1000_adapter *adapter =
3316                 E1000_DEV_PRIVATE(dev->data->dev_private);
3317         struct ether_addr addr;
3318
3319         PMD_INIT_FUNC_TRACE();
3320
3321         e1000_reset_hw(hw);
3322
3323         igbvf_dev_stop(dev);
3324         adapter->stopped = 1;
3325         igb_dev_free_queues(dev);
3326
3327         /**
3328          * reprogram the RAR with a zero mac address,
3329          * to ensure that the VF traffic goes to the PF
3330          * after stop, close and detach of the VF.
3331          **/
3332
3333         memset(&addr, 0, sizeof(addr));
3334         igbvf_default_mac_addr_set(dev, &addr);
3335 }
3336
3337 static void
3338 igbvf_promiscuous_enable(struct rte_eth_dev *dev)
3339 {
3340         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3341
3342         /* Set both unicast and multicast promisc */
3343         e1000_promisc_set_vf(hw, e1000_promisc_enabled);
3344 }
3345
3346 static void
3347 igbvf_promiscuous_disable(struct rte_eth_dev *dev)
3348 {
3349         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3350
3351         /* If in allmulticast mode leave multicast promisc */
3352         if (dev->data->all_multicast == 1)
3353                 e1000_promisc_set_vf(hw, e1000_promisc_multicast);
3354         else
3355                 e1000_promisc_set_vf(hw, e1000_promisc_disabled);
3356 }
3357
3358 static void
3359 igbvf_allmulticast_enable(struct rte_eth_dev *dev)
3360 {
3361         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3362
3363         /* In promiscuous mode multicast promisc already set */
3364         if (dev->data->promiscuous == 0)
3365                 e1000_promisc_set_vf(hw, e1000_promisc_multicast);
3366 }
3367
3368 static void
3369 igbvf_allmulticast_disable(struct rte_eth_dev *dev)
3370 {
3371         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3372
3373         /* In promiscuous mode leave multicast promisc enabled */
3374         if (dev->data->promiscuous == 0)
3375                 e1000_promisc_set_vf(hw, e1000_promisc_disabled);
3376 }
3377
3378 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on)
3379 {
3380         struct e1000_mbx_info *mbx = &hw->mbx;
3381         uint32_t msgbuf[2];
3382         s32 err;
3383
3384         /* After set vlan, vlan strip will also be enabled in igb driver*/
3385         msgbuf[0] = E1000_VF_SET_VLAN;
3386         msgbuf[1] = vid;
3387         /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */
3388         if (on)
3389                 msgbuf[0] |= E1000_VF_SET_VLAN_ADD;
3390
3391         err = mbx->ops.write_posted(hw, msgbuf, 2, 0);
3392         if (err)
3393                 goto mbx_err;
3394
3395         err = mbx->ops.read_posted(hw, msgbuf, 2, 0);
3396         if (err)
3397                 goto mbx_err;
3398
3399         msgbuf[0] &= ~E1000_VT_MSGTYPE_CTS;
3400         if (msgbuf[0] == (E1000_VF_SET_VLAN | E1000_VT_MSGTYPE_NACK))
3401                 err = -EINVAL;
3402
3403 mbx_err:
3404         return err;
3405 }
3406
3407 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on)
3408 {
3409         struct e1000_hw *hw =
3410                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3411         struct e1000_vfta * shadow_vfta =
3412                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
3413         int i = 0, j = 0, vfta = 0, mask = 1;
3414
3415         for (i = 0; i < IGB_VFTA_SIZE; i++){
3416                 vfta = shadow_vfta->vfta[i];
3417                 if(vfta){
3418                         mask = 1;
3419                         for (j = 0; j < 32; j++){
3420                                 if(vfta & mask)
3421                                         igbvf_set_vfta(hw,
3422                                                 (uint16_t)((i<<5)+j), on);
3423                                 mask<<=1;
3424                         }
3425                 }
3426         }
3427
3428 }
3429
3430 static int
3431 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
3432 {
3433         struct e1000_hw *hw =
3434                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3435         struct e1000_vfta * shadow_vfta =
3436                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
3437         uint32_t vid_idx = 0;
3438         uint32_t vid_bit = 0;
3439         int ret = 0;
3440
3441         PMD_INIT_FUNC_TRACE();
3442
3443         /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/
3444         ret = igbvf_set_vfta(hw, vlan_id, !!on);
3445         if(ret){
3446                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
3447                 return ret;
3448         }
3449         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
3450         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
3451
3452         /*Save what we set and retore it after device reset*/
3453         if (on)
3454                 shadow_vfta->vfta[vid_idx] |= vid_bit;
3455         else
3456                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
3457
3458         return 0;
3459 }
3460
3461 static void
3462 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr)
3463 {
3464         struct e1000_hw *hw =
3465                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3466
3467         /* index is not used by rar_set() */
3468         hw->mac.ops.rar_set(hw, (void *)addr, 0);
3469 }
3470
3471
3472 static int
3473 eth_igb_rss_reta_update(struct rte_eth_dev *dev,
3474                         struct rte_eth_rss_reta_entry64 *reta_conf,
3475                         uint16_t reta_size)
3476 {
3477         uint8_t i, j, mask;
3478         uint32_t reta, r;
3479         uint16_t idx, shift;
3480         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3481
3482         if (reta_size != ETH_RSS_RETA_SIZE_128) {
3483                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3484                         "(%d) doesn't match the number hardware can supported "
3485                         "(%d)", reta_size, ETH_RSS_RETA_SIZE_128);
3486                 return -EINVAL;
3487         }
3488
3489         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
3490                 idx = i / RTE_RETA_GROUP_SIZE;
3491                 shift = i % RTE_RETA_GROUP_SIZE;
3492                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
3493                                                 IGB_4_BIT_MASK);
3494                 if (!mask)
3495                         continue;
3496                 if (mask == IGB_4_BIT_MASK)
3497                         r = 0;
3498                 else
3499                         r = E1000_READ_REG(hw, E1000_RETA(i >> 2));
3500                 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) {
3501                         if (mask & (0x1 << j))
3502                                 reta |= reta_conf[idx].reta[shift + j] <<
3503                                                         (CHAR_BIT * j);
3504                         else
3505                                 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j));
3506                 }
3507                 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta);
3508         }
3509
3510         return 0;
3511 }
3512
3513 static int
3514 eth_igb_rss_reta_query(struct rte_eth_dev *dev,
3515                        struct rte_eth_rss_reta_entry64 *reta_conf,
3516                        uint16_t reta_size)
3517 {
3518         uint8_t i, j, mask;
3519         uint32_t reta;
3520         uint16_t idx, shift;
3521         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3522
3523         if (reta_size != ETH_RSS_RETA_SIZE_128) {
3524                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
3525                         "(%d) doesn't match the number hardware can supported "
3526                         "(%d)", reta_size, ETH_RSS_RETA_SIZE_128);
3527                 return -EINVAL;
3528         }
3529
3530         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
3531                 idx = i / RTE_RETA_GROUP_SIZE;
3532                 shift = i % RTE_RETA_GROUP_SIZE;
3533                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
3534                                                 IGB_4_BIT_MASK);
3535                 if (!mask)
3536                         continue;
3537                 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2));
3538                 for (j = 0; j < IGB_4_BIT_WIDTH; j++) {
3539                         if (mask & (0x1 << j))
3540                                 reta_conf[idx].reta[shift + j] =
3541                                         ((reta >> (CHAR_BIT * j)) &
3542                                                 IGB_8_BIT_MASK);
3543                 }
3544         }
3545
3546         return 0;
3547 }
3548
3549 #define MAC_TYPE_FILTER_SUP(type)    do {\
3550         if ((type) != e1000_82580 && (type) != e1000_i350 &&\
3551                 (type) != e1000_82576)\
3552                 return -ENOTSUP;\
3553 } while (0)
3554
3555 static int
3556 eth_igb_syn_filter_set(struct rte_eth_dev *dev,
3557                         struct rte_eth_syn_filter *filter,
3558                         bool add)
3559 {
3560         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3561         uint32_t synqf, rfctl;
3562
3563         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
3564                 return -EINVAL;
3565
3566         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
3567
3568         if (add) {
3569                 if (synqf & E1000_SYN_FILTER_ENABLE)
3570                         return -EINVAL;
3571
3572                 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) &
3573                         E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE);
3574
3575                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
3576                 if (filter->hig_pri)
3577                         rfctl |= E1000_RFCTL_SYNQFP;
3578                 else
3579                         rfctl &= ~E1000_RFCTL_SYNQFP;
3580
3581                 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl);
3582         } else {
3583                 if (!(synqf & E1000_SYN_FILTER_ENABLE))
3584                         return -ENOENT;
3585                 synqf = 0;
3586         }
3587
3588         E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf);
3589         E1000_WRITE_FLUSH(hw);
3590         return 0;
3591 }
3592
3593 static int
3594 eth_igb_syn_filter_get(struct rte_eth_dev *dev,
3595                         struct rte_eth_syn_filter *filter)
3596 {
3597         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3598         uint32_t synqf, rfctl;
3599
3600         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
3601         if (synqf & E1000_SYN_FILTER_ENABLE) {
3602                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
3603                 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0;
3604                 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >>
3605                                 E1000_SYN_FILTER_QUEUE_SHIFT);
3606                 return 0;
3607         }
3608
3609         return -ENOENT;
3610 }
3611
3612 static int
3613 eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
3614                         enum rte_filter_op filter_op,
3615                         void *arg)
3616 {
3617         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3618         int ret;
3619
3620         MAC_TYPE_FILTER_SUP(hw->mac.type);
3621
3622         if (filter_op == RTE_ETH_FILTER_NOP)
3623                 return 0;
3624
3625         if (arg == NULL) {
3626                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
3627                             filter_op);
3628                 return -EINVAL;
3629         }
3630
3631         switch (filter_op) {
3632         case RTE_ETH_FILTER_ADD:
3633                 ret = eth_igb_syn_filter_set(dev,
3634                                 (struct rte_eth_syn_filter *)arg,
3635                                 TRUE);
3636                 break;
3637         case RTE_ETH_FILTER_DELETE:
3638                 ret = eth_igb_syn_filter_set(dev,
3639                                 (struct rte_eth_syn_filter *)arg,
3640                                 FALSE);
3641                 break;
3642         case RTE_ETH_FILTER_GET:
3643                 ret = eth_igb_syn_filter_get(dev,
3644                                 (struct rte_eth_syn_filter *)arg);
3645                 break;
3646         default:
3647                 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op);
3648                 ret = -EINVAL;
3649                 break;
3650         }
3651
3652         return ret;
3653 }
3654
3655 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
3656         if ((type) != e1000_82580 && (type) != e1000_i350)\
3657                 return -ENOSYS; \
3658 } while (0)
3659
3660 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/
3661 static inline int
3662 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter,
3663                         struct e1000_2tuple_filter_info *filter_info)
3664 {
3665         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
3666                 return -EINVAL;
3667         if (filter->priority > E1000_2TUPLE_MAX_PRI)
3668                 return -EINVAL;  /* filter index is out of range. */
3669         if (filter->tcp_flags > TCP_FLAG_ALL)
3670                 return -EINVAL;  /* flags is invalid. */
3671
3672         switch (filter->dst_port_mask) {
3673         case UINT16_MAX:
3674                 filter_info->dst_port_mask = 0;
3675                 filter_info->dst_port = filter->dst_port;
3676                 break;
3677         case 0:
3678                 filter_info->dst_port_mask = 1;
3679                 break;
3680         default:
3681                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
3682                 return -EINVAL;
3683         }
3684
3685         switch (filter->proto_mask) {
3686         case UINT8_MAX:
3687                 filter_info->proto_mask = 0;
3688                 filter_info->proto = filter->proto;
3689                 break;
3690         case 0:
3691                 filter_info->proto_mask = 1;
3692                 break;
3693         default:
3694                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
3695                 return -EINVAL;
3696         }
3697
3698         filter_info->priority = (uint8_t)filter->priority;
3699         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
3700                 filter_info->tcp_flags = filter->tcp_flags;
3701         else
3702                 filter_info->tcp_flags = 0;
3703
3704         return 0;
3705 }
3706
3707 static inline struct e1000_2tuple_filter *
3708 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list,
3709                         struct e1000_2tuple_filter_info *key)
3710 {
3711         struct e1000_2tuple_filter *it;
3712
3713         TAILQ_FOREACH(it, filter_list, entries) {
3714                 if (memcmp(key, &it->filter_info,
3715                         sizeof(struct e1000_2tuple_filter_info)) == 0) {
3716                         return it;
3717                 }
3718         }
3719         return NULL;
3720 }
3721
3722 /*
3723  * igb_add_2tuple_filter - add a 2tuple filter
3724  *
3725  * @param
3726  * dev: Pointer to struct rte_eth_dev.
3727  * ntuple_filter: ponter to the filter that will be added.
3728  *
3729  * @return
3730  *    - On success, zero.
3731  *    - On failure, a negative value.
3732  */
3733 static int
3734 igb_add_2tuple_filter(struct rte_eth_dev *dev,
3735                         struct rte_eth_ntuple_filter *ntuple_filter)
3736 {
3737         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3738         struct e1000_filter_info *filter_info =
3739                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3740         struct e1000_2tuple_filter *filter;
3741         uint32_t ttqf = E1000_TTQF_DISABLE_MASK;
3742         uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP;
3743         int i, ret;
3744
3745         filter = rte_zmalloc("e1000_2tuple_filter",
3746                         sizeof(struct e1000_2tuple_filter), 0);
3747         if (filter == NULL)
3748                 return -ENOMEM;
3749
3750         ret = ntuple_filter_to_2tuple(ntuple_filter,
3751                                       &filter->filter_info);
3752         if (ret < 0) {
3753                 rte_free(filter);
3754                 return ret;
3755         }
3756         if (igb_2tuple_filter_lookup(&filter_info->twotuple_list,
3757                                          &filter->filter_info) != NULL) {
3758                 PMD_DRV_LOG(ERR, "filter exists.");
3759                 rte_free(filter);
3760                 return -EEXIST;
3761         }
3762         filter->queue = ntuple_filter->queue;
3763
3764         /*
3765          * look for an unused 2tuple filter index,
3766          * and insert the filter to list.
3767          */
3768         for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) {
3769                 if (!(filter_info->twotuple_mask & (1 << i))) {
3770                         filter_info->twotuple_mask |= 1 << i;
3771                         filter->index = i;
3772                         TAILQ_INSERT_TAIL(&filter_info->twotuple_list,
3773                                           filter,
3774                                           entries);
3775                         break;
3776                 }
3777         }
3778         if (i >= E1000_MAX_TTQF_FILTERS) {
3779                 PMD_DRV_LOG(ERR, "2tuple filters are full.");
3780                 rte_free(filter);
3781                 return -ENOSYS;
3782         }
3783
3784         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
3785         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
3786                 imir |= E1000_IMIR_PORT_BP;
3787         else
3788                 imir &= ~E1000_IMIR_PORT_BP;
3789
3790         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
3791
3792         ttqf |= E1000_TTQF_QUEUE_ENABLE;
3793         ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT);
3794         ttqf |= (uint32_t)(filter->filter_info.proto & E1000_TTQF_PROTOCOL_MASK);
3795         if (filter->filter_info.proto_mask == 0)
3796                 ttqf &= ~E1000_TTQF_MASK_ENABLE;
3797
3798         /* tcp flags bits setting. */
3799         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
3800                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
3801                         imir_ext |= E1000_IMIREXT_CTRL_URG;
3802                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
3803                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
3804                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
3805                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
3806                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
3807                         imir_ext |= E1000_IMIREXT_CTRL_RST;
3808                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
3809                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
3810                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
3811                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
3812         } else
3813                 imir_ext |= E1000_IMIREXT_CTRL_BP;
3814         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
3815         E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf);
3816         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
3817         return 0;
3818 }
3819
3820 /*
3821  * igb_remove_2tuple_filter - remove a 2tuple filter
3822  *
3823  * @param
3824  * dev: Pointer to struct rte_eth_dev.
3825  * ntuple_filter: ponter to the filter that will be removed.
3826  *
3827  * @return
3828  *    - On success, zero.
3829  *    - On failure, a negative value.
3830  */
3831 static int
3832 igb_remove_2tuple_filter(struct rte_eth_dev *dev,
3833                         struct rte_eth_ntuple_filter *ntuple_filter)
3834 {
3835         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3836         struct e1000_filter_info *filter_info =
3837                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3838         struct e1000_2tuple_filter_info filter_2tuple;
3839         struct e1000_2tuple_filter *filter;
3840         int ret;
3841
3842         memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info));
3843         ret = ntuple_filter_to_2tuple(ntuple_filter,
3844                                       &filter_2tuple);
3845         if (ret < 0)
3846                 return ret;
3847
3848         filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list,
3849                                          &filter_2tuple);
3850         if (filter == NULL) {
3851                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3852                 return -ENOENT;
3853         }
3854
3855         filter_info->twotuple_mask &= ~(1 << filter->index);
3856         TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries);
3857         rte_free(filter);
3858
3859         E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK);
3860         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
3861         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
3862         return 0;
3863 }
3864
3865 static inline struct e1000_flex_filter *
3866 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list,
3867                         struct e1000_flex_filter_info *key)
3868 {
3869         struct e1000_flex_filter *it;
3870
3871         TAILQ_FOREACH(it, filter_list, entries) {
3872                 if (memcmp(key, &it->filter_info,
3873                         sizeof(struct e1000_flex_filter_info)) == 0)
3874                         return it;
3875         }
3876
3877         return NULL;
3878 }
3879
3880 static int
3881 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
3882                         struct rte_eth_flex_filter *filter,
3883                         bool add)
3884 {
3885         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3886         struct e1000_filter_info *filter_info =
3887                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3888         struct e1000_flex_filter *flex_filter, *it;
3889         uint32_t wufc, queueing, mask;
3890         uint32_t reg_off;
3891         uint8_t shift, i, j = 0;
3892
3893         flex_filter = rte_zmalloc("e1000_flex_filter",
3894                         sizeof(struct e1000_flex_filter), 0);
3895         if (flex_filter == NULL)
3896                 return -ENOMEM;
3897
3898         flex_filter->filter_info.len = filter->len;
3899         flex_filter->filter_info.priority = filter->priority;
3900         memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len);
3901         for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) {
3902                 mask = 0;
3903                 /* reverse bits in flex filter's mask*/
3904                 for (shift = 0; shift < CHAR_BIT; shift++) {
3905                         if (filter->mask[i] & (0x01 << shift))
3906                                 mask |= (0x80 >> shift);
3907                 }
3908                 flex_filter->filter_info.mask[i] = mask;
3909         }
3910
3911         wufc = E1000_READ_REG(hw, E1000_WUFC);
3912         if (flex_filter->index < E1000_MAX_FHFT)
3913                 reg_off = E1000_FHFT(flex_filter->index);
3914         else
3915                 reg_off = E1000_FHFT_EXT(flex_filter->index - E1000_MAX_FHFT);
3916
3917         if (add) {
3918                 if (eth_igb_flex_filter_lookup(&filter_info->flex_list,
3919                                 &flex_filter->filter_info) != NULL) {
3920                         PMD_DRV_LOG(ERR, "filter exists.");
3921                         rte_free(flex_filter);
3922                         return -EEXIST;
3923                 }
3924                 flex_filter->queue = filter->queue;
3925                 /*
3926                  * look for an unused flex filter index
3927                  * and insert the filter into the list.
3928                  */
3929                 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) {
3930                         if (!(filter_info->flex_mask & (1 << i))) {
3931                                 filter_info->flex_mask |= 1 << i;
3932                                 flex_filter->index = i;
3933                                 TAILQ_INSERT_TAIL(&filter_info->flex_list,
3934                                         flex_filter,
3935                                         entries);
3936                                 break;
3937                         }
3938                 }
3939                 if (i >= E1000_MAX_FLEX_FILTERS) {
3940                         PMD_DRV_LOG(ERR, "flex filters are full.");
3941                         rte_free(flex_filter);
3942                         return -ENOSYS;
3943                 }
3944
3945                 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ |
3946                                 (E1000_WUFC_FLX0 << flex_filter->index));
3947                 queueing = filter->len |
3948                         (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) |
3949                         (filter->priority << E1000_FHFT_QUEUEING_PRIO_SHIFT);
3950                 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET,
3951                                 queueing);
3952                 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) {
3953                         E1000_WRITE_REG(hw, reg_off,
3954                                         flex_filter->filter_info.dwords[j]);
3955                         reg_off += sizeof(uint32_t);
3956                         E1000_WRITE_REG(hw, reg_off,
3957                                         flex_filter->filter_info.dwords[++j]);
3958                         reg_off += sizeof(uint32_t);
3959                         E1000_WRITE_REG(hw, reg_off,
3960                                 (uint32_t)flex_filter->filter_info.mask[i]);
3961                         reg_off += sizeof(uint32_t) * 2;
3962                         ++j;
3963                 }
3964         } else {
3965                 it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
3966                                 &flex_filter->filter_info);
3967                 if (it == NULL) {
3968                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3969                         rte_free(flex_filter);
3970                         return -ENOENT;
3971                 }
3972
3973                 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++)
3974                         E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0);
3975                 E1000_WRITE_REG(hw, E1000_WUFC, wufc &
3976                         (~(E1000_WUFC_FLX0 << it->index)));
3977
3978                 filter_info->flex_mask &= ~(1 << it->index);
3979                 TAILQ_REMOVE(&filter_info->flex_list, it, entries);
3980                 rte_free(it);
3981                 rte_free(flex_filter);
3982         }
3983
3984         return 0;
3985 }
3986
3987 static int
3988 eth_igb_get_flex_filter(struct rte_eth_dev *dev,
3989                         struct rte_eth_flex_filter *filter)
3990 {
3991         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3992         struct e1000_filter_info *filter_info =
3993                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3994         struct e1000_flex_filter flex_filter, *it;
3995         uint32_t wufc, queueing, wufc_en = 0;
3996
3997         memset(&flex_filter, 0, sizeof(struct e1000_flex_filter));
3998         flex_filter.filter_info.len = filter->len;
3999         flex_filter.filter_info.priority = filter->priority;
4000         memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len);
4001         memcpy(flex_filter.filter_info.mask, filter->mask,
4002                         RTE_ALIGN(filter->len, sizeof(char)) / sizeof(char));
4003
4004         it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
4005                                 &flex_filter.filter_info);
4006         if (it == NULL) {
4007                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
4008                 return -ENOENT;
4009         }
4010
4011         wufc = E1000_READ_REG(hw, E1000_WUFC);
4012         wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index);
4013
4014         if ((wufc & wufc_en) == wufc_en) {
4015                 uint32_t reg_off = 0;
4016                 if (it->index < E1000_MAX_FHFT)
4017                         reg_off = E1000_FHFT(it->index);
4018                 else
4019                         reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT);
4020
4021                 queueing = E1000_READ_REG(hw,
4022                                 reg_off + E1000_FHFT_QUEUEING_OFFSET);
4023                 filter->len = queueing & E1000_FHFT_QUEUEING_LEN;
4024                 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >>
4025                         E1000_FHFT_QUEUEING_PRIO_SHIFT;
4026                 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >>
4027                         E1000_FHFT_QUEUEING_QUEUE_SHIFT;
4028                 return 0;
4029         }
4030         return -ENOENT;
4031 }
4032
4033 static int
4034 eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
4035                         enum rte_filter_op filter_op,
4036                         void *arg)
4037 {
4038         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4039         struct rte_eth_flex_filter *filter;
4040         int ret = 0;
4041
4042         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
4043
4044         if (filter_op == RTE_ETH_FILTER_NOP)
4045                 return ret;
4046
4047         if (arg == NULL) {
4048                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
4049                             filter_op);
4050                 return -EINVAL;
4051         }
4052
4053         filter = (struct rte_eth_flex_filter *)arg;
4054         if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN
4055             || filter->len % sizeof(uint64_t) != 0) {
4056                 PMD_DRV_LOG(ERR, "filter's length is out of range");
4057                 return -EINVAL;
4058         }
4059         if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) {
4060                 PMD_DRV_LOG(ERR, "filter's priority is out of range");
4061                 return -EINVAL;
4062         }
4063
4064         switch (filter_op) {
4065         case RTE_ETH_FILTER_ADD:
4066                 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE);
4067                 break;
4068         case RTE_ETH_FILTER_DELETE:
4069                 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE);
4070                 break;
4071         case RTE_ETH_FILTER_GET:
4072                 ret = eth_igb_get_flex_filter(dev, filter);
4073                 break;
4074         default:
4075                 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op);
4076                 ret = -EINVAL;
4077                 break;
4078         }
4079
4080         return ret;
4081 }
4082
4083 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/
4084 static inline int
4085 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter,
4086                         struct e1000_5tuple_filter_info *filter_info)
4087 {
4088         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576)
4089                 return -EINVAL;
4090         if (filter->priority > E1000_2TUPLE_MAX_PRI)
4091                 return -EINVAL;  /* filter index is out of range. */
4092         if (filter->tcp_flags > TCP_FLAG_ALL)
4093                 return -EINVAL;  /* flags is invalid. */
4094
4095         switch (filter->dst_ip_mask) {
4096         case UINT32_MAX:
4097                 filter_info->dst_ip_mask = 0;
4098                 filter_info->dst_ip = filter->dst_ip;
4099                 break;
4100         case 0:
4101                 filter_info->dst_ip_mask = 1;
4102                 break;
4103         default:
4104                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
4105                 return -EINVAL;
4106         }
4107
4108         switch (filter->src_ip_mask) {
4109         case UINT32_MAX:
4110                 filter_info->src_ip_mask = 0;
4111                 filter_info->src_ip = filter->src_ip;
4112                 break;
4113         case 0:
4114                 filter_info->src_ip_mask = 1;
4115                 break;
4116         default:
4117                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
4118                 return -EINVAL;
4119         }
4120
4121         switch (filter->dst_port_mask) {
4122         case UINT16_MAX:
4123                 filter_info->dst_port_mask = 0;
4124                 filter_info->dst_port = filter->dst_port;
4125                 break;
4126         case 0:
4127                 filter_info->dst_port_mask = 1;
4128                 break;
4129         default:
4130                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
4131                 return -EINVAL;
4132         }
4133
4134         switch (filter->src_port_mask) {
4135         case UINT16_MAX:
4136                 filter_info->src_port_mask = 0;
4137                 filter_info->src_port = filter->src_port;
4138                 break;
4139         case 0:
4140                 filter_info->src_port_mask = 1;
4141                 break;
4142         default:
4143                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
4144                 return -EINVAL;
4145         }
4146
4147         switch (filter->proto_mask) {
4148         case UINT8_MAX:
4149                 filter_info->proto_mask = 0;
4150                 filter_info->proto = filter->proto;
4151                 break;
4152         case 0:
4153                 filter_info->proto_mask = 1;
4154                 break;
4155         default:
4156                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
4157                 return -EINVAL;
4158         }
4159
4160         filter_info->priority = (uint8_t)filter->priority;
4161         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
4162                 filter_info->tcp_flags = filter->tcp_flags;
4163         else
4164                 filter_info->tcp_flags = 0;
4165
4166         return 0;
4167 }
4168
4169 static inline struct e1000_5tuple_filter *
4170 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list,
4171                         struct e1000_5tuple_filter_info *key)
4172 {
4173         struct e1000_5tuple_filter *it;
4174
4175         TAILQ_FOREACH(it, filter_list, entries) {
4176                 if (memcmp(key, &it->filter_info,
4177                         sizeof(struct e1000_5tuple_filter_info)) == 0) {
4178                         return it;
4179                 }
4180         }
4181         return NULL;
4182 }
4183
4184 /*
4185  * igb_add_5tuple_filter_82576 - add a 5tuple filter
4186  *
4187  * @param
4188  * dev: Pointer to struct rte_eth_dev.
4189  * ntuple_filter: ponter to the filter that will be added.
4190  *
4191  * @return
4192  *    - On success, zero.
4193  *    - On failure, a negative value.
4194  */
4195 static int
4196 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
4197                         struct rte_eth_ntuple_filter *ntuple_filter)
4198 {
4199         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4200         struct e1000_filter_info *filter_info =
4201                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4202         struct e1000_5tuple_filter *filter;
4203         uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK;
4204         uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP;
4205         uint8_t i;
4206         int ret;
4207
4208         filter = rte_zmalloc("e1000_5tuple_filter",
4209                         sizeof(struct e1000_5tuple_filter), 0);
4210         if (filter == NULL)
4211                 return -ENOMEM;
4212
4213         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
4214                                             &filter->filter_info);
4215         if (ret < 0) {
4216                 rte_free(filter);
4217                 return ret;
4218         }
4219
4220         if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
4221                                          &filter->filter_info) != NULL) {
4222                 PMD_DRV_LOG(ERR, "filter exists.");
4223                 rte_free(filter);
4224                 return -EEXIST;
4225         }
4226         filter->queue = ntuple_filter->queue;
4227
4228         /*
4229          * look for an unused 5tuple filter index,
4230          * and insert the filter to list.
4231          */
4232         for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) {
4233                 if (!(filter_info->fivetuple_mask & (1 << i))) {
4234                         filter_info->fivetuple_mask |= 1 << i;
4235                         filter->index = i;
4236                         TAILQ_INSERT_TAIL(&filter_info->fivetuple_list,
4237                                           filter,
4238                                           entries);
4239                         break;
4240                 }
4241         }
4242         if (i >= E1000_MAX_FTQF_FILTERS) {
4243                 PMD_DRV_LOG(ERR, "5tuple filters are full.");
4244                 rte_free(filter);
4245                 return -ENOSYS;
4246         }
4247
4248         ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK;
4249         if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */
4250                 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP;
4251         if (filter->filter_info.dst_ip_mask == 0)
4252                 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP;
4253         if (filter->filter_info.src_port_mask == 0)
4254                 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
4255         if (filter->filter_info.proto_mask == 0)
4256                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP;
4257         ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) &
4258                 E1000_FTQF_QUEUE_MASK;
4259         ftqf |= E1000_FTQF_QUEUE_ENABLE;
4260         E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf);
4261         E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip);
4262         E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip);
4263
4264         spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT;
4265         E1000_WRITE_REG(hw, E1000_SPQF(i), spqf);
4266
4267         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
4268         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
4269                 imir |= E1000_IMIR_PORT_BP;
4270         else
4271                 imir &= ~E1000_IMIR_PORT_BP;
4272         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
4273
4274         /* tcp flags bits setting. */
4275         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
4276                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
4277                         imir_ext |= E1000_IMIREXT_CTRL_URG;
4278                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
4279                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
4280                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
4281                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
4282                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
4283                         imir_ext |= E1000_IMIREXT_CTRL_RST;
4284                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
4285                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
4286                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
4287                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
4288         } else
4289                 imir_ext |= E1000_IMIREXT_CTRL_BP;
4290         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
4291         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
4292         return 0;
4293 }
4294
4295 /*
4296  * igb_remove_5tuple_filter_82576 - remove a 5tuple filter
4297  *
4298  * @param
4299  * dev: Pointer to struct rte_eth_dev.
4300  * ntuple_filter: ponter to the filter that will be removed.
4301  *
4302  * @return
4303  *    - On success, zero.
4304  *    - On failure, a negative value.
4305  */
4306 static int
4307 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
4308                                 struct rte_eth_ntuple_filter *ntuple_filter)
4309 {
4310         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4311         struct e1000_filter_info *filter_info =
4312                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4313         struct e1000_5tuple_filter_info filter_5tuple;
4314         struct e1000_5tuple_filter *filter;
4315         int ret;
4316
4317         memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info));
4318         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
4319                                             &filter_5tuple);
4320         if (ret < 0)
4321                 return ret;
4322
4323         filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
4324                                          &filter_5tuple);
4325         if (filter == NULL) {
4326                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
4327                 return -ENOENT;
4328         }
4329
4330         filter_info->fivetuple_mask &= ~(1 << filter->index);
4331         TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries);
4332         rte_free(filter);
4333
4334         E1000_WRITE_REG(hw, E1000_FTQF(filter->index),
4335                         E1000_FTQF_VF_BP | E1000_FTQF_MASK);
4336         E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0);
4337         E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0);
4338         E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0);
4339         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
4340         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
4341         return 0;
4342 }
4343
4344 static int
4345 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
4346 {
4347         uint32_t rctl;
4348         struct e1000_hw *hw;
4349         struct rte_eth_dev_info dev_info;
4350         uint32_t frame_size = mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN +
4351                                      VLAN_TAG_SIZE);
4352
4353         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4354
4355 #ifdef RTE_LIBRTE_82571_SUPPORT
4356         /* XXX: not bigger than max_rx_pktlen */
4357         if (hw->mac.type == e1000_82571)
4358                 return -ENOTSUP;
4359 #endif
4360         eth_igb_infos_get(dev, &dev_info);
4361
4362         /* check that mtu is within the allowed range */
4363         if ((mtu < ETHER_MIN_MTU) ||
4364             (frame_size > dev_info.max_rx_pktlen))
4365                 return -EINVAL;
4366
4367         /* refuse mtu that requires the support of scattered packets when this
4368          * feature has not been enabled before. */
4369         if (!dev->data->scattered_rx &&
4370             frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)
4371                 return -EINVAL;
4372
4373         rctl = E1000_READ_REG(hw, E1000_RCTL);
4374
4375         /* switch to jumbo mode if needed */
4376         if (frame_size > ETHER_MAX_LEN) {
4377                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
4378                 rctl |= E1000_RCTL_LPE;
4379         } else {
4380                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
4381                 rctl &= ~E1000_RCTL_LPE;
4382         }
4383         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4384
4385         /* update max frame size */
4386         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
4387
4388         E1000_WRITE_REG(hw, E1000_RLPML,
4389                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
4390
4391         return 0;
4392 }
4393
4394 /*
4395  * igb_add_del_ntuple_filter - add or delete a ntuple filter
4396  *
4397  * @param
4398  * dev: Pointer to struct rte_eth_dev.
4399  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
4400  * add: if true, add filter, if false, remove filter
4401  *
4402  * @return
4403  *    - On success, zero.
4404  *    - On failure, a negative value.
4405  */
4406 static int
4407 igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
4408                         struct rte_eth_ntuple_filter *ntuple_filter,
4409                         bool add)
4410 {
4411         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4412         int ret;
4413
4414         switch (ntuple_filter->flags) {
4415         case RTE_5TUPLE_FLAGS:
4416         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
4417                 if (hw->mac.type != e1000_82576)
4418                         return -ENOTSUP;
4419                 if (add)
4420                         ret = igb_add_5tuple_filter_82576(dev,
4421                                                           ntuple_filter);
4422                 else
4423                         ret = igb_remove_5tuple_filter_82576(dev,
4424                                                              ntuple_filter);
4425                 break;
4426         case RTE_2TUPLE_FLAGS:
4427         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
4428                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
4429                         return -ENOTSUP;
4430                 if (add)
4431                         ret = igb_add_2tuple_filter(dev, ntuple_filter);
4432                 else
4433                         ret = igb_remove_2tuple_filter(dev, ntuple_filter);
4434                 break;
4435         default:
4436                 ret = -EINVAL;
4437                 break;
4438         }
4439
4440         return ret;
4441 }
4442
4443 /*
4444  * igb_get_ntuple_filter - get a ntuple filter
4445  *
4446  * @param
4447  * dev: Pointer to struct rte_eth_dev.
4448  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
4449  *
4450  * @return
4451  *    - On success, zero.
4452  *    - On failure, a negative value.
4453  */
4454 static int
4455 igb_get_ntuple_filter(struct rte_eth_dev *dev,
4456                         struct rte_eth_ntuple_filter *ntuple_filter)
4457 {
4458         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4459         struct e1000_filter_info *filter_info =
4460                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4461         struct e1000_5tuple_filter_info filter_5tuple;
4462         struct e1000_2tuple_filter_info filter_2tuple;
4463         struct e1000_5tuple_filter *p_5tuple_filter;
4464         struct e1000_2tuple_filter *p_2tuple_filter;
4465         int ret;
4466
4467         switch (ntuple_filter->flags) {
4468         case RTE_5TUPLE_FLAGS:
4469         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
4470                 if (hw->mac.type != e1000_82576)
4471                         return -ENOTSUP;
4472                 memset(&filter_5tuple,
4473                         0,
4474                         sizeof(struct e1000_5tuple_filter_info));
4475                 ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
4476                                                     &filter_5tuple);
4477                 if (ret < 0)
4478                         return ret;
4479                 p_5tuple_filter = igb_5tuple_filter_lookup_82576(
4480                                         &filter_info->fivetuple_list,
4481                                         &filter_5tuple);
4482                 if (p_5tuple_filter == NULL) {
4483                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
4484                         return -ENOENT;
4485                 }
4486                 ntuple_filter->queue = p_5tuple_filter->queue;
4487                 break;
4488         case RTE_2TUPLE_FLAGS:
4489         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
4490                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
4491                         return -ENOTSUP;
4492                 memset(&filter_2tuple,
4493                         0,
4494                         sizeof(struct e1000_2tuple_filter_info));
4495                 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple);
4496                 if (ret < 0)
4497                         return ret;
4498                 p_2tuple_filter = igb_2tuple_filter_lookup(
4499                                         &filter_info->twotuple_list,
4500                                         &filter_2tuple);
4501                 if (p_2tuple_filter == NULL) {
4502                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
4503                         return -ENOENT;
4504                 }
4505                 ntuple_filter->queue = p_2tuple_filter->queue;
4506                 break;
4507         default:
4508                 ret = -EINVAL;
4509                 break;
4510         }
4511
4512         return 0;
4513 }
4514
4515 /*
4516  * igb_ntuple_filter_handle - Handle operations for ntuple filter.
4517  * @dev: pointer to rte_eth_dev structure
4518  * @filter_op:operation will be taken.
4519  * @arg: a pointer to specific structure corresponding to the filter_op
4520  */
4521 static int
4522 igb_ntuple_filter_handle(struct rte_eth_dev *dev,
4523                                 enum rte_filter_op filter_op,
4524                                 void *arg)
4525 {
4526         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4527         int ret;
4528
4529         MAC_TYPE_FILTER_SUP(hw->mac.type);
4530
4531         if (filter_op == RTE_ETH_FILTER_NOP)
4532                 return 0;
4533
4534         if (arg == NULL) {
4535                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
4536                             filter_op);
4537                 return -EINVAL;
4538         }
4539
4540         switch (filter_op) {
4541         case RTE_ETH_FILTER_ADD:
4542                 ret = igb_add_del_ntuple_filter(dev,
4543                         (struct rte_eth_ntuple_filter *)arg,
4544                         TRUE);
4545                 break;
4546         case RTE_ETH_FILTER_DELETE:
4547                 ret = igb_add_del_ntuple_filter(dev,
4548                         (struct rte_eth_ntuple_filter *)arg,
4549                         FALSE);
4550                 break;
4551         case RTE_ETH_FILTER_GET:
4552                 ret = igb_get_ntuple_filter(dev,
4553                         (struct rte_eth_ntuple_filter *)arg);
4554                 break;
4555         default:
4556                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
4557                 ret = -EINVAL;
4558                 break;
4559         }
4560         return ret;
4561 }
4562
4563 static inline int
4564 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info,
4565                         uint16_t ethertype)
4566 {
4567         int i;
4568
4569         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
4570                 if (filter_info->ethertype_filters[i] == ethertype &&
4571                     (filter_info->ethertype_mask & (1 << i)))
4572                         return i;
4573         }
4574         return -1;
4575 }
4576
4577 static inline int
4578 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info,
4579                         uint16_t ethertype)
4580 {
4581         int i;
4582
4583         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
4584                 if (!(filter_info->ethertype_mask & (1 << i))) {
4585                         filter_info->ethertype_mask |= 1 << i;
4586                         filter_info->ethertype_filters[i] = ethertype;
4587                         return i;
4588                 }
4589         }
4590         return -1;
4591 }
4592
4593 static inline int
4594 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info,
4595                         uint8_t idx)
4596 {
4597         if (idx >= E1000_MAX_ETQF_FILTERS)
4598                 return -1;
4599         filter_info->ethertype_mask &= ~(1 << idx);
4600         filter_info->ethertype_filters[idx] = 0;
4601         return idx;
4602 }
4603
4604
4605 static int
4606 igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
4607                         struct rte_eth_ethertype_filter *filter,
4608                         bool add)
4609 {
4610         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4611         struct e1000_filter_info *filter_info =
4612                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4613         uint32_t etqf = 0;
4614         int ret;
4615
4616         if (filter->ether_type == ETHER_TYPE_IPv4 ||
4617                 filter->ether_type == ETHER_TYPE_IPv6) {
4618                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
4619                         " ethertype filter.", filter->ether_type);
4620                 return -EINVAL;
4621         }
4622
4623         if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) {
4624                 PMD_DRV_LOG(ERR, "mac compare is unsupported.");
4625                 return -EINVAL;
4626         }
4627         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) {
4628                 PMD_DRV_LOG(ERR, "drop option is unsupported.");
4629                 return -EINVAL;
4630         }
4631
4632         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
4633         if (ret >= 0 && add) {
4634                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.",
4635                             filter->ether_type);
4636                 return -EEXIST;
4637         }
4638         if (ret < 0 && !add) {
4639                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
4640                             filter->ether_type);
4641                 return -ENOENT;
4642         }
4643
4644         if (add) {
4645                 ret = igb_ethertype_filter_insert(filter_info,
4646                         filter->ether_type);
4647                 if (ret < 0) {
4648                         PMD_DRV_LOG(ERR, "ethertype filters are full.");
4649                         return -ENOSYS;
4650                 }
4651
4652                 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE;
4653                 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE);
4654                 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT;
4655         } else {
4656                 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret);
4657                 if (ret < 0)
4658                         return -ENOSYS;
4659         }
4660         E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf);
4661         E1000_WRITE_FLUSH(hw);
4662
4663         return 0;
4664 }
4665
4666 static int
4667 igb_get_ethertype_filter(struct rte_eth_dev *dev,
4668                         struct rte_eth_ethertype_filter *filter)
4669 {
4670         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4671         struct e1000_filter_info *filter_info =
4672                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
4673         uint32_t etqf;
4674         int ret;
4675
4676         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
4677         if (ret < 0) {
4678                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
4679                             filter->ether_type);
4680                 return -ENOENT;
4681         }
4682
4683         etqf = E1000_READ_REG(hw, E1000_ETQF(ret));
4684         if (etqf & E1000_ETQF_FILTER_ENABLE) {
4685                 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE;
4686                 filter->flags = 0;
4687                 filter->queue = (etqf & E1000_ETQF_QUEUE) >>
4688                                 E1000_ETQF_QUEUE_SHIFT;
4689                 return 0;
4690         }
4691
4692         return -ENOENT;
4693 }
4694
4695 /*
4696  * igb_ethertype_filter_handle - Handle operations for ethertype filter.
4697  * @dev: pointer to rte_eth_dev structure
4698  * @filter_op:operation will be taken.
4699  * @arg: a pointer to specific structure corresponding to the filter_op
4700  */
4701 static int
4702 igb_ethertype_filter_handle(struct rte_eth_dev *dev,
4703                                 enum rte_filter_op filter_op,
4704                                 void *arg)
4705 {
4706         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4707         int ret;
4708
4709         MAC_TYPE_FILTER_SUP(hw->mac.type);
4710
4711         if (filter_op == RTE_ETH_FILTER_NOP)
4712                 return 0;
4713
4714         if (arg == NULL) {
4715                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
4716                             filter_op);
4717                 return -EINVAL;
4718         }
4719
4720         switch (filter_op) {
4721         case RTE_ETH_FILTER_ADD:
4722                 ret = igb_add_del_ethertype_filter(dev,
4723                         (struct rte_eth_ethertype_filter *)arg,
4724                         TRUE);
4725                 break;
4726         case RTE_ETH_FILTER_DELETE:
4727                 ret = igb_add_del_ethertype_filter(dev,
4728                         (struct rte_eth_ethertype_filter *)arg,
4729                         FALSE);
4730                 break;
4731         case RTE_ETH_FILTER_GET:
4732                 ret = igb_get_ethertype_filter(dev,
4733                         (struct rte_eth_ethertype_filter *)arg);
4734                 break;
4735         default:
4736                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
4737                 ret = -EINVAL;
4738                 break;
4739         }
4740         return ret;
4741 }
4742
4743 static int
4744 eth_igb_filter_ctrl(struct rte_eth_dev *dev,
4745                      enum rte_filter_type filter_type,
4746                      enum rte_filter_op filter_op,
4747                      void *arg)
4748 {
4749         int ret = -EINVAL;
4750
4751         switch (filter_type) {
4752         case RTE_ETH_FILTER_NTUPLE:
4753                 ret = igb_ntuple_filter_handle(dev, filter_op, arg);
4754                 break;
4755         case RTE_ETH_FILTER_ETHERTYPE:
4756                 ret = igb_ethertype_filter_handle(dev, filter_op, arg);
4757                 break;
4758         case RTE_ETH_FILTER_SYN:
4759                 ret = eth_igb_syn_filter_handle(dev, filter_op, arg);
4760                 break;
4761         case RTE_ETH_FILTER_FLEXIBLE:
4762                 ret = eth_igb_flex_filter_handle(dev, filter_op, arg);
4763                 break;
4764         default:
4765                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
4766                                                         filter_type);
4767                 break;
4768         }
4769
4770         return ret;
4771 }
4772
4773 static int
4774 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
4775                          struct ether_addr *mc_addr_set,
4776                          uint32_t nb_mc_addr)
4777 {
4778         struct e1000_hw *hw;
4779
4780         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4781         e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr);
4782         return 0;
4783 }
4784
4785 static uint64_t
4786 igb_read_systime_cyclecounter(struct rte_eth_dev *dev)
4787 {
4788         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4789         uint64_t systime_cycles;
4790
4791         switch (hw->mac.type) {
4792         case e1000_i210:
4793         case e1000_i211:
4794                 /*
4795                  * Need to read System Time Residue Register to be able
4796                  * to read the other two registers.
4797                  */
4798                 E1000_READ_REG(hw, E1000_SYSTIMR);
4799                 /* SYSTIMEL stores ns and SYSTIMEH stores seconds. */
4800                 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML);
4801                 systime_cycles += (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH)
4802                                 * NSEC_PER_SEC;
4803                 break;
4804         case e1000_82580:
4805         case e1000_i350:
4806         case e1000_i354:
4807                 /*
4808                  * Need to read System Time Residue Register to be able
4809                  * to read the other two registers.
4810                  */
4811                 E1000_READ_REG(hw, E1000_SYSTIMR);
4812                 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML);
4813                 /* Only the 8 LSB are valid. */
4814                 systime_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_SYSTIMH)
4815                                 & 0xff) << 32;
4816                 break;
4817         default:
4818                 systime_cycles = (uint64_t)E1000_READ_REG(hw, E1000_SYSTIML);
4819                 systime_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_SYSTIMH)
4820                                 << 32;
4821                 break;
4822         }
4823
4824         return systime_cycles;
4825 }
4826
4827 static uint64_t
4828 igb_read_rx_tstamp_cyclecounter(struct rte_eth_dev *dev)
4829 {
4830         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4831         uint64_t rx_tstamp_cycles;
4832
4833         switch (hw->mac.type) {
4834         case e1000_i210:
4835         case e1000_i211:
4836                 /* RXSTMPL stores ns and RXSTMPH stores seconds. */
4837                 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL);
4838                 rx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH)
4839                                 * NSEC_PER_SEC;
4840                 break;
4841         case e1000_82580:
4842         case e1000_i350:
4843         case e1000_i354:
4844                 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL);
4845                 /* Only the 8 LSB are valid. */
4846                 rx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_RXSTMPH)
4847                                 & 0xff) << 32;
4848                 break;
4849         default:
4850                 rx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPL);
4851                 rx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_RXSTMPH)
4852                                 << 32;
4853                 break;
4854         }
4855
4856         return rx_tstamp_cycles;
4857 }
4858
4859 static uint64_t
4860 igb_read_tx_tstamp_cyclecounter(struct rte_eth_dev *dev)
4861 {
4862         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4863         uint64_t tx_tstamp_cycles;
4864
4865         switch (hw->mac.type) {
4866         case e1000_i210:
4867         case e1000_i211:
4868                 /* RXSTMPL stores ns and RXSTMPH stores seconds. */
4869                 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL);
4870                 tx_tstamp_cycles += (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH)
4871                                 * NSEC_PER_SEC;
4872                 break;
4873         case e1000_82580:
4874         case e1000_i350:
4875         case e1000_i354:
4876                 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL);
4877                 /* Only the 8 LSB are valid. */
4878                 tx_tstamp_cycles |= (uint64_t)(E1000_READ_REG(hw, E1000_TXSTMPH)
4879                                 & 0xff) << 32;
4880                 break;
4881         default:
4882                 tx_tstamp_cycles = (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPL);
4883                 tx_tstamp_cycles |= (uint64_t)E1000_READ_REG(hw, E1000_TXSTMPH)
4884                                 << 32;
4885                 break;
4886         }
4887
4888         return tx_tstamp_cycles;
4889 }
4890
4891 static void
4892 igb_start_timecounters(struct rte_eth_dev *dev)
4893 {
4894         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4895         struct e1000_adapter *adapter =
4896                 (struct e1000_adapter *)dev->data->dev_private;
4897         uint32_t incval = 1;
4898         uint32_t shift = 0;
4899         uint64_t mask = E1000_CYCLECOUNTER_MASK;
4900
4901         switch (hw->mac.type) {
4902         case e1000_82580:
4903         case e1000_i350:
4904         case e1000_i354:
4905                 /* 32 LSB bits + 8 MSB bits = 40 bits */
4906                 mask = (1ULL << 40) - 1;
4907                 /* fall-through */
4908         case e1000_i210:
4909         case e1000_i211:
4910                 /*
4911                  * Start incrementing the register
4912                  * used to timestamp PTP packets.
4913                  */
4914                 E1000_WRITE_REG(hw, E1000_TIMINCA, incval);
4915                 break;
4916         case e1000_82576:
4917                 incval = E1000_INCVALUE_82576;
4918                 shift = IGB_82576_TSYNC_SHIFT;
4919                 E1000_WRITE_REG(hw, E1000_TIMINCA,
4920                                 E1000_INCPERIOD_82576 | incval);
4921                 break;
4922         default:
4923                 /* Not supported */
4924                 return;
4925         }
4926
4927         memset(&adapter->systime_tc, 0, sizeof(struct rte_timecounter));
4928         memset(&adapter->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
4929         memset(&adapter->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
4930
4931         adapter->systime_tc.cc_mask = mask;
4932         adapter->systime_tc.cc_shift = shift;
4933         adapter->systime_tc.nsec_mask = (1ULL << shift) - 1;
4934
4935         adapter->rx_tstamp_tc.cc_mask = mask;
4936         adapter->rx_tstamp_tc.cc_shift = shift;
4937         adapter->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
4938
4939         adapter->tx_tstamp_tc.cc_mask = mask;
4940         adapter->tx_tstamp_tc.cc_shift = shift;
4941         adapter->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
4942 }
4943
4944 static int
4945 igb_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
4946 {
4947         struct e1000_adapter *adapter =
4948                         (struct e1000_adapter *)dev->data->dev_private;
4949
4950         adapter->systime_tc.nsec += delta;
4951         adapter->rx_tstamp_tc.nsec += delta;
4952         adapter->tx_tstamp_tc.nsec += delta;
4953
4954         return 0;
4955 }
4956
4957 static int
4958 igb_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
4959 {
4960         uint64_t ns;
4961         struct e1000_adapter *adapter =
4962                         (struct e1000_adapter *)dev->data->dev_private;
4963
4964         ns = rte_timespec_to_ns(ts);
4965
4966         /* Set the timecounters to a new value. */
4967         adapter->systime_tc.nsec = ns;
4968         adapter->rx_tstamp_tc.nsec = ns;
4969         adapter->tx_tstamp_tc.nsec = ns;
4970
4971         return 0;
4972 }
4973
4974 static int
4975 igb_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
4976 {
4977         uint64_t ns, systime_cycles;
4978         struct e1000_adapter *adapter =
4979                         (struct e1000_adapter *)dev->data->dev_private;
4980
4981         systime_cycles = igb_read_systime_cyclecounter(dev);
4982         ns = rte_timecounter_update(&adapter->systime_tc, systime_cycles);
4983         *ts = rte_ns_to_timespec(ns);
4984
4985         return 0;
4986 }
4987
4988 static int
4989 igb_timesync_enable(struct rte_eth_dev *dev)
4990 {
4991         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4992         uint32_t tsync_ctl;
4993         uint32_t tsauxc;
4994
4995         /* Stop the timesync system time. */
4996         E1000_WRITE_REG(hw, E1000_TIMINCA, 0x0);
4997         /* Reset the timesync system time value. */
4998         switch (hw->mac.type) {
4999         case e1000_82580:
5000         case e1000_i350:
5001         case e1000_i354:
5002         case e1000_i210:
5003         case e1000_i211:
5004                 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x0);
5005                 /* fall-through */
5006         case e1000_82576:
5007                 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x0);
5008                 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x0);
5009                 break;
5010         default:
5011                 /* Not supported. */
5012                 return -ENOTSUP;
5013         }
5014
5015         /* Enable system time for it isn't on by default. */
5016         tsauxc = E1000_READ_REG(hw, E1000_TSAUXC);
5017         tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME;
5018         E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc);
5019
5020         igb_start_timecounters(dev);
5021
5022         /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
5023         E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588),
5024                         (ETHER_TYPE_1588 |
5025                          E1000_ETQF_FILTER_ENABLE |
5026                          E1000_ETQF_1588));
5027
5028         /* Enable timestamping of received PTP packets. */
5029         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
5030         tsync_ctl |= E1000_TSYNCRXCTL_ENABLED;
5031         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl);
5032
5033         /* Enable Timestamping of transmitted PTP packets. */
5034         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
5035         tsync_ctl |= E1000_TSYNCTXCTL_ENABLED;
5036         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl);
5037
5038         return 0;
5039 }
5040
5041 static int
5042 igb_timesync_disable(struct rte_eth_dev *dev)
5043 {
5044         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5045         uint32_t tsync_ctl;
5046
5047         /* Disable timestamping of transmitted PTP packets. */
5048         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
5049         tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED;
5050         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl);
5051
5052         /* Disable timestamping of received PTP packets. */
5053         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
5054         tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED;
5055         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl);
5056
5057         /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
5058         E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0);
5059
5060         /* Stop incrementating the System Time registers. */
5061         E1000_WRITE_REG(hw, E1000_TIMINCA, 0);
5062
5063         return 0;
5064 }
5065
5066 static int
5067 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
5068                                struct timespec *timestamp,
5069                                uint32_t flags __rte_unused)
5070 {
5071         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5072         struct e1000_adapter *adapter =
5073                         (struct e1000_adapter *)dev->data->dev_private;
5074         uint32_t tsync_rxctl;
5075         uint64_t rx_tstamp_cycles;
5076         uint64_t ns;
5077
5078         tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
5079         if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0)
5080                 return -EINVAL;
5081
5082         rx_tstamp_cycles = igb_read_rx_tstamp_cyclecounter(dev);
5083         ns = rte_timecounter_update(&adapter->rx_tstamp_tc, rx_tstamp_cycles);
5084         *timestamp = rte_ns_to_timespec(ns);
5085
5086         return  0;
5087 }
5088
5089 static int
5090 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
5091                                struct timespec *timestamp)
5092 {
5093         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5094         struct e1000_adapter *adapter =
5095                         (struct e1000_adapter *)dev->data->dev_private;
5096         uint32_t tsync_txctl;
5097         uint64_t tx_tstamp_cycles;
5098         uint64_t ns;
5099
5100         tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
5101         if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0)
5102                 return -EINVAL;
5103
5104         tx_tstamp_cycles = igb_read_tx_tstamp_cyclecounter(dev);
5105         ns = rte_timecounter_update(&adapter->tx_tstamp_tc, tx_tstamp_cycles);
5106         *timestamp = rte_ns_to_timespec(ns);
5107
5108         return  0;
5109 }
5110
5111 static int
5112 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused)
5113 {
5114         int count = 0;
5115         int g_ind = 0;
5116         const struct reg_info *reg_group;
5117
5118         while ((reg_group = igb_regs[g_ind++]))
5119                 count += igb_reg_group_count(reg_group);
5120
5121         return count;
5122 }
5123
5124 static int
5125 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused)
5126 {
5127         int count = 0;
5128         int g_ind = 0;
5129         const struct reg_info *reg_group;
5130
5131         while ((reg_group = igbvf_regs[g_ind++]))
5132                 count += igb_reg_group_count(reg_group);
5133
5134         return count;
5135 }
5136
5137 static int
5138 eth_igb_get_regs(struct rte_eth_dev *dev,
5139         struct rte_dev_reg_info *regs)
5140 {
5141         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5142         uint32_t *data = regs->data;
5143         int g_ind = 0;
5144         int count = 0;
5145         const struct reg_info *reg_group;
5146
5147         if (data == NULL) {
5148                 regs->length = eth_igb_get_reg_length(dev);
5149                 regs->width = sizeof(uint32_t);
5150                 return 0;
5151         }
5152
5153         /* Support only full register dump */
5154         if ((regs->length == 0) ||
5155             (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) {
5156                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
5157                         hw->device_id;
5158                 while ((reg_group = igb_regs[g_ind++]))
5159                         count += igb_read_regs_group(dev, &data[count],
5160                                                         reg_group);
5161                 return 0;
5162         }
5163
5164         return -ENOTSUP;
5165 }
5166
5167 static int
5168 igbvf_get_regs(struct rte_eth_dev *dev,
5169         struct rte_dev_reg_info *regs)
5170 {
5171         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5172         uint32_t *data = regs->data;
5173         int g_ind = 0;
5174         int count = 0;
5175         const struct reg_info *reg_group;
5176
5177         if (data == NULL) {
5178                 regs->length = igbvf_get_reg_length(dev);
5179                 regs->width = sizeof(uint32_t);
5180                 return 0;
5181         }
5182
5183         /* Support only full register dump */
5184         if ((regs->length == 0) ||
5185             (regs->length == (uint32_t)igbvf_get_reg_length(dev))) {
5186                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
5187                         hw->device_id;
5188                 while ((reg_group = igbvf_regs[g_ind++]))
5189                         count += igb_read_regs_group(dev, &data[count],
5190                                                         reg_group);
5191                 return 0;
5192         }
5193
5194         return -ENOTSUP;
5195 }
5196
5197 static int
5198 eth_igb_get_eeprom_length(struct rte_eth_dev *dev)
5199 {
5200         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5201
5202         /* Return unit is byte count */
5203         return hw->nvm.word_size * 2;
5204 }
5205
5206 static int
5207 eth_igb_get_eeprom(struct rte_eth_dev *dev,
5208         struct rte_dev_eeprom_info *in_eeprom)
5209 {
5210         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5211         struct e1000_nvm_info *nvm = &hw->nvm;
5212         uint16_t *data = in_eeprom->data;
5213         int first, length;
5214
5215         first = in_eeprom->offset >> 1;
5216         length = in_eeprom->length >> 1;
5217         if ((first >= hw->nvm.word_size) ||
5218             ((first + length) >= hw->nvm.word_size))
5219                 return -EINVAL;
5220
5221         in_eeprom->magic = hw->vendor_id |
5222                 ((uint32_t)hw->device_id << 16);
5223
5224         if ((nvm->ops.read) == NULL)
5225                 return -ENOTSUP;
5226
5227         return nvm->ops.read(hw, first, length, data);
5228 }
5229
5230 static int
5231 eth_igb_set_eeprom(struct rte_eth_dev *dev,
5232         struct rte_dev_eeprom_info *in_eeprom)
5233 {
5234         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5235         struct e1000_nvm_info *nvm = &hw->nvm;
5236         uint16_t *data = in_eeprom->data;
5237         int first, length;
5238
5239         first = in_eeprom->offset >> 1;
5240         length = in_eeprom->length >> 1;
5241         if ((first >= hw->nvm.word_size) ||
5242             ((first + length) >= hw->nvm.word_size))
5243                 return -EINVAL;
5244
5245         in_eeprom->magic = (uint32_t)hw->vendor_id |
5246                 ((uint32_t)hw->device_id << 16);
5247
5248         if ((nvm->ops.write) == NULL)
5249                 return -ENOTSUP;
5250         return nvm->ops.write(hw,  first, length, data);
5251 }
5252
5253 static int
5254 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
5255 {
5256         struct e1000_hw *hw =
5257                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5258         uint32_t mask = 1 << queue_id;
5259
5260         E1000_WRITE_REG(hw, E1000_EIMC, mask);
5261         E1000_WRITE_FLUSH(hw);
5262
5263         return 0;
5264 }
5265
5266 static int
5267 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
5268 {
5269         struct e1000_hw *hw =
5270                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5271         struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev);
5272         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
5273         uint32_t mask = 1 << queue_id;
5274         uint32_t regval;
5275
5276         regval = E1000_READ_REG(hw, E1000_EIMS);
5277         E1000_WRITE_REG(hw, E1000_EIMS, regval | mask);
5278         E1000_WRITE_FLUSH(hw);
5279
5280         rte_intr_enable(intr_handle);
5281
5282         return 0;
5283 }
5284
5285 static void
5286 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t  msix_vector,
5287                    uint8_t index, uint8_t offset)
5288 {
5289         uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
5290
5291         /* clear bits */
5292         val &= ~((uint32_t)0xFF << offset);
5293
5294         /* write vector and valid bit */
5295         val |= (msix_vector | E1000_IVAR_VALID) << offset;
5296
5297         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val);
5298 }
5299
5300 static void
5301 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction,
5302                            uint8_t queue, uint8_t msix_vector)
5303 {
5304         uint32_t tmp = 0;
5305
5306         if (hw->mac.type == e1000_82575) {
5307                 if (direction == 0)
5308                         tmp = E1000_EICR_RX_QUEUE0 << queue;
5309                 else if (direction == 1)
5310                         tmp = E1000_EICR_TX_QUEUE0 << queue;
5311                 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp);
5312         } else if (hw->mac.type == e1000_82576) {
5313                 if ((direction == 0) || (direction == 1))
5314                         eth_igb_write_ivar(hw, msix_vector, queue & 0x7,
5315                                            ((queue & 0x8) << 1) +
5316                                            8 * direction);
5317         } else if ((hw->mac.type == e1000_82580) ||
5318                         (hw->mac.type == e1000_i350) ||
5319                         (hw->mac.type == e1000_i354) ||
5320                         (hw->mac.type == e1000_i210) ||
5321                         (hw->mac.type == e1000_i211)) {
5322                 if ((direction == 0) || (direction == 1))
5323                         eth_igb_write_ivar(hw, msix_vector,
5324                                            queue >> 1,
5325                                            ((queue & 0x1) << 4) +
5326                                            8 * direction);
5327         }
5328 }
5329
5330 /* Sets up the hardware to generate MSI-X interrupts properly
5331  * @hw
5332  *  board private structure
5333  */
5334 static void
5335 eth_igb_configure_msix_intr(struct rte_eth_dev *dev)
5336 {
5337         int queue_id;
5338         uint32_t tmpval, regval, intr_mask;
5339         struct e1000_hw *hw =
5340                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5341         uint32_t vec = E1000_MISC_VEC_ID;
5342         uint32_t base = E1000_MISC_VEC_ID;
5343         uint32_t misc_shift = 0;
5344         struct rte_pci_device *pci_dev = E1000_DEV_TO_PCI(dev);
5345         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
5346
5347         /* won't configure msix register if no mapping is done
5348          * between intr vector and event fd
5349          */
5350         if (!rte_intr_dp_is_en(intr_handle))
5351                 return;
5352
5353         if (rte_intr_allow_others(intr_handle)) {
5354                 vec = base = E1000_RX_VEC_START;
5355                 misc_shift = 1;
5356         }
5357
5358         /* set interrupt vector for other causes */
5359         if (hw->mac.type == e1000_82575) {
5360                 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT);
5361                 /* enable MSI-X PBA support */
5362                 tmpval |= E1000_CTRL_EXT_PBA_CLR;
5363
5364                 /* Auto-Mask interrupts upon ICR read */
5365                 tmpval |= E1000_CTRL_EXT_EIAME;
5366                 tmpval |= E1000_CTRL_EXT_IRCA;
5367
5368                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval);
5369
5370                 /* enable msix_other interrupt */
5371                 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER);
5372                 regval = E1000_READ_REG(hw, E1000_EIAC);
5373                 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER);
5374                 regval = E1000_READ_REG(hw, E1000_EIAM);
5375                 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER);
5376         } else if ((hw->mac.type == e1000_82576) ||
5377                         (hw->mac.type == e1000_82580) ||
5378                         (hw->mac.type == e1000_i350) ||
5379                         (hw->mac.type == e1000_i354) ||
5380                         (hw->mac.type == e1000_i210) ||
5381                         (hw->mac.type == e1000_i211)) {
5382                 /* turn on MSI-X capability first */
5383                 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE |
5384                                         E1000_GPIE_PBA | E1000_GPIE_EIAME |
5385                                         E1000_GPIE_NSICR);
5386                 intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) <<
5387                         misc_shift;
5388                 regval = E1000_READ_REG(hw, E1000_EIAC);
5389                 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask);
5390
5391                 /* enable msix_other interrupt */
5392                 regval = E1000_READ_REG(hw, E1000_EIMS);
5393                 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask);
5394                 tmpval = (dev->data->nb_rx_queues | E1000_IVAR_VALID) << 8;
5395                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval);
5396         }
5397
5398         /* use EIAM to auto-mask when MSI-X interrupt
5399          * is asserted, this saves a register write for every interrupt
5400          */
5401         intr_mask = RTE_LEN2MASK(intr_handle->nb_efd, uint32_t) <<
5402                 misc_shift;
5403         regval = E1000_READ_REG(hw, E1000_EIAM);
5404         E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask);
5405
5406         for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) {
5407                 eth_igb_assign_msix_vector(hw, 0, queue_id, vec);
5408                 intr_handle->intr_vec[queue_id] = vec;
5409                 if (vec < base + intr_handle->nb_efd - 1)
5410                         vec++;
5411         }
5412
5413         E1000_WRITE_FLUSH(hw);
5414 }
5415
5416 RTE_PMD_REGISTER_PCI(net_e1000_igb, rte_igb_pmd);
5417 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb, pci_id_igb_map);
5418 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb, "* igb_uio | uio_pci_generic | vfio");
5419 RTE_PMD_REGISTER_PCI(net_e1000_igb_vf, rte_igbvf_pmd);
5420 RTE_PMD_REGISTER_PCI_TABLE(net_e1000_igb_vf, pci_id_igbvf_map);
5421 RTE_PMD_REGISTER_KMOD_DEP(net_e1000_igb_vf, "* igb_uio | vfio");