drivers/net/e1000/igb_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <stdarg.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_interrupts.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_log.h>
  44 #include <rte_debug.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_memory.h>
  49 #include <rte_memzone.h>
  50 #include <rte_eal.h>
  51 #include <rte_atomic.h>
  52 #include <rte_malloc.h>
  53 #include <rte_dev.h>
  54
  55 #include "e1000_logs.h"
  56 #include "base/e1000_api.h"
  57 #include "e1000_ethdev.h"
  58 #include "igb_regs.h"
  59
  60 /*
  61  * Default values for port configuration
  62  */
  63 #define IGB_DEFAULT_RX_FREE_THRESH  32
  64 #define IGB_DEFAULT_RX_PTHRESH      8
  65 #define IGB_DEFAULT_RX_HTHRESH      8
  66 #define IGB_DEFAULT_RX_WTHRESH      0
  67
  68 #define IGB_DEFAULT_TX_PTHRESH      32
  69 #define IGB_DEFAULT_TX_HTHRESH      0
  70 #define IGB_DEFAULT_TX_WTHRESH      0
  71
  72 #define IGB_HKEY_MAX_INDEX 10
  73
  74 /* Bit shift and mask */
  75 #define IGB_4_BIT_WIDTH  (CHAR_BIT / 2)
  76 #define IGB_4_BIT_MASK   RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t)
  77 #define IGB_8_BIT_WIDTH  CHAR_BIT
  78 #define IGB_8_BIT_MASK   UINT8_MAX
  79
  80 /* Additional timesync values. */
  81 #define E1000_ETQF_FILTER_1588 3
  82 #define E1000_TIMINCA_INCVALUE 16000000
  83 #define E1000_TIMINCA_INIT     ((0x02 << E1000_TIMINCA_16NS_SHIFT) \
  84                                 | E1000_TIMINCA_INCVALUE)
  85 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000
  86
  87 static int  eth_igb_configure(struct rte_eth_dev *dev);
  88 static int  eth_igb_start(struct rte_eth_dev *dev);
  89 static void eth_igb_stop(struct rte_eth_dev *dev);
  90 static void eth_igb_close(struct rte_eth_dev *dev);
  91 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev);
  92 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev);
  93 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev);
  94 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev);
  95 static int  eth_igb_link_update(struct rte_eth_dev *dev,
  96                                 int wait_to_complete);
  97 static void eth_igb_stats_get(struct rte_eth_dev *dev,
  98                                 struct rte_eth_stats *rte_stats);
  99 static void eth_igb_stats_reset(struct rte_eth_dev *dev);
 100 static void eth_igb_infos_get(struct rte_eth_dev *dev,
 101                               struct rte_eth_dev_info *dev_info);
 102 static void eth_igbvf_infos_get(struct rte_eth_dev *dev,
 103                                 struct rte_eth_dev_info *dev_info);
 104 static int  eth_igb_flow_ctrl_get(struct rte_eth_dev *dev,
 105                                 struct rte_eth_fc_conf *fc_conf);
 106 static int  eth_igb_flow_ctrl_set(struct rte_eth_dev *dev,
 107                                 struct rte_eth_fc_conf *fc_conf);
 108 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev);
 109 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev);
 110 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev);
 111 static int eth_igb_interrupt_action(struct rte_eth_dev *dev);
 112 static void eth_igb_interrupt_handler(struct rte_intr_handle *handle,
 113                                                         void *param);
 114 static int  igb_hardware_init(struct e1000_hw *hw);
 115 static void igb_hw_control_acquire(struct e1000_hw *hw);
 116 static void igb_hw_control_release(struct e1000_hw *hw);
 117 static void igb_init_manageability(struct e1000_hw *hw);
 118 static void igb_release_manageability(struct e1000_hw *hw);
 119
 120 static int  eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 121
 122 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev,
 123                 uint16_t vlan_id, int on);
 124 static void eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 125 static void eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 126
 127 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 128 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 129 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev);
 130 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev);
 131 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 132 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 133
 134 static int eth_igb_led_on(struct rte_eth_dev *dev);
 135 static int eth_igb_led_off(struct rte_eth_dev *dev);
 136
 137 static void igb_intr_disable(struct e1000_hw *hw);
 138 static int  igb_get_rx_buffer_size(struct e1000_hw *hw);
 139 static void eth_igb_rar_set(struct rte_eth_dev *dev,
 140                 struct ether_addr *mac_addr,
 141                 uint32_t index, uint32_t pool);
 142 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 143 static void eth_igb_default_mac_addr_set(struct rte_eth_dev *dev,
 144                 struct ether_addr *addr);
 145
 146 static void igbvf_intr_disable(struct e1000_hw *hw);
 147 static int igbvf_dev_configure(struct rte_eth_dev *dev);
 148 static int igbvf_dev_start(struct rte_eth_dev *dev);
 149 static void igbvf_dev_stop(struct rte_eth_dev *dev);
 150 static void igbvf_dev_close(struct rte_eth_dev *dev);
 151 static int eth_igbvf_link_update(struct e1000_hw *hw);
 152 static void eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats);
 153 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev);
 154 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev,
 155                 uint16_t vlan_id, int on);
 156 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on);
 157 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 158 static void igbvf_default_mac_addr_set(struct rte_eth_dev *dev,
 159                 struct ether_addr *addr);
 160 static int igbvf_get_reg_length(struct rte_eth_dev *dev);
 161 static int igbvf_get_regs(struct rte_eth_dev *dev,
 162                 struct rte_dev_reg_info *regs);
 163
 164 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev,
 165                                    struct rte_eth_rss_reta_entry64 *reta_conf,
 166                                    uint16_t reta_size);
 167 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev,
 168                                   struct rte_eth_rss_reta_entry64 *reta_conf,
 169                                   uint16_t reta_size);
 170
 171 static int eth_igb_syn_filter_set(struct rte_eth_dev *dev,
 172                         struct rte_eth_syn_filter *filter,
 173                         bool add);
 174 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev,
 175                         struct rte_eth_syn_filter *filter);
 176 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
 177                         enum rte_filter_op filter_op,
 178                         void *arg);
 179 static int igb_add_2tuple_filter(struct rte_eth_dev *dev,
 180                         struct rte_eth_ntuple_filter *ntuple_filter);
 181 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev,
 182                         struct rte_eth_ntuple_filter *ntuple_filter);
 183 static int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
 184                         struct rte_eth_flex_filter *filter,
 185                         bool add);
 186 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev,
 187                         struct rte_eth_flex_filter *filter);
 188 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
 189                         enum rte_filter_op filter_op,
 190                         void *arg);
 191 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
 192                         struct rte_eth_ntuple_filter *ntuple_filter);
 193 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
 194                         struct rte_eth_ntuple_filter *ntuple_filter);
 195 static int igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
 196                         struct rte_eth_ntuple_filter *filter,
 197                         bool add);
 198 static int igb_get_ntuple_filter(struct rte_eth_dev *dev,
 199                         struct rte_eth_ntuple_filter *filter);
 200 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev,
 201                                 enum rte_filter_op filter_op,
 202                                 void *arg);
 203 static int igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
 204                         struct rte_eth_ethertype_filter *filter,
 205                         bool add);
 206 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev,
 207                                 enum rte_filter_op filter_op,
 208                                 void *arg);
 209 static int igb_get_ethertype_filter(struct rte_eth_dev *dev,
 210                         struct rte_eth_ethertype_filter *filter);
 211 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev,
 212                      enum rte_filter_type filter_type,
 213                      enum rte_filter_op filter_op,
 214                      void *arg);
 215 static int eth_igb_get_reg_length(struct rte_eth_dev *dev);
 216 static int eth_igb_get_regs(struct rte_eth_dev *dev,
 217                 struct rte_dev_reg_info *regs);
 218 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev);
 219 static int eth_igb_get_eeprom(struct rte_eth_dev *dev,
 220                 struct rte_dev_eeprom_info *eeprom);
 221 static int eth_igb_set_eeprom(struct rte_eth_dev *dev,
 222                 struct rte_dev_eeprom_info *eeprom);
 223 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
 224                                     struct ether_addr *mc_addr_set,
 225                                     uint32_t nb_mc_addr);
 226 static int igb_timesync_enable(struct rte_eth_dev *dev);
 227 static int igb_timesync_disable(struct rte_eth_dev *dev);
 228 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 229                                           struct timespec *timestamp,
 230                                           uint32_t flags);
 231 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 232                                           struct timespec *timestamp);
 233 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev,
 234                                         uint16_t queue_id);
 235 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev,
 236                                          uint16_t queue_id);
 237 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction,
 238                                        uint8_t queue, uint8_t msix_vector);
 239 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector,
 240                                uint8_t index, uint8_t offset);
 241 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev);
 242
 243 /*
 244  * Define VF Stats MACRO for Non "cleared on read" register
 245  */
 246 #define UPDATE_VF_STAT(reg, last, cur)            \
 247 {                                                 \
 248         u32 latest = E1000_READ_REG(hw, reg);     \
 249         cur += (latest - last) & UINT_MAX;        \
 250         last = latest;                            \
 251 }
 252
 253 #define IGB_FC_PAUSE_TIME 0x0680
 254 #define IGB_LINK_UPDATE_CHECK_TIMEOUT  90  /* 9s */
 255 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */
 256
 257 #define IGBVF_PMD_NAME "rte_igbvf_pmd"     /* PMD name */
 258
 259 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full;
 260
 261 /*
 262  * The set of PCI devices this driver supports
 263  */
 264 static const struct rte_pci_id pci_id_igb_map[] = {
 265
 266 #define RTE_PCI_DEV_ID_DECL_IGB(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 267 #include "rte_pci_dev_ids.h"
 268
 269 {0},
 270 };
 271
 272 /*
 273  * The set of PCI devices this driver supports (for 82576&I350 VF)
 274  */
 275 static const struct rte_pci_id pci_id_igbvf_map[] = {
 276
 277 #define RTE_PCI_DEV_ID_DECL_IGBVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 278 #include "rte_pci_dev_ids.h"
 279
 280 {0},
 281 };
 282
 283 static const struct eth_dev_ops eth_igb_ops = {
 284         .dev_configure        = eth_igb_configure,
 285         .dev_start            = eth_igb_start,
 286         .dev_stop             = eth_igb_stop,
 287         .dev_close            = eth_igb_close,
 288         .promiscuous_enable   = eth_igb_promiscuous_enable,
 289         .promiscuous_disable  = eth_igb_promiscuous_disable,
 290         .allmulticast_enable  = eth_igb_allmulticast_enable,
 291         .allmulticast_disable = eth_igb_allmulticast_disable,
 292         .link_update          = eth_igb_link_update,
 293         .stats_get            = eth_igb_stats_get,
 294         .stats_reset          = eth_igb_stats_reset,
 295         .dev_infos_get        = eth_igb_infos_get,
 296         .mtu_set              = eth_igb_mtu_set,
 297         .vlan_filter_set      = eth_igb_vlan_filter_set,
 298         .vlan_tpid_set        = eth_igb_vlan_tpid_set,
 299         .vlan_offload_set     = eth_igb_vlan_offload_set,
 300         .rx_queue_setup       = eth_igb_rx_queue_setup,
 301         .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable,
 302         .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable,
 303         .rx_queue_release     = eth_igb_rx_queue_release,
 304         .rx_queue_count       = eth_igb_rx_queue_count,
 305         .rx_descriptor_done   = eth_igb_rx_descriptor_done,
 306         .tx_queue_setup       = eth_igb_tx_queue_setup,
 307         .tx_queue_release     = eth_igb_tx_queue_release,
 308         .dev_led_on           = eth_igb_led_on,
 309         .dev_led_off          = eth_igb_led_off,
 310         .flow_ctrl_get        = eth_igb_flow_ctrl_get,
 311         .flow_ctrl_set        = eth_igb_flow_ctrl_set,
 312         .mac_addr_add         = eth_igb_rar_set,
 313         .mac_addr_remove      = eth_igb_rar_clear,
 314         .mac_addr_set         = eth_igb_default_mac_addr_set,
 315         .reta_update          = eth_igb_rss_reta_update,
 316         .reta_query           = eth_igb_rss_reta_query,
 317         .rss_hash_update      = eth_igb_rss_hash_update,
 318         .rss_hash_conf_get    = eth_igb_rss_hash_conf_get,
 319         .filter_ctrl          = eth_igb_filter_ctrl,
 320         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 321         .timesync_enable      = igb_timesync_enable,
 322         .timesync_disable     = igb_timesync_disable,
 323         .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp,
 324         .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp,
 325         .get_reg_length       = eth_igb_get_reg_length,
 326         .get_reg              = eth_igb_get_regs,
 327         .get_eeprom_length    = eth_igb_get_eeprom_length,
 328         .get_eeprom           = eth_igb_get_eeprom,
 329         .set_eeprom           = eth_igb_set_eeprom,
 330 };
 331
 332 /*
 333  * dev_ops for virtual function, bare necessities for basic vf
 334  * operation have been implemented
 335  */
 336 static const struct eth_dev_ops igbvf_eth_dev_ops = {
 337         .dev_configure        = igbvf_dev_configure,
 338         .dev_start            = igbvf_dev_start,
 339         .dev_stop             = igbvf_dev_stop,
 340         .dev_close            = igbvf_dev_close,
 341         .link_update          = eth_igb_link_update,
 342         .stats_get            = eth_igbvf_stats_get,
 343         .stats_reset          = eth_igbvf_stats_reset,
 344         .vlan_filter_set      = igbvf_vlan_filter_set,
 345         .dev_infos_get        = eth_igbvf_infos_get,
 346         .rx_queue_setup       = eth_igb_rx_queue_setup,
 347         .rx_queue_release     = eth_igb_rx_queue_release,
 348         .tx_queue_setup       = eth_igb_tx_queue_setup,
 349         .tx_queue_release     = eth_igb_tx_queue_release,
 350         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 351         .mac_addr_set         = igbvf_default_mac_addr_set,
 352         .get_reg_length       = igbvf_get_reg_length,
 353         .get_reg              = igbvf_get_regs,
 354 };
 355
 356 /**
 357  * Atomically reads the link status information from global
 358  * structure rte_eth_dev.
 359  *
 360  * @param dev
 361  *   - Pointer to the structure rte_eth_dev to read from.
 362  *   - Pointer to the buffer to be saved with the link status.
 363  *
 364  * @return
 365  *   - On success, zero.
 366  *   - On failure, negative value.
 367  */
 368 static inline int
 369 rte_igb_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 370                                 struct rte_eth_link *link)
 371 {
 372         struct rte_eth_link *dst = link;
 373         struct rte_eth_link *src = &(dev->data->dev_link);
 374
 375         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 376                                         *(uint64_t *)src) == 0)
 377                 return -1;
 378
 379         return 0;
 380 }
 381
 382 /**
 383  * Atomically writes the link status information into global
 384  * structure rte_eth_dev.
 385  *
 386  * @param dev
 387  *   - Pointer to the structure rte_eth_dev to read from.
 388  *   - Pointer to the buffer to be saved with the link status.
 389  *
 390  * @return
 391  *   - On success, zero.
 392  *   - On failure, negative value.
 393  */
 394 static inline int
 395 rte_igb_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 396                                 struct rte_eth_link *link)
 397 {
 398         struct rte_eth_link *dst = &(dev->data->dev_link);
 399         struct rte_eth_link *src = link;
 400
 401         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 402                                         *(uint64_t *)src) == 0)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static inline void
 409 igb_intr_enable(struct rte_eth_dev *dev)
 410 {
 411         struct e1000_interrupt *intr =
 412                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 413         struct e1000_hw *hw =
 414                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 415
 416         E1000_WRITE_REG(hw, E1000_IMS, intr->mask);
 417         E1000_WRITE_FLUSH(hw);
 418 }
 419
 420 static void
 421 igb_intr_disable(struct e1000_hw *hw)
 422 {
 423         E1000_WRITE_REG(hw, E1000_IMC, ~0);
 424         E1000_WRITE_FLUSH(hw);
 425 }
 426
 427 static inline int32_t
 428 igb_pf_reset_hw(struct e1000_hw *hw)
 429 {
 430         uint32_t ctrl_ext;
 431         int32_t status;
 432
 433         status = e1000_reset_hw(hw);
 434
 435         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 436         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 437         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 438         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 439         E1000_WRITE_FLUSH(hw);
 440
 441         return status;
 442 }
 443
 444 static void
 445 igb_identify_hardware(struct rte_eth_dev *dev)
 446 {
 447         struct e1000_hw *hw =
 448                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 449
 450         hw->vendor_id = dev->pci_dev->id.vendor_id;
 451         hw->device_id = dev->pci_dev->id.device_id;
 452         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
 453         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
 454
 455         e1000_set_mac_type(hw);
 456
 457         /* need to check if it is a vf device below */
 458 }
 459
 460 static int
 461 igb_reset_swfw_lock(struct e1000_hw *hw)
 462 {
 463         int ret_val;
 464
 465         /*
 466          * Do mac ops initialization manually here, since we will need
 467          * some function pointers set by this call.
 468          */
 469         ret_val = e1000_init_mac_params(hw);
 470         if (ret_val)
 471                 return ret_val;
 472
 473         /*
 474          * SMBI lock should not fail in this early stage. If this is the case,
 475          * it is due to an improper exit of the application.
 476          * So force the release of the faulty lock.
 477          */
 478         if (e1000_get_hw_semaphore_generic(hw) < 0) {
 479                 PMD_DRV_LOG(DEBUG, "SMBI lock released");
 480         }
 481         e1000_put_hw_semaphore_generic(hw);
 482
 483         if (hw->mac.ops.acquire_swfw_sync != NULL) {
 484                 uint16_t mask;
 485
 486                 /*
 487                  * Phy lock should not fail in this early stage. If this is the case,
 488                  * it is due to an improper exit of the application.
 489                  * So force the release of the faulty lock.
 490                  */
 491                 mask = E1000_SWFW_PHY0_SM << hw->bus.func;
 492                 if (hw->bus.func > E1000_FUNC_1)
 493                         mask <<= 2;
 494                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 495                         PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released",
 496                                     hw->bus.func);
 497                 }
 498                 hw->mac.ops.release_swfw_sync(hw, mask);
 499
 500                 /*
 501                  * This one is more tricky since it is common to all ports; but
 502                  * swfw_sync retries last long enough (1s) to be almost sure that if
 503                  * lock can not be taken it is due to an improper lock of the
 504                  * semaphore.
 505                  */
 506                 mask = E1000_SWFW_EEP_SM;
 507                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 508                         PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 509                 }
 510                 hw->mac.ops.release_swfw_sync(hw, mask);
 511         }
 512
 513         return E1000_SUCCESS;
 514 }
 515
 516 static int
 517 eth_igb_dev_init(struct rte_eth_dev *eth_dev)
 518 {
 519         int error = 0;
 520         struct rte_pci_device *pci_dev;
 521         struct e1000_hw *hw =
 522                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 523         struct e1000_vfta * shadow_vfta =
 524                 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 525         struct e1000_filter_info *filter_info =
 526                 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
 527         struct e1000_adapter *adapter =
 528                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 529
 530         uint32_t ctrl_ext;
 531
 532         pci_dev = eth_dev->pci_dev;
 533         eth_dev->dev_ops = &eth_igb_ops;
 534         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 535         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 536
 537         /* for secondary processes, we don't initialise any further as primary
 538          * has already done this work. Only check we don't need a different
 539          * RX function */
 540         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 541                 if (eth_dev->data->scattered_rx)
 542                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 543                 return 0;
 544         }
 545
 546         hw->hw_addr= (void *)pci_dev->mem_resource[0].addr;
 547
 548         igb_identify_hardware(eth_dev);
 549         if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) {
 550                 error = -EIO;
 551                 goto err_late;
 552         }
 553
 554         e1000_get_bus_info(hw);
 555
 556         /* Reset any pending lock */
 557         if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) {
 558                 error = -EIO;
 559                 goto err_late;
 560         }
 561
 562         /* Finish initialization */
 563         if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) {
 564                 error = -EIO;
 565                 goto err_late;
 566         }
 567
 568         hw->mac.autoneg = 1;
 569         hw->phy.autoneg_wait_to_complete = 0;
 570         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 571
 572         /* Copper options */
 573         if (hw->phy.media_type == e1000_media_type_copper) {
 574                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
 575                 hw->phy.disable_polarity_correction = 0;
 576                 hw->phy.ms_type = e1000_ms_hw_default;
 577         }
 578
 579         /*
 580          * Start from a known state, this is important in reading the nvm
 581          * and mac from that.
 582          */
 583         igb_pf_reset_hw(hw);
 584
 585         /* Make sure we have a good EEPROM before we read from it */
 586         if (e1000_validate_nvm_checksum(hw) < 0) {
 587                 /*
 588                  * Some PCI-E parts fail the first check due to
 589                  * the link being in sleep state, call it again,
 590                  * if it fails a second time its a real issue.
 591                  */
 592                 if (e1000_validate_nvm_checksum(hw) < 0) {
 593                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
 594                         error = -EIO;
 595                         goto err_late;
 596                 }
 597         }
 598
 599         /* Read the permanent MAC address out of the EEPROM */
 600         if (e1000_read_mac_addr(hw) != 0) {
 601                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
 602                 error = -EIO;
 603                 goto err_late;
 604         }
 605
 606         /* Allocate memory for storing MAC addresses */
 607         eth_dev->data->mac_addrs = rte_zmalloc("e1000",
 608                 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
 609         if (eth_dev->data->mac_addrs == NULL) {
 610                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to "
 611                                                 "store MAC addresses",
 612                                 ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 613                 error = -ENOMEM;
 614                 goto err_late;
 615         }
 616
 617         /* Copy the permanent MAC address */
 618         ether_addr_copy((struct ether_addr *)hw->mac.addr, &eth_dev->data->mac_addrs[0]);
 619
 620         /* initialize the vfta */
 621         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 622
 623         /* Now initialize the hardware */
 624         if (igb_hardware_init(hw) != 0) {
 625                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
 626                 rte_free(eth_dev->data->mac_addrs);
 627                 eth_dev->data->mac_addrs = NULL;
 628                 error = -ENODEV;
 629                 goto err_late;
 630         }
 631         hw->mac.get_link_status = 1;
 632         adapter->stopped = 0;
 633
 634         /* Indicate SOL/IDER usage */
 635         if (e1000_check_reset_block(hw) < 0) {
 636                 PMD_INIT_LOG(ERR, "PHY reset is blocked due to"
 637                                         "SOL/IDER session");
 638         }
 639
 640         /* initialize PF if max_vfs not zero */
 641         igb_pf_host_init(eth_dev);
 642
 643         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 644         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 645         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 646         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 647         E1000_WRITE_FLUSH(hw);
 648
 649         PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x",
 650                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 651                      pci_dev->id.device_id);
 652
 653         /* enable support intr */
 654         igb_intr_enable(eth_dev);
 655
 656         TAILQ_INIT(&filter_info->flex_list);
 657         filter_info->flex_mask = 0;
 658         TAILQ_INIT(&filter_info->twotuple_list);
 659         filter_info->twotuple_mask = 0;
 660         TAILQ_INIT(&filter_info->fivetuple_list);
 661         filter_info->fivetuple_mask = 0;
 662
 663         return 0;
 664
 665 err_late:
 666         igb_hw_control_release(hw);
 667
 668         return (error);
 669 }
 670
 671 static int
 672 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev)
 673 {
 674         struct rte_pci_device *pci_dev;
 675         struct e1000_hw *hw;
 676         struct e1000_adapter *adapter =
 677                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 678
 679         PMD_INIT_FUNC_TRACE();
 680
 681         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 682                 return -EPERM;
 683
 684         hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 685         pci_dev = eth_dev->pci_dev;
 686
 687         if (adapter->stopped == 0)
 688                 eth_igb_close(eth_dev);
 689
 690         eth_dev->dev_ops = NULL;
 691         eth_dev->rx_pkt_burst = NULL;
 692         eth_dev->tx_pkt_burst = NULL;
 693
 694         /* Reset any pending lock */
 695         igb_reset_swfw_lock(hw);
 696
 697         rte_free(eth_dev->data->mac_addrs);
 698         eth_dev->data->mac_addrs = NULL;
 699
 700         /* uninitialize PF if max_vfs not zero */
 701         igb_pf_host_uninit(eth_dev);
 702
 703         /* disable uio intr before callback unregister */
 704         rte_intr_disable(&(pci_dev->intr_handle));
 705         rte_intr_callback_unregister(&(pci_dev->intr_handle),
 706                 eth_igb_interrupt_handler, (void *)eth_dev);
 707
 708         return 0;
 709 }
 710
 711 /*
 712  * Virtual Function device init
 713  */
 714 static int
 715 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev)
 716 {
 717         struct rte_pci_device *pci_dev;
 718         struct e1000_adapter *adapter =
 719                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 720         struct e1000_hw *hw =
 721                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 722         int diag;
 723
 724         PMD_INIT_FUNC_TRACE();
 725
 726         eth_dev->dev_ops = &igbvf_eth_dev_ops;
 727         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 728         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 729
 730         /* for secondary processes, we don't initialise any further as primary
 731          * has already done this work. Only check we don't need a different
 732          * RX function */
 733         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 734                 if (eth_dev->data->scattered_rx)
 735                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 736                 return 0;
 737         }
 738
 739         pci_dev = eth_dev->pci_dev;
 740
 741         hw->device_id = pci_dev->id.device_id;
 742         hw->vendor_id = pci_dev->id.vendor_id;
 743         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 744         adapter->stopped = 0;
 745
 746         /* Initialize the shared code (base driver) */
 747         diag = e1000_setup_init_funcs(hw, TRUE);
 748         if (diag != 0) {
 749                 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d",
 750                         diag);
 751                 return -EIO;
 752         }
 753
 754         /* init_mailbox_params */
 755         hw->mbx.ops.init_params(hw);
 756
 757         /* Disable the interrupts for VF */
 758         igbvf_intr_disable(hw);
 759
 760         diag = hw->mac.ops.reset_hw(hw);
 761
 762         /* Allocate memory for storing MAC addresses */
 763         eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN *
 764                 hw->mac.rar_entry_count, 0);
 765         if (eth_dev->data->mac_addrs == NULL) {
 766                 PMD_INIT_LOG(ERR,
 767                         "Failed to allocate %d bytes needed to store MAC "
 768                         "addresses",
 769                         ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 770                 return -ENOMEM;
 771         }
 772
 773         /* Copy the permanent MAC address */
 774         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 775                         &eth_dev->data->mac_addrs[0]);
 776
 777         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x "
 778                      "mac.type=%s",
 779                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 780                      pci_dev->id.device_id, "igb_mac_82576_vf");
 781
 782         return 0;
 783 }
 784
 785 static int
 786 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev)
 787 {
 788         struct e1000_adapter *adapter =
 789                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 790
 791         PMD_INIT_FUNC_TRACE();
 792
 793         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 794                 return -EPERM;
 795
 796         if (adapter->stopped == 0)
 797                 igbvf_dev_close(eth_dev);
 798
 799         eth_dev->dev_ops = NULL;
 800         eth_dev->rx_pkt_burst = NULL;
 801         eth_dev->tx_pkt_burst = NULL;
 802
 803         rte_free(eth_dev->data->mac_addrs);
 804         eth_dev->data->mac_addrs = NULL;
 805
 806         return 0;
 807 }
 808
 809 static struct eth_driver rte_igb_pmd = {
 810         .pci_drv = {
 811                 .name = "rte_igb_pmd",
 812                 .id_table = pci_id_igb_map,
 813                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
 814                         RTE_PCI_DRV_DETACHABLE,
 815         },
 816         .eth_dev_init = eth_igb_dev_init,
 817         .eth_dev_uninit = eth_igb_dev_uninit,
 818         .dev_private_size = sizeof(struct e1000_adapter),
 819 };
 820
 821 /*
 822  * virtual function driver struct
 823  */
 824 static struct eth_driver rte_igbvf_pmd = {
 825         .pci_drv = {
 826                 .name = "rte_igbvf_pmd",
 827                 .id_table = pci_id_igbvf_map,
 828                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
 829         },
 830         .eth_dev_init = eth_igbvf_dev_init,
 831         .eth_dev_uninit = eth_igbvf_dev_uninit,
 832         .dev_private_size = sizeof(struct e1000_adapter),
 833 };
 834
 835 static int
 836 rte_igb_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 837 {
 838         rte_eth_driver_register(&rte_igb_pmd);
 839         return 0;
 840 }
 841
 842 static void
 843 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 844 {
 845         struct e1000_hw *hw =
 846                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 847         /* RCTL: enable VLAN filter since VMDq always use VLAN filter */
 848         uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL);
 849         rctl |= E1000_RCTL_VFE;
 850         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
 851 }
 852
 853 /*
 854  * VF Driver initialization routine.
 855  * Invoked one at EAL init time.
 856  * Register itself as the [Virtual Poll Mode] Driver of PCI IGB devices.
 857  */
 858 static int
 859 rte_igbvf_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 860 {
 861         PMD_INIT_FUNC_TRACE();
 862
 863         rte_eth_driver_register(&rte_igbvf_pmd);
 864         return (0);
 865 }
 866
 867 static int
 868 eth_igb_configure(struct rte_eth_dev *dev)
 869 {
 870         struct e1000_interrupt *intr =
 871                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 872
 873         PMD_INIT_FUNC_TRACE();
 874         intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
 875         PMD_INIT_FUNC_TRACE();
 876
 877         return (0);
 878 }
 879
 880 static int
 881 eth_igb_start(struct rte_eth_dev *dev)
 882 {
 883         struct e1000_hw *hw =
 884                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 885         struct e1000_adapter *adapter =
 886                 E1000_DEV_PRIVATE(dev->data->dev_private);
 887         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 888         int ret, mask;
 889         uint32_t intr_vector = 0;
 890         uint32_t ctrl_ext;
 891
 892         PMD_INIT_FUNC_TRACE();
 893
 894         /* Power up the phy. Needed to make the link go Up */
 895         e1000_power_up_phy(hw);
 896
 897         /*
 898          * Packet Buffer Allocation (PBA)
 899          * Writing PBA sets the receive portion of the buffer
 900          * the remainder is used for the transmit buffer.
 901          */
 902         if (hw->mac.type == e1000_82575) {
 903                 uint32_t pba;
 904
 905                 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
 906                 E1000_WRITE_REG(hw, E1000_PBA, pba);
 907         }
 908
 909         /* Put the address into the Receive Address Array */
 910         e1000_rar_set(hw, hw->mac.addr, 0);
 911
 912         /* Initialize the hardware */
 913         if (igb_hardware_init(hw)) {
 914                 PMD_INIT_LOG(ERR, "Unable to initialize the hardware");
 915                 return (-EIO);
 916         }
 917         adapter->stopped = 0;
 918
 919         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
 920
 921         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 922         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 923         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 924         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 925         E1000_WRITE_FLUSH(hw);
 926
 927         /* configure PF module if SRIOV enabled */
 928         igb_pf_host_configure(dev);
 929
 930         /* check and configure queue intr-vector mapping */
 931         if (dev->data->dev_conf.intr_conf.rxq != 0)
 932                 intr_vector = dev->data->nb_rx_queues;
 933
 934         if (rte_intr_efd_enable(intr_handle, intr_vector))
 935                 return -1;
 936
 937         if (rte_intr_dp_is_en(intr_handle)) {
 938                 intr_handle->intr_vec =
 939                         rte_zmalloc("intr_vec",
 940                                     dev->data->nb_rx_queues * sizeof(int), 0);
 941                 if (intr_handle->intr_vec == NULL) {
 942                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
 943                                      " intr_vec\n", dev->data->nb_rx_queues);
 944                         return -ENOMEM;
 945                 }
 946         }
 947
 948         /* confiugre msix for rx interrupt */
 949         eth_igb_configure_msix_intr(dev);
 950
 951         /* Configure for OS presence */
 952         igb_init_manageability(hw);
 953
 954         eth_igb_tx_init(dev);
 955
 956         /* This can fail when allocating mbufs for descriptor rings */
 957         ret = eth_igb_rx_init(dev);
 958         if (ret) {
 959                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
 960                 igb_dev_clear_queues(dev);
 961                 return ret;
 962         }
 963
 964         e1000_clear_hw_cntrs_base_generic(hw);
 965
 966         /*
 967          * VLAN Offload Settings
 968          */
 969         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
 970                         ETH_VLAN_EXTEND_MASK;
 971         eth_igb_vlan_offload_set(dev, mask);
 972
 973         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
 974                 /* Enable VLAN filter since VMDq always use VLAN filter */
 975                 igb_vmdq_vlan_hw_filter_enable(dev);
 976         }
 977
 978         if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) ||
 979                 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) ||
 980                 (hw->mac.type == e1000_i211)) {
 981                 /* Configure EITR with the maximum possible value (0xFFFF) */
 982                 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF);
 983         }
 984
 985         /* Setup link speed and duplex */
 986         switch (dev->data->dev_conf.link_speed) {
 987         case ETH_LINK_SPEED_AUTONEG:
 988                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
 989                         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 990                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
 991                         hw->phy.autoneg_advertised = E1000_ALL_HALF_DUPLEX;
 992                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
 993                         hw->phy.autoneg_advertised = E1000_ALL_FULL_DUPLEX;
 994                 else
 995                         goto error_invalid_config;
 996                 break;
 997         case ETH_LINK_SPEED_10:
 998                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
 999                         hw->phy.autoneg_advertised = E1000_ALL_10_SPEED;
1000                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
1001                         hw->phy.autoneg_advertised = ADVERTISE_10_HALF;
1002                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
1003                         hw->phy.autoneg_advertised = ADVERTISE_10_FULL;
1004                 else
1005                         goto error_invalid_config;
1006                 break;
1007         case ETH_LINK_SPEED_100:
1008                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
1009                         hw->phy.autoneg_advertised = E1000_ALL_100_SPEED;
1010                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
1011                         hw->phy.autoneg_advertised = ADVERTISE_100_HALF;
1012                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
1013                         hw->phy.autoneg_advertised = ADVERTISE_100_FULL;
1014                 else
1015                         goto error_invalid_config;
1016                 break;
1017         case ETH_LINK_SPEED_1000:
1018                 if ((dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX) ||
1019                                 (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX))
1020                         hw->phy.autoneg_advertised = ADVERTISE_1000_FULL;
1021                 else
1022                         goto error_invalid_config;
1023                 break;
1024         case ETH_LINK_SPEED_10000:
1025         default:
1026                 goto error_invalid_config;
1027         }
1028         e1000_setup_link(hw);
1029
1030         /* check if lsc interrupt feature is enabled */
1031         if (dev->data->dev_conf.intr_conf.lsc != 0) {
1032                 if (rte_intr_allow_others(intr_handle)) {
1033                         rte_intr_callback_register(intr_handle,
1034                                                    eth_igb_interrupt_handler,
1035                                                    (void *)dev);
1036                         eth_igb_lsc_interrupt_setup(dev);
1037                 } else
1038                         PMD_INIT_LOG(INFO, "lsc won't enable because of"
1039                                      " no intr multiplex\n");
1040         }
1041
1042         /* check if rxq interrupt is enabled */
1043         if (dev->data->dev_conf.intr_conf.rxq != 0)
1044                 eth_igb_rxq_interrupt_setup(dev);
1045
1046         /* enable uio/vfio intr/eventfd mapping */
1047         rte_intr_enable(intr_handle);
1048
1049         /* resume enabled intr since hw reset */
1050         igb_intr_enable(dev);
1051
1052         PMD_INIT_LOG(DEBUG, "<<");
1053
1054         return (0);
1055
1056 error_invalid_config:
1057         PMD_INIT_LOG(ERR, "Invalid link_speed/link_duplex (%u/%u) for port %u",
1058                      dev->data->dev_conf.link_speed,
1059                      dev->data->dev_conf.link_duplex, dev->data->port_id);
1060         igb_dev_clear_queues(dev);
1061         return (-EINVAL);
1062 }
1063
1064 /*********************************************************************
1065  *
1066  *  This routine disables all traffic on the adapter by issuing a
1067  *  global reset on the MAC.
1068  *
1069  **********************************************************************/
1070 static void
1071 eth_igb_stop(struct rte_eth_dev *dev)
1072 {
1073         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1074         struct e1000_filter_info *filter_info =
1075                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
1076         struct rte_eth_link link;
1077         struct e1000_flex_filter *p_flex;
1078         struct e1000_5tuple_filter *p_5tuple, *p_5tuple_next;
1079         struct e1000_2tuple_filter *p_2tuple, *p_2tuple_next;
1080         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1081
1082         igb_intr_disable(hw);
1083
1084         /* disable intr eventfd mapping */
1085         rte_intr_disable(intr_handle);
1086
1087         igb_pf_reset_hw(hw);
1088         E1000_WRITE_REG(hw, E1000_WUC, 0);
1089
1090         /* Set bit for Go Link disconnect */
1091         if (hw->mac.type >= e1000_82580) {
1092                 uint32_t phpm_reg;
1093
1094                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1095                 phpm_reg |= E1000_82580_PM_GO_LINKD;
1096                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1097         }
1098
1099         /* Power down the phy. Needed to make the link go Down */
1100         if (hw->phy.media_type == e1000_media_type_copper)
1101                 e1000_power_down_phy(hw);
1102         else
1103                 e1000_shutdown_fiber_serdes_link(hw);
1104
1105         igb_dev_clear_queues(dev);
1106
1107         /* clear the recorded link status */
1108         memset(&link, 0, sizeof(link));
1109         rte_igb_dev_atomic_write_link_status(dev, &link);
1110
1111         /* Remove all flex filters of the device */
1112         while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) {
1113                 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries);
1114                 rte_free(p_flex);
1115         }
1116         filter_info->flex_mask = 0;
1117
1118         /* Remove all ntuple filters of the device */
1119         for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list);
1120              p_5tuple != NULL; p_5tuple = p_5tuple_next) {
1121                 p_5tuple_next = TAILQ_NEXT(p_5tuple, entries);
1122                 TAILQ_REMOVE(&filter_info->fivetuple_list,
1123                              p_5tuple, entries);
1124                 rte_free(p_5tuple);
1125         }
1126         filter_info->fivetuple_mask = 0;
1127         for (p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list);
1128              p_2tuple != NULL; p_2tuple = p_2tuple_next) {
1129                 p_2tuple_next = TAILQ_NEXT(p_2tuple, entries);
1130                 TAILQ_REMOVE(&filter_info->twotuple_list,
1131                              p_2tuple, entries);
1132                 rte_free(p_2tuple);
1133         }
1134         filter_info->twotuple_mask = 0;
1135
1136         /* Clean datapath event and queue/vec mapping */
1137         rte_intr_efd_disable(intr_handle);
1138         if (intr_handle->intr_vec != NULL) {
1139                 rte_free(intr_handle->intr_vec);
1140                 intr_handle->intr_vec = NULL;
1141         }
1142 }
1143
1144 static void
1145 eth_igb_close(struct rte_eth_dev *dev)
1146 {
1147         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1148         struct e1000_adapter *adapter =
1149                 E1000_DEV_PRIVATE(dev->data->dev_private);
1150         struct rte_eth_link link;
1151         struct rte_pci_device *pci_dev;
1152
1153         eth_igb_stop(dev);
1154         adapter->stopped = 1;
1155
1156         e1000_phy_hw_reset(hw);
1157         igb_release_manageability(hw);
1158         igb_hw_control_release(hw);
1159
1160         /* Clear bit for Go Link disconnect */
1161         if (hw->mac.type >= e1000_82580) {
1162                 uint32_t phpm_reg;
1163
1164                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1165                 phpm_reg &= ~E1000_82580_PM_GO_LINKD;
1166                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1167         }
1168
1169         igb_dev_free_queues(dev);
1170
1171         pci_dev = dev->pci_dev;
1172         if (pci_dev->intr_handle.intr_vec) {
1173                 rte_free(pci_dev->intr_handle.intr_vec);
1174                 pci_dev->intr_handle.intr_vec = NULL;
1175         }
1176
1177         memset(&link, 0, sizeof(link));
1178         rte_igb_dev_atomic_write_link_status(dev, &link);
1179 }
1180
1181 static int
1182 igb_get_rx_buffer_size(struct e1000_hw *hw)
1183 {
1184         uint32_t rx_buf_size;
1185         if (hw->mac.type == e1000_82576) {
1186                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10;
1187         } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) {
1188                 /* PBS needs to be translated according to a lookup table */
1189                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf);
1190                 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size);
1191                 rx_buf_size = (rx_buf_size << 10);
1192         } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) {
1193                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10;
1194         } else {
1195                 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10;
1196         }
1197
1198         return rx_buf_size;
1199 }
1200
1201 /*********************************************************************
1202  *
1203  *  Initialize the hardware
1204  *
1205  **********************************************************************/
1206 static int
1207 igb_hardware_init(struct e1000_hw *hw)
1208 {
1209         uint32_t rx_buf_size;
1210         int diag;
1211
1212         /* Let the firmware know the OS is in control */
1213         igb_hw_control_acquire(hw);
1214
1215         /*
1216          * These parameters control the automatic generation (Tx) and
1217          * response (Rx) to Ethernet PAUSE frames.
1218          * - High water mark should allow for at least two standard size (1518)
1219          *   frames to be received after sending an XOFF.
1220          * - Low water mark works best when it is very near the high water mark.
1221          *   This allows the receiver to restart by sending XON when it has
1222          *   drained a bit. Here we use an arbitrary value of 1500 which will
1223          *   restart after one full frame is pulled from the buffer. There
1224          *   could be several smaller frames in the buffer and if so they will
1225          *   not trigger the XON until their total number reduces the buffer
1226          *   by 1500.
1227          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
1228          */
1229         rx_buf_size = igb_get_rx_buffer_size(hw);
1230
1231         hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2);
1232         hw->fc.low_water = hw->fc.high_water - 1500;
1233         hw->fc.pause_time = IGB_FC_PAUSE_TIME;
1234         hw->fc.send_xon = 1;
1235
1236         /* Set Flow control, use the tunable location if sane */
1237         if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4))
1238                 hw->fc.requested_mode = igb_fc_setting;
1239         else
1240                 hw->fc.requested_mode = e1000_fc_none;
1241
1242         /* Issue a global reset */
1243         igb_pf_reset_hw(hw);
1244         E1000_WRITE_REG(hw, E1000_WUC, 0);
1245
1246         diag = e1000_init_hw(hw);
1247         if (diag < 0)
1248                 return (diag);
1249
1250         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1251         e1000_get_phy_info(hw);
1252         e1000_check_for_link(hw);
1253
1254         return (0);
1255 }
1256
1257 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */
1258 static void
1259 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1260 {
1261         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1262         struct e1000_hw_stats *stats =
1263                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1264         int pause_frames;
1265
1266         if(hw->phy.media_type == e1000_media_type_copper ||
1267             (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
1268                 stats->symerrs +=
1269                     E1000_READ_REG(hw,E1000_SYMERRS);
1270                 stats->sec += E1000_READ_REG(hw, E1000_SEC);
1271         }
1272
1273         stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
1274         stats->mpc += E1000_READ_REG(hw, E1000_MPC);
1275         stats->scc += E1000_READ_REG(hw, E1000_SCC);
1276         stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
1277
1278         stats->mcc += E1000_READ_REG(hw, E1000_MCC);
1279         stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
1280         stats->colc += E1000_READ_REG(hw, E1000_COLC);
1281         stats->dc += E1000_READ_REG(hw, E1000_DC);
1282         stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
1283         stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
1284         stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
1285         /*
1286         ** For watchdog management we need to know if we have been
1287         ** paused during the last interval, so capture that here.
1288         */
1289         pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC);
1290         stats->xoffrxc += pause_frames;
1291         stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
1292         stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
1293         stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
1294         stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
1295         stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
1296         stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
1297         stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
1298         stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
1299         stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
1300         stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
1301         stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
1302         stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
1303
1304         /* For the 64-bit byte counters the low dword must be read first. */
1305         /* Both registers clear on the read of the high dword */
1306
1307         stats->gorc += E1000_READ_REG(hw, E1000_GORCL);
1308         stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32);
1309         stats->gotc += E1000_READ_REG(hw, E1000_GOTCL);
1310         stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32);
1311
1312         stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
1313         stats->ruc += E1000_READ_REG(hw, E1000_RUC);
1314         stats->rfc += E1000_READ_REG(hw, E1000_RFC);
1315         stats->roc += E1000_READ_REG(hw, E1000_ROC);
1316         stats->rjc += E1000_READ_REG(hw, E1000_RJC);
1317
1318         stats->tor += E1000_READ_REG(hw, E1000_TORL);
1319         stats->tor += ((uint64_t)E1000_READ_REG(hw, E1000_TORH) << 32);
1320         stats->tot += E1000_READ_REG(hw, E1000_TOTL);
1321         stats->tot += ((uint64_t)E1000_READ_REG(hw, E1000_TOTH) << 32);
1322
1323         stats->tpr += E1000_READ_REG(hw, E1000_TPR);
1324         stats->tpt += E1000_READ_REG(hw, E1000_TPT);
1325         stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
1326         stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
1327         stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
1328         stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
1329         stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
1330         stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
1331         stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
1332         stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
1333
1334         /* Interrupt Counts */
1335
1336         stats->iac += E1000_READ_REG(hw, E1000_IAC);
1337         stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
1338         stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
1339         stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
1340         stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
1341         stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
1342         stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
1343         stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
1344         stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
1345
1346         /* Host to Card Statistics */
1347
1348         stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
1349         stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
1350         stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
1351         stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
1352         stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
1353         stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
1354         stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
1355         stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL);
1356         stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32);
1357         stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL);
1358         stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32);
1359         stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
1360         stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
1361         stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
1362
1363         stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
1364         stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
1365         stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
1366         stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
1367         stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
1368         stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
1369
1370         if (rte_stats == NULL)
1371                 return;
1372
1373         /* Rx Errors */
1374         rte_stats->ibadcrc = stats->crcerrs;
1375         rte_stats->ibadlen = stats->rlec + stats->ruc + stats->roc;
1376         rte_stats->imissed = stats->mpc;
1377         rte_stats->ierrors = rte_stats->ibadcrc +
1378                              rte_stats->ibadlen +
1379                              rte_stats->imissed +
1380                              stats->rxerrc + stats->algnerrc + stats->cexterr;
1381
1382         /* Tx Errors */
1383         rte_stats->oerrors = stats->ecol + stats->latecol;
1384
1385         /* XON/XOFF pause frames */
1386         rte_stats->tx_pause_xon  = stats->xontxc;
1387         rte_stats->rx_pause_xon  = stats->xonrxc;
1388         rte_stats->tx_pause_xoff = stats->xofftxc;
1389         rte_stats->rx_pause_xoff = stats->xoffrxc;
1390
1391         rte_stats->ipackets = stats->gprc;
1392         rte_stats->opackets = stats->gptc;
1393         rte_stats->ibytes   = stats->gorc;
1394         rte_stats->obytes   = stats->gotc;
1395 }
1396
1397 static void
1398 eth_igb_stats_reset(struct rte_eth_dev *dev)
1399 {
1400         struct e1000_hw_stats *hw_stats =
1401                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1402
1403         /* HW registers are cleared on read */
1404         eth_igb_stats_get(dev, NULL);
1405
1406         /* Reset software totals */
1407         memset(hw_stats, 0, sizeof(*hw_stats));
1408 }
1409
1410 static void
1411 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1412 {
1413         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1414         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1415                           E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1416
1417         /* Good Rx packets, include VF loopback */
1418         UPDATE_VF_STAT(E1000_VFGPRC,
1419             hw_stats->last_gprc, hw_stats->gprc);
1420
1421         /* Good Rx octets, include VF loopback */
1422         UPDATE_VF_STAT(E1000_VFGORC,
1423             hw_stats->last_gorc, hw_stats->gorc);
1424
1425         /* Good Tx packets, include VF loopback */
1426         UPDATE_VF_STAT(E1000_VFGPTC,
1427             hw_stats->last_gptc, hw_stats->gptc);
1428
1429         /* Good Tx octets, include VF loopback */
1430         UPDATE_VF_STAT(E1000_VFGOTC,
1431             hw_stats->last_gotc, hw_stats->gotc);
1432
1433         /* Rx Multicst packets */
1434         UPDATE_VF_STAT(E1000_VFMPRC,
1435             hw_stats->last_mprc, hw_stats->mprc);
1436
1437         /* Good Rx loopback packets */
1438         UPDATE_VF_STAT(E1000_VFGPRLBC,
1439             hw_stats->last_gprlbc, hw_stats->gprlbc);
1440
1441         /* Good Rx loopback octets */
1442         UPDATE_VF_STAT(E1000_VFGORLBC,
1443             hw_stats->last_gorlbc, hw_stats->gorlbc);
1444
1445         /* Good Tx loopback packets */
1446         UPDATE_VF_STAT(E1000_VFGPTLBC,
1447             hw_stats->last_gptlbc, hw_stats->gptlbc);
1448
1449         /* Good Tx loopback octets */
1450         UPDATE_VF_STAT(E1000_VFGOTLBC,
1451             hw_stats->last_gotlbc, hw_stats->gotlbc);
1452
1453         if (rte_stats == NULL)
1454                 return;
1455
1456         rte_stats->ipackets = hw_stats->gprc;
1457         rte_stats->ibytes = hw_stats->gorc;
1458         rte_stats->opackets = hw_stats->gptc;
1459         rte_stats->obytes = hw_stats->gotc;
1460         rte_stats->imcasts = hw_stats->mprc;
1461         rte_stats->ilbpackets = hw_stats->gprlbc;
1462         rte_stats->ilbbytes = hw_stats->gorlbc;
1463         rte_stats->olbpackets = hw_stats->gptlbc;
1464         rte_stats->olbbytes = hw_stats->gotlbc;
1465
1466 }
1467
1468 static void
1469 eth_igbvf_stats_reset(struct rte_eth_dev *dev)
1470 {
1471         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1472                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1473
1474         /* Sync HW register to the last stats */
1475         eth_igbvf_stats_get(dev, NULL);
1476
1477         /* reset HW current stats*/
1478         memset(&hw_stats->gprc, 0, sizeof(*hw_stats) -
1479                offsetof(struct e1000_vf_stats, gprc));
1480
1481 }
1482
1483 static void
1484 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1485 {
1486         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1487
1488         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1489         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1490         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1491         dev_info->rx_offload_capa =
1492                 DEV_RX_OFFLOAD_VLAN_STRIP |
1493                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1494                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1495                 DEV_RX_OFFLOAD_TCP_CKSUM;
1496         dev_info->tx_offload_capa =
1497                 DEV_TX_OFFLOAD_VLAN_INSERT |
1498                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1499                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1500                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1501                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
1502                 DEV_TX_OFFLOAD_TCP_TSO;
1503
1504         switch (hw->mac.type) {
1505         case e1000_82575:
1506                 dev_info->max_rx_queues = 4;
1507                 dev_info->max_tx_queues = 4;
1508                 dev_info->max_vmdq_pools = 0;
1509                 break;
1510
1511         case e1000_82576:
1512                 dev_info->max_rx_queues = 16;
1513                 dev_info->max_tx_queues = 16;
1514                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1515                 dev_info->vmdq_queue_num = 16;
1516                 break;
1517
1518         case e1000_82580:
1519                 dev_info->max_rx_queues = 8;
1520                 dev_info->max_tx_queues = 8;
1521                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1522                 dev_info->vmdq_queue_num = 8;
1523                 break;
1524
1525         case e1000_i350:
1526                 dev_info->max_rx_queues = 8;
1527                 dev_info->max_tx_queues = 8;
1528                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1529                 dev_info->vmdq_queue_num = 8;
1530                 break;
1531
1532         case e1000_i354:
1533                 dev_info->max_rx_queues = 8;
1534                 dev_info->max_tx_queues = 8;
1535                 break;
1536
1537         case e1000_i210:
1538                 dev_info->max_rx_queues = 4;
1539                 dev_info->max_tx_queues = 4;
1540                 dev_info->max_vmdq_pools = 0;
1541                 break;
1542
1543         case e1000_i211:
1544                 dev_info->max_rx_queues = 2;
1545                 dev_info->max_tx_queues = 2;
1546                 dev_info->max_vmdq_pools = 0;
1547                 break;
1548
1549         default:
1550                 /* Should not happen */
1551                 break;
1552         }
1553         dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t);
1554         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
1555         dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL;
1556
1557         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1558                 .rx_thresh = {
1559                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1560                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1561                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
1562                 },
1563                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
1564                 .rx_drop_en = 0,
1565         };
1566
1567         dev_info->default_txconf = (struct rte_eth_txconf) {
1568                 .tx_thresh = {
1569                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
1570                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
1571                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
1572                 },
1573                 .txq_flags = 0,
1574         };
1575 }
1576
1577 static void
1578 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1579 {
1580         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1581
1582         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1583         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1584         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1585         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
1586                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1587                                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1588                                 DEV_RX_OFFLOAD_TCP_CKSUM;
1589         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
1590                                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1591                                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1592                                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1593                                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
1594                                 DEV_TX_OFFLOAD_TCP_TSO;
1595         switch (hw->mac.type) {
1596         case e1000_vfadapt:
1597                 dev_info->max_rx_queues = 2;
1598                 dev_info->max_tx_queues = 2;
1599                 break;
1600         case e1000_vfadapt_i350:
1601                 dev_info->max_rx_queues = 1;
1602                 dev_info->max_tx_queues = 1;
1603                 break;
1604         default:
1605                 /* Should not happen */
1606                 break;
1607         }
1608
1609         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1610                 .rx_thresh = {
1611                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1612                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1613                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
1614                 },
1615                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
1616                 .rx_drop_en = 0,
1617         };
1618
1619         dev_info->default_txconf = (struct rte_eth_txconf) {
1620                 .tx_thresh = {
1621                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
1622                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
1623                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
1624                 },
1625                 .txq_flags = 0,
1626         };
1627 }
1628
1629 /* return 0 means link status changed, -1 means not changed */
1630 static int
1631 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1632 {
1633         struct e1000_hw *hw =
1634                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1635         struct rte_eth_link link, old;
1636         int link_check, count;
1637
1638         link_check = 0;
1639         hw->mac.get_link_status = 1;
1640
1641         /* possible wait-to-complete in up to 9 seconds */
1642         for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) {
1643                 /* Read the real link status */
1644                 switch (hw->phy.media_type) {
1645                 case e1000_media_type_copper:
1646                         /* Do the work to read phy */
1647                         e1000_check_for_link(hw);
1648                         link_check = !hw->mac.get_link_status;
1649                         break;
1650
1651                 case e1000_media_type_fiber:
1652                         e1000_check_for_link(hw);
1653                         link_check = (E1000_READ_REG(hw, E1000_STATUS) &
1654                                       E1000_STATUS_LU);
1655                         break;
1656
1657                 case e1000_media_type_internal_serdes:
1658                         e1000_check_for_link(hw);
1659                         link_check = hw->mac.serdes_has_link;
1660                         break;
1661
1662                 /* VF device is type_unknown */
1663                 case e1000_media_type_unknown:
1664                         eth_igbvf_link_update(hw);
1665                         link_check = !hw->mac.get_link_status;
1666                         break;
1667
1668                 default:
1669                         break;
1670                 }
1671                 if (link_check || wait_to_complete == 0)
1672                         break;
1673                 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL);
1674         }
1675         memset(&link, 0, sizeof(link));
1676         rte_igb_dev_atomic_read_link_status(dev, &link);
1677         old = link;
1678
1679         /* Now we check if a transition has happened */
1680         if (link_check) {
1681                 hw->mac.ops.get_link_up_info(hw, &link.link_speed,
1682                                           &link.link_duplex);
1683                 link.link_status = 1;
1684         } else if (!link_check) {
1685                 link.link_speed = 0;
1686                 link.link_duplex = 0;
1687                 link.link_status = 0;
1688         }
1689         rte_igb_dev_atomic_write_link_status(dev, &link);
1690
1691         /* not changed */
1692         if (old.link_status == link.link_status)
1693                 return -1;
1694
1695         /* changed */
1696         return 0;
1697 }
1698
1699 /*
1700  * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
1701  * For ASF and Pass Through versions of f/w this means
1702  * that the driver is loaded.
1703  */
1704 static void
1705 igb_hw_control_acquire(struct e1000_hw *hw)
1706 {
1707         uint32_t ctrl_ext;
1708
1709         /* Let firmware know the driver has taken over */
1710         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1711         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1712 }
1713
1714 /*
1715  * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
1716  * For ASF and Pass Through versions of f/w this means that the
1717  * driver is no longer loaded.
1718  */
1719 static void
1720 igb_hw_control_release(struct e1000_hw *hw)
1721 {
1722         uint32_t ctrl_ext;
1723
1724         /* Let firmware taken over control of h/w */
1725         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1726         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1727                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1728 }
1729
1730 /*
1731  * Bit of a misnomer, what this really means is
1732  * to enable OS management of the system... aka
1733  * to disable special hardware management features.
1734  */
1735 static void
1736 igb_init_manageability(struct e1000_hw *hw)
1737 {
1738         if (e1000_enable_mng_pass_thru(hw)) {
1739                 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H);
1740                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
1741
1742                 /* disable hardware interception of ARP */
1743                 manc &= ~(E1000_MANC_ARP_EN);
1744
1745                 /* enable receiving management packets to the host */
1746                 manc |= E1000_MANC_EN_MNG2HOST;
1747                 manc2h |= 1 << 5;  /* Mng Port 623 */
1748                 manc2h |= 1 << 6;  /* Mng Port 664 */
1749                 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h);
1750                 E1000_WRITE_REG(hw, E1000_MANC, manc);
1751         }
1752 }
1753
1754 static void
1755 igb_release_manageability(struct e1000_hw *hw)
1756 {
1757         if (e1000_enable_mng_pass_thru(hw)) {
1758                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
1759
1760                 manc |= E1000_MANC_ARP_EN;
1761                 manc &= ~E1000_MANC_EN_MNG2HOST;
1762
1763                 E1000_WRITE_REG(hw, E1000_MANC, manc);
1764         }
1765 }
1766
1767 static void
1768 eth_igb_promiscuous_enable(struct rte_eth_dev *dev)
1769 {
1770         struct e1000_hw *hw =
1771                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1772         uint32_t rctl;
1773
1774         rctl = E1000_READ_REG(hw, E1000_RCTL);
1775         rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1776         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1777 }
1778
1779 static void
1780 eth_igb_promiscuous_disable(struct rte_eth_dev *dev)
1781 {
1782         struct e1000_hw *hw =
1783                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1784         uint32_t rctl;
1785
1786         rctl = E1000_READ_REG(hw, E1000_RCTL);
1787         rctl &= (~E1000_RCTL_UPE);
1788         if (dev->data->all_multicast == 1)
1789                 rctl |= E1000_RCTL_MPE;
1790         else
1791                 rctl &= (~E1000_RCTL_MPE);
1792         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1793 }
1794
1795 static void
1796 eth_igb_allmulticast_enable(struct rte_eth_dev *dev)
1797 {
1798         struct e1000_hw *hw =
1799                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1800         uint32_t rctl;
1801
1802         rctl = E1000_READ_REG(hw, E1000_RCTL);
1803         rctl |= E1000_RCTL_MPE;
1804         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1805 }
1806
1807 static void
1808 eth_igb_allmulticast_disable(struct rte_eth_dev *dev)
1809 {
1810         struct e1000_hw *hw =
1811                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1812         uint32_t rctl;
1813
1814         if (dev->data->promiscuous == 1)
1815                 return; /* must remain in all_multicast mode */
1816         rctl = E1000_READ_REG(hw, E1000_RCTL);
1817         rctl &= (~E1000_RCTL_MPE);
1818         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1819 }
1820
1821 static int
1822 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1823 {
1824         struct e1000_hw *hw =
1825                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1826         struct e1000_vfta * shadow_vfta =
1827                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1828         uint32_t vfta;
1829         uint32_t vid_idx;
1830         uint32_t vid_bit;
1831
1832         vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) &
1833                               E1000_VFTA_ENTRY_MASK);
1834         vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK));
1835         vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx);
1836         if (on)
1837                 vfta |= vid_bit;
1838         else
1839                 vfta &= ~vid_bit;
1840         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta);
1841
1842         /* update local VFTA copy */
1843         shadow_vfta->vfta[vid_idx] = vfta;
1844
1845         return 0;
1846 }
1847
1848 static void
1849 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1850 {
1851         struct e1000_hw *hw =
1852                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1853         uint32_t reg = ETHER_TYPE_VLAN ;
1854
1855         reg |= (tpid << 16);
1856         E1000_WRITE_REG(hw, E1000_VET, reg);
1857 }
1858
1859 static void
1860 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1861 {
1862         struct e1000_hw *hw =
1863                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1864         uint32_t reg;
1865
1866         /* Filter Table Disable */
1867         reg = E1000_READ_REG(hw, E1000_RCTL);
1868         reg &= ~E1000_RCTL_CFIEN;
1869         reg &= ~E1000_RCTL_VFE;
1870         E1000_WRITE_REG(hw, E1000_RCTL, reg);
1871 }
1872
1873 static void
1874 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1875 {
1876         struct e1000_hw *hw =
1877                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1878         struct e1000_vfta * shadow_vfta =
1879                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1880         uint32_t reg;
1881         int i;
1882
1883         /* Filter Table Enable, CFI not used for packet acceptance */
1884         reg = E1000_READ_REG(hw, E1000_RCTL);
1885         reg &= ~E1000_RCTL_CFIEN;
1886         reg |= E1000_RCTL_VFE;
1887         E1000_WRITE_REG(hw, E1000_RCTL, reg);
1888
1889         /* restore VFTA table */
1890         for (i = 0; i < IGB_VFTA_SIZE; i++)
1891                 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]);
1892 }
1893
1894 static void
1895 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev)
1896 {
1897         struct e1000_hw *hw =
1898                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1899         uint32_t reg;
1900
1901         /* VLAN Mode Disable */
1902         reg = E1000_READ_REG(hw, E1000_CTRL);
1903         reg &= ~E1000_CTRL_VME;
1904         E1000_WRITE_REG(hw, E1000_CTRL, reg);
1905 }
1906
1907 static void
1908 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev)
1909 {
1910         struct e1000_hw *hw =
1911                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1912         uint32_t reg;
1913
1914         /* VLAN Mode Enable */
1915         reg = E1000_READ_REG(hw, E1000_CTRL);
1916         reg |= E1000_CTRL_VME;
1917         E1000_WRITE_REG(hw, E1000_CTRL, reg);
1918 }
1919
1920 static void
1921 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1922 {
1923         struct e1000_hw *hw =
1924                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1925         uint32_t reg;
1926
1927         /* CTRL_EXT: Extended VLAN */
1928         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
1929         reg &= ~E1000_CTRL_EXT_EXTEND_VLAN;
1930         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
1931
1932         /* Update maximum packet length */
1933         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
1934                 E1000_WRITE_REG(hw, E1000_RLPML,
1935                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
1936                                                 VLAN_TAG_SIZE);
1937 }
1938
1939 static void
1940 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1941 {
1942         struct e1000_hw *hw =
1943                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1944         uint32_t reg;
1945
1946         /* CTRL_EXT: Extended VLAN */
1947         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
1948         reg |= E1000_CTRL_EXT_EXTEND_VLAN;
1949         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
1950
1951         /* Update maximum packet length */
1952         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
1953                 E1000_WRITE_REG(hw, E1000_RLPML,
1954                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
1955                                                 2 * VLAN_TAG_SIZE);
1956 }
1957
1958 static void
1959 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1960 {
1961         if(mask & ETH_VLAN_STRIP_MASK){
1962                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1963                         igb_vlan_hw_strip_enable(dev);
1964                 else
1965                         igb_vlan_hw_strip_disable(dev);
1966         }
1967
1968         if(mask & ETH_VLAN_FILTER_MASK){
1969                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1970                         igb_vlan_hw_filter_enable(dev);
1971                 else
1972                         igb_vlan_hw_filter_disable(dev);
1973         }
1974
1975         if(mask & ETH_VLAN_EXTEND_MASK){
1976                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1977                         igb_vlan_hw_extend_enable(dev);
1978                 else
1979                         igb_vlan_hw_extend_disable(dev);
1980         }
1981 }
1982
1983
1984 /**
1985  * It enables the interrupt mask and then enable the interrupt.
1986  *
1987  * @param dev
1988  *  Pointer to struct rte_eth_dev.
1989  *
1990  * @return
1991  *  - On success, zero.
1992  *  - On failure, a negative value.
1993  */
1994 static int
1995 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev)
1996 {
1997         struct e1000_interrupt *intr =
1998                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1999
2000         intr->mask |= E1000_ICR_LSC;
2001
2002         return 0;
2003 }
2004
2005 /* It clears the interrupt causes and enables the interrupt.
2006  * It will be called once only during nic initialized.
2007  *
2008  * @param dev
2009  *  Pointer to struct rte_eth_dev.
2010  *
2011  * @return
2012  *  - On success, zero.
2013  *  - On failure, a negative value.
2014  */
2015 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev)
2016 {
2017         uint32_t mask, regval;
2018         struct e1000_hw *hw =
2019                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2020         struct rte_eth_dev_info dev_info;
2021
2022         memset(&dev_info, 0, sizeof(dev_info));
2023         eth_igb_infos_get(dev, &dev_info);
2024
2025         mask = 0xFFFFFFFF >> (32 - dev_info.max_rx_queues);
2026         regval = E1000_READ_REG(hw, E1000_EIMS);
2027         E1000_WRITE_REG(hw, E1000_EIMS, regval | mask);
2028
2029         return 0;
2030 }
2031
2032 /*
2033  * It reads ICR and gets interrupt causes, check it and set a bit flag
2034  * to update link status.
2035  *
2036  * @param dev
2037  *  Pointer to struct rte_eth_dev.
2038  *
2039  * @return
2040  *  - On success, zero.
2041  *  - On failure, a negative value.
2042  */
2043 static int
2044 eth_igb_interrupt_get_status(struct rte_eth_dev *dev)
2045 {
2046         uint32_t icr;
2047         struct e1000_hw *hw =
2048                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2049         struct e1000_interrupt *intr =
2050                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2051
2052         igb_intr_disable(hw);
2053
2054         /* read-on-clear nic registers here */
2055         icr = E1000_READ_REG(hw, E1000_ICR);
2056
2057         intr->flags = 0;
2058         if (icr & E1000_ICR_LSC) {
2059                 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
2060         }
2061
2062         if (icr & E1000_ICR_VMMB)
2063                 intr->flags |= E1000_FLAG_MAILBOX;
2064
2065         return 0;
2066 }
2067
2068 /*
2069  * It executes link_update after knowing an interrupt is prsent.
2070  *
2071  * @param dev
2072  *  Pointer to struct rte_eth_dev.
2073  *
2074  * @return
2075  *  - On success, zero.
2076  *  - On failure, a negative value.
2077  */
2078 static int
2079 eth_igb_interrupt_action(struct rte_eth_dev *dev)
2080 {
2081         struct e1000_hw *hw =
2082                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2083         struct e1000_interrupt *intr =
2084                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2085         uint32_t tctl, rctl;
2086         struct rte_eth_link link;
2087         int ret;
2088
2089         if (intr->flags & E1000_FLAG_MAILBOX) {
2090                 igb_pf_mbx_process(dev);
2091                 intr->flags &= ~E1000_FLAG_MAILBOX;
2092         }
2093
2094         igb_intr_enable(dev);
2095         rte_intr_enable(&(dev->pci_dev->intr_handle));
2096
2097         if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) {
2098                 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE;
2099
2100                 /* set get_link_status to check register later */
2101                 hw->mac.get_link_status = 1;
2102                 ret = eth_igb_link_update(dev, 0);
2103
2104                 /* check if link has changed */
2105                 if (ret < 0)
2106                         return 0;
2107
2108                 memset(&link, 0, sizeof(link));
2109                 rte_igb_dev_atomic_read_link_status(dev, &link);
2110                 if (link.link_status) {
2111                         PMD_INIT_LOG(INFO,
2112                                      " Port %d: Link Up - speed %u Mbps - %s",
2113                                      dev->data->port_id,
2114                                      (unsigned)link.link_speed,
2115                                      link.link_duplex == ETH_LINK_FULL_DUPLEX ?
2116                                      "full-duplex" : "half-duplex");
2117                 } else {
2118                         PMD_INIT_LOG(INFO, " Port %d: Link Down",
2119                                      dev->data->port_id);
2120                 }
2121
2122                 PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d",
2123                              dev->pci_dev->addr.domain,
2124                              dev->pci_dev->addr.bus,
2125                              dev->pci_dev->addr.devid,
2126                              dev->pci_dev->addr.function);
2127                 tctl = E1000_READ_REG(hw, E1000_TCTL);
2128                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2129                 if (link.link_status) {
2130                         /* enable Tx/Rx */
2131                         tctl |= E1000_TCTL_EN;
2132                         rctl |= E1000_RCTL_EN;
2133                 } else {
2134                         /* disable Tx/Rx */
2135                         tctl &= ~E1000_TCTL_EN;
2136                         rctl &= ~E1000_RCTL_EN;
2137                 }
2138                 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
2139                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2140                 E1000_WRITE_FLUSH(hw);
2141                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
2142         }
2143
2144         return 0;
2145 }
2146
2147 /**
2148  * Interrupt handler which shall be registered at first.
2149  *
2150  * @param handle
2151  *  Pointer to interrupt handle.
2152  * @param param
2153  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2154  *
2155  * @return
2156  *  void
2157  */
2158 static void
2159 eth_igb_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
2160                                                         void *param)
2161 {
2162         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2163
2164         eth_igb_interrupt_get_status(dev);
2165         eth_igb_interrupt_action(dev);
2166 }
2167
2168 static int
2169 eth_igb_led_on(struct rte_eth_dev *dev)
2170 {
2171         struct e1000_hw *hw;
2172
2173         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2174         return (e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
2175 }
2176
2177 static int
2178 eth_igb_led_off(struct rte_eth_dev *dev)
2179 {
2180         struct e1000_hw *hw;
2181
2182         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2183         return (e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
2184 }
2185
2186 static int
2187 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2188 {
2189         struct e1000_hw *hw;
2190         uint32_t ctrl;
2191         int tx_pause;
2192         int rx_pause;
2193
2194         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2195         fc_conf->pause_time = hw->fc.pause_time;
2196         fc_conf->high_water = hw->fc.high_water;
2197         fc_conf->low_water = hw->fc.low_water;
2198         fc_conf->send_xon = hw->fc.send_xon;
2199         fc_conf->autoneg = hw->mac.autoneg;
2200
2201         /*
2202          * Return rx_pause and tx_pause status according to actual setting of
2203          * the TFCE and RFCE bits in the CTRL register.
2204          */
2205         ctrl = E1000_READ_REG(hw, E1000_CTRL);
2206         if (ctrl & E1000_CTRL_TFCE)
2207                 tx_pause = 1;
2208         else
2209                 tx_pause = 0;
2210
2211         if (ctrl & E1000_CTRL_RFCE)
2212                 rx_pause = 1;
2213         else
2214                 rx_pause = 0;
2215
2216         if (rx_pause && tx_pause)
2217                 fc_conf->mode = RTE_FC_FULL;
2218         else if (rx_pause)
2219                 fc_conf->mode = RTE_FC_RX_PAUSE;
2220         else if (tx_pause)
2221                 fc_conf->mode = RTE_FC_TX_PAUSE;
2222         else
2223                 fc_conf->mode = RTE_FC_NONE;
2224
2225         return 0;
2226 }
2227
2228 static int
2229 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2230 {
2231         struct e1000_hw *hw;
2232         int err;
2233         enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = {
2234                 e1000_fc_none,
2235                 e1000_fc_rx_pause,
2236                 e1000_fc_tx_pause,
2237                 e1000_fc_full
2238         };
2239         uint32_t rx_buf_size;
2240         uint32_t max_high_water;
2241         uint32_t rctl;
2242
2243         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2244         if (fc_conf->autoneg != hw->mac.autoneg)
2245                 return -ENOTSUP;
2246         rx_buf_size = igb_get_rx_buffer_size(hw);
2247         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2248
2249         /* At least reserve one Ethernet frame for watermark */
2250         max_high_water = rx_buf_size - ETHER_MAX_LEN;
2251         if ((fc_conf->high_water > max_high_water) ||
2252             (fc_conf->high_water < fc_conf->low_water)) {
2253                 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value");
2254                 PMD_INIT_LOG(ERR, "high water must <=  0x%x", max_high_water);
2255                 return (-EINVAL);
2256         }
2257
2258         hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode];
2259         hw->fc.pause_time     = fc_conf->pause_time;
2260         hw->fc.high_water     = fc_conf->high_water;
2261         hw->fc.low_water      = fc_conf->low_water;
2262         hw->fc.send_xon       = fc_conf->send_xon;
2263
2264         err = e1000_setup_link_generic(hw);
2265         if (err == E1000_SUCCESS) {
2266
2267                 /* check if we want to forward MAC frames - driver doesn't have native
2268                  * capability to do that, so we'll write the registers ourselves */
2269
2270                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2271
2272                 /* set or clear MFLCN.PMCF bit depending on configuration */
2273                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2274                         rctl |= E1000_RCTL_PMCF;
2275                 else
2276                         rctl &= ~E1000_RCTL_PMCF;
2277
2278                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2279                 E1000_WRITE_FLUSH(hw);
2280
2281                 return 0;
2282         }
2283
2284         PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err);
2285         return (-EIO);
2286 }
2287
2288 #define E1000_RAH_POOLSEL_SHIFT      (18)
2289 static void
2290 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2291                 uint32_t index, __rte_unused uint32_t pool)
2292 {
2293         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2294         uint32_t rah;
2295
2296         e1000_rar_set(hw, mac_addr->addr_bytes, index);
2297         rah = E1000_READ_REG(hw, E1000_RAH(index));
2298         rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool));
2299         E1000_WRITE_REG(hw, E1000_RAH(index), rah);
2300 }
2301
2302 static void
2303 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index)
2304 {
2305         uint8_t addr[ETHER_ADDR_LEN];
2306         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2307
2308         memset(addr, 0, sizeof(addr));
2309
2310         e1000_rar_set(hw, addr, index);
2311 }
2312
2313 static void
2314 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev,
2315                                 struct ether_addr *addr)
2316 {
2317         eth_igb_rar_clear(dev, 0);
2318
2319         eth_igb_rar_set(dev, (void *)addr, 0, 0);
2320 }
2321 /*
2322  * Virtual Function operations
2323  */
2324 static void
2325 igbvf_intr_disable(struct e1000_hw *hw)
2326 {
2327         PMD_INIT_FUNC_TRACE();
2328
2329         /* Clear interrupt mask to stop from interrupts being generated */
2330         E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF);
2331
2332         E1000_WRITE_FLUSH(hw);
2333 }
2334
2335 static void
2336 igbvf_stop_adapter(struct rte_eth_dev *dev)
2337 {
2338         u32 reg_val;
2339         u16 i;
2340         struct rte_eth_dev_info dev_info;
2341         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2342
2343         memset(&dev_info, 0, sizeof(dev_info));
2344         eth_igbvf_infos_get(dev, &dev_info);
2345
2346         /* Clear interrupt mask to stop from interrupts being generated */
2347         igbvf_intr_disable(hw);
2348
2349         /* Clear any pending interrupts, flush previous writes */
2350         E1000_READ_REG(hw, E1000_EICR);
2351
2352         /* Disable the transmit unit.  Each queue must be disabled. */
2353         for (i = 0; i < dev_info.max_tx_queues; i++)
2354                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH);
2355
2356         /* Disable the receive unit by stopping each queue */
2357         for (i = 0; i < dev_info.max_rx_queues; i++) {
2358                 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i));
2359                 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE;
2360                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val);
2361                 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE)
2362                         ;
2363         }
2364
2365         /* flush all queues disables */
2366         E1000_WRITE_FLUSH(hw);
2367         msec_delay(2);
2368 }
2369
2370 static int eth_igbvf_link_update(struct e1000_hw *hw)
2371 {
2372         struct e1000_mbx_info *mbx = &hw->mbx;
2373         struct e1000_mac_info *mac = &hw->mac;
2374         int ret_val = E1000_SUCCESS;
2375
2376         PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf");
2377
2378         /*
2379          * We only want to run this if there has been a rst asserted.
2380          * in this case that could mean a link change, device reset,
2381          * or a virtual function reset
2382          */
2383
2384         /* If we were hit with a reset or timeout drop the link */
2385         if (!e1000_check_for_rst(hw, 0) || !mbx->timeout)
2386                 mac->get_link_status = TRUE;
2387
2388         if (!mac->get_link_status)
2389                 goto out;
2390
2391         /* if link status is down no point in checking to see if pf is up */
2392         if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU))
2393                 goto out;
2394
2395         /* if we passed all the tests above then the link is up and we no
2396          * longer need to check for link */
2397         mac->get_link_status = FALSE;
2398
2399 out:
2400         return ret_val;
2401 }
2402
2403
2404 static int
2405 igbvf_dev_configure(struct rte_eth_dev *dev)
2406 {
2407         struct rte_eth_conf* conf = &dev->data->dev_conf;
2408
2409         PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
2410                      dev->data->port_id);
2411
2412         /*
2413          * VF has no ability to enable/disable HW CRC
2414          * Keep the persistent behavior the same as Host PF
2415          */
2416 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC
2417         if (!conf->rxmode.hw_strip_crc) {
2418                 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip");
2419                 conf->rxmode.hw_strip_crc = 1;
2420         }
2421 #else
2422         if (conf->rxmode.hw_strip_crc) {
2423                 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip");
2424                 conf->rxmode.hw_strip_crc = 0;
2425         }
2426 #endif
2427
2428         return 0;
2429 }
2430
2431 static int
2432 igbvf_dev_start(struct rte_eth_dev *dev)
2433 {
2434         struct e1000_hw *hw =
2435                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2436         struct e1000_adapter *adapter =
2437                 E1000_DEV_PRIVATE(dev->data->dev_private);
2438         int ret;
2439
2440         PMD_INIT_FUNC_TRACE();
2441
2442         hw->mac.ops.reset_hw(hw);
2443         adapter->stopped = 0;
2444
2445         /* Set all vfta */
2446         igbvf_set_vfta_all(dev,1);
2447
2448         eth_igbvf_tx_init(dev);
2449
2450         /* This can fail when allocating mbufs for descriptor rings */
2451         ret = eth_igbvf_rx_init(dev);
2452         if (ret) {
2453                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
2454                 igb_dev_clear_queues(dev);
2455                 return ret;
2456         }
2457
2458         return 0;
2459 }
2460
2461 static void
2462 igbvf_dev_stop(struct rte_eth_dev *dev)
2463 {
2464         PMD_INIT_FUNC_TRACE();
2465
2466         igbvf_stop_adapter(dev);
2467
2468         /*
2469           * Clear what we set, but we still keep shadow_vfta to
2470           * restore after device starts
2471           */
2472         igbvf_set_vfta_all(dev,0);
2473
2474         igb_dev_clear_queues(dev);
2475 }
2476
2477 static void
2478 igbvf_dev_close(struct rte_eth_dev *dev)
2479 {
2480         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2481         struct e1000_adapter *adapter =
2482                 E1000_DEV_PRIVATE(dev->data->dev_private);
2483
2484         PMD_INIT_FUNC_TRACE();
2485
2486         e1000_reset_hw(hw);
2487
2488         igbvf_dev_stop(dev);
2489         adapter->stopped = 1;
2490         igb_dev_free_queues(dev);
2491 }
2492
2493 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on)
2494 {
2495         struct e1000_mbx_info *mbx = &hw->mbx;
2496         uint32_t msgbuf[2];
2497
2498         /* After set vlan, vlan strip will also be enabled in igb driver*/
2499         msgbuf[0] = E1000_VF_SET_VLAN;
2500         msgbuf[1] = vid;
2501         /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */
2502         if (on)
2503                 msgbuf[0] |= E1000_VF_SET_VLAN_ADD;
2504
2505         return (mbx->ops.write_posted(hw, msgbuf, 2, 0));
2506 }
2507
2508 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2509 {
2510         struct e1000_hw *hw =
2511                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2512         struct e1000_vfta * shadow_vfta =
2513                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2514         int i = 0, j = 0, vfta = 0, mask = 1;
2515
2516         for (i = 0; i < IGB_VFTA_SIZE; i++){
2517                 vfta = shadow_vfta->vfta[i];
2518                 if(vfta){
2519                         mask = 1;
2520                         for (j = 0; j < 32; j++){
2521                                 if(vfta & mask)
2522                                         igbvf_set_vfta(hw,
2523                                                 (uint16_t)((i<<5)+j), on);
2524                                 mask<<=1;
2525                         }
2526                 }
2527         }
2528
2529 }
2530
2531 static int
2532 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2533 {
2534         struct e1000_hw *hw =
2535                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2536         struct e1000_vfta * shadow_vfta =
2537                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2538         uint32_t vid_idx = 0;
2539         uint32_t vid_bit = 0;
2540         int ret = 0;
2541
2542         PMD_INIT_FUNC_TRACE();
2543
2544         /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/
2545         ret = igbvf_set_vfta(hw, vlan_id, !!on);
2546         if(ret){
2547                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2548                 return ret;
2549         }
2550         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2551         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2552
2553         /*Save what we set and retore it after device reset*/
2554         if (on)
2555                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2556         else
2557                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2558
2559         return 0;
2560 }
2561
2562 static void
2563 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr)
2564 {
2565         struct e1000_hw *hw =
2566                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2567
2568         /* index is not used by rar_set() */
2569         hw->mac.ops.rar_set(hw, (void *)addr, 0);
2570 }
2571
2572
2573 static int
2574 eth_igb_rss_reta_update(struct rte_eth_dev *dev,
2575                         struct rte_eth_rss_reta_entry64 *reta_conf,
2576                         uint16_t reta_size)
2577 {
2578         uint8_t i, j, mask;
2579         uint32_t reta, r;
2580         uint16_t idx, shift;
2581         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2582
2583         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2584                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2585                         "(%d) doesn't match the number hardware can supported "
2586                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2587                 return -EINVAL;
2588         }
2589
2590         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
2591                 idx = i / RTE_RETA_GROUP_SIZE;
2592                 shift = i % RTE_RETA_GROUP_SIZE;
2593                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2594                                                 IGB_4_BIT_MASK);
2595                 if (!mask)
2596                         continue;
2597                 if (mask == IGB_4_BIT_MASK)
2598                         r = 0;
2599                 else
2600                         r = E1000_READ_REG(hw, E1000_RETA(i >> 2));
2601                 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) {
2602                         if (mask & (0x1 << j))
2603                                 reta |= reta_conf[idx].reta[shift + j] <<
2604                                                         (CHAR_BIT * j);
2605                         else
2606                                 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j));
2607                 }
2608                 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta);
2609         }
2610
2611         return 0;
2612 }
2613
2614 static int
2615 eth_igb_rss_reta_query(struct rte_eth_dev *dev,
2616                        struct rte_eth_rss_reta_entry64 *reta_conf,
2617                        uint16_t reta_size)
2618 {
2619         uint8_t i, j, mask;
2620         uint32_t reta;
2621         uint16_t idx, shift;
2622         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2623
2624         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2625                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2626                         "(%d) doesn't match the number hardware can supported "
2627                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2628                 return -EINVAL;
2629         }
2630
2631         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
2632                 idx = i / RTE_RETA_GROUP_SIZE;
2633                 shift = i % RTE_RETA_GROUP_SIZE;
2634                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2635                                                 IGB_4_BIT_MASK);
2636                 if (!mask)
2637                         continue;
2638                 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2));
2639                 for (j = 0; j < IGB_4_BIT_WIDTH; j++) {
2640                         if (mask & (0x1 << j))
2641                                 reta_conf[idx].reta[shift + j] =
2642                                         ((reta >> (CHAR_BIT * j)) &
2643                                                 IGB_8_BIT_MASK);
2644                 }
2645         }
2646
2647         return 0;
2648 }
2649
2650 #define MAC_TYPE_FILTER_SUP(type)    do {\
2651         if ((type) != e1000_82580 && (type) != e1000_i350 &&\
2652                 (type) != e1000_82576)\
2653                 return -ENOTSUP;\
2654 } while (0)
2655
2656 static int
2657 eth_igb_syn_filter_set(struct rte_eth_dev *dev,
2658                         struct rte_eth_syn_filter *filter,
2659                         bool add)
2660 {
2661         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2662         uint32_t synqf, rfctl;
2663
2664         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
2665                 return -EINVAL;
2666
2667         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
2668
2669         if (add) {
2670                 if (synqf & E1000_SYN_FILTER_ENABLE)
2671                         return -EINVAL;
2672
2673                 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) &
2674                         E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE);
2675
2676                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
2677                 if (filter->hig_pri)
2678                         rfctl |= E1000_RFCTL_SYNQFP;
2679                 else
2680                         rfctl &= ~E1000_RFCTL_SYNQFP;
2681
2682                 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl);
2683         } else {
2684                 if (!(synqf & E1000_SYN_FILTER_ENABLE))
2685                         return -ENOENT;
2686                 synqf = 0;
2687         }
2688
2689         E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf);
2690         E1000_WRITE_FLUSH(hw);
2691         return 0;
2692 }
2693
2694 static int
2695 eth_igb_syn_filter_get(struct rte_eth_dev *dev,
2696                         struct rte_eth_syn_filter *filter)
2697 {
2698         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2699         uint32_t synqf, rfctl;
2700
2701         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
2702         if (synqf & E1000_SYN_FILTER_ENABLE) {
2703                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
2704                 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0;
2705                 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >>
2706                                 E1000_SYN_FILTER_QUEUE_SHIFT);
2707                 return 0;
2708         }
2709
2710         return -ENOENT;
2711 }
2712
2713 static int
2714 eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
2715                         enum rte_filter_op filter_op,
2716                         void *arg)
2717 {
2718         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2719         int ret;
2720
2721         MAC_TYPE_FILTER_SUP(hw->mac.type);
2722
2723         if (filter_op == RTE_ETH_FILTER_NOP)
2724                 return 0;
2725
2726         if (arg == NULL) {
2727                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
2728                             filter_op);
2729                 return -EINVAL;
2730         }
2731
2732         switch (filter_op) {
2733         case RTE_ETH_FILTER_ADD:
2734                 ret = eth_igb_syn_filter_set(dev,
2735                                 (struct rte_eth_syn_filter *)arg,
2736                                 TRUE);
2737                 break;
2738         case RTE_ETH_FILTER_DELETE:
2739                 ret = eth_igb_syn_filter_set(dev,
2740                                 (struct rte_eth_syn_filter *)arg,
2741                                 FALSE);
2742                 break;
2743         case RTE_ETH_FILTER_GET:
2744                 ret = eth_igb_syn_filter_get(dev,
2745                                 (struct rte_eth_syn_filter *)arg);
2746                 break;
2747         default:
2748                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
2749                 ret = -EINVAL;
2750                 break;
2751         }
2752
2753         return ret;
2754 }
2755
2756 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
2757         if ((type) != e1000_82580 && (type) != e1000_i350)\
2758                 return -ENOSYS; \
2759 } while (0)
2760
2761 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/
2762 static inline int
2763 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter,
2764                         struct e1000_2tuple_filter_info *filter_info)
2765 {
2766         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
2767                 return -EINVAL;
2768         if (filter->priority > E1000_2TUPLE_MAX_PRI)
2769                 return -EINVAL;  /* filter index is out of range. */
2770         if (filter->tcp_flags > TCP_FLAG_ALL)
2771                 return -EINVAL;  /* flags is invalid. */
2772
2773         switch (filter->dst_port_mask) {
2774         case UINT16_MAX:
2775                 filter_info->dst_port_mask = 0;
2776                 filter_info->dst_port = filter->dst_port;
2777                 break;
2778         case 0:
2779                 filter_info->dst_port_mask = 1;
2780                 break;
2781         default:
2782                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
2783                 return -EINVAL;
2784         }
2785
2786         switch (filter->proto_mask) {
2787         case UINT8_MAX:
2788                 filter_info->proto_mask = 0;
2789                 filter_info->proto = filter->proto;
2790                 break;
2791         case 0:
2792                 filter_info->proto_mask = 1;
2793                 break;
2794         default:
2795                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
2796                 return -EINVAL;
2797         }
2798
2799         filter_info->priority = (uint8_t)filter->priority;
2800         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
2801                 filter_info->tcp_flags = filter->tcp_flags;
2802         else
2803                 filter_info->tcp_flags = 0;
2804
2805         return 0;
2806 }
2807
2808 static inline struct e1000_2tuple_filter *
2809 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list,
2810                         struct e1000_2tuple_filter_info *key)
2811 {
2812         struct e1000_2tuple_filter *it;
2813
2814         TAILQ_FOREACH(it, filter_list, entries) {
2815                 if (memcmp(key, &it->filter_info,
2816                         sizeof(struct e1000_2tuple_filter_info)) == 0) {
2817                         return it;
2818                 }
2819         }
2820         return NULL;
2821 }
2822
2823 /*
2824  * igb_add_2tuple_filter - add a 2tuple filter
2825  *
2826  * @param
2827  * dev: Pointer to struct rte_eth_dev.
2828  * ntuple_filter: ponter to the filter that will be added.
2829  *
2830  * @return
2831  *    - On success, zero.
2832  *    - On failure, a negative value.
2833  */
2834 static int
2835 igb_add_2tuple_filter(struct rte_eth_dev *dev,
2836                         struct rte_eth_ntuple_filter *ntuple_filter)
2837 {
2838         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2839         struct e1000_filter_info *filter_info =
2840                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
2841         struct e1000_2tuple_filter *filter;
2842         uint32_t ttqf = E1000_TTQF_DISABLE_MASK;
2843         uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP;
2844         int i, ret;
2845
2846         filter = rte_zmalloc("e1000_2tuple_filter",
2847                         sizeof(struct e1000_2tuple_filter), 0);
2848         if (filter == NULL)
2849                 return -ENOMEM;
2850
2851         ret = ntuple_filter_to_2tuple(ntuple_filter,
2852                                       &filter->filter_info);
2853         if (ret < 0) {
2854                 rte_free(filter);
2855                 return ret;
2856         }
2857         if (igb_2tuple_filter_lookup(&filter_info->twotuple_list,
2858                                          &filter->filter_info) != NULL) {
2859                 PMD_DRV_LOG(ERR, "filter exists.");
2860                 rte_free(filter);
2861                 return -EEXIST;
2862         }
2863         filter->queue = ntuple_filter->queue;
2864
2865         /*
2866          * look for an unused 2tuple filter index,
2867          * and insert the filter to list.
2868          */
2869         for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) {
2870                 if (!(filter_info->twotuple_mask & (1 << i))) {
2871                         filter_info->twotuple_mask |= 1 << i;
2872                         filter->index = i;
2873                         TAILQ_INSERT_TAIL(&filter_info->twotuple_list,
2874                                           filter,
2875                                           entries);
2876                         break;
2877                 }
2878         }
2879         if (i >= E1000_MAX_TTQF_FILTERS) {
2880                 PMD_DRV_LOG(ERR, "2tuple filters are full.");
2881                 rte_free(filter);
2882                 return -ENOSYS;
2883         }
2884
2885         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
2886         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
2887                 imir |= E1000_IMIR_PORT_BP;
2888         else
2889                 imir &= ~E1000_IMIR_PORT_BP;
2890
2891         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
2892
2893         ttqf |= E1000_TTQF_QUEUE_ENABLE;
2894         ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT);
2895         ttqf |= (uint32_t)(filter->filter_info.proto & E1000_TTQF_PROTOCOL_MASK);
2896         if (filter->filter_info.proto_mask == 0)
2897                 ttqf &= ~E1000_TTQF_MASK_ENABLE;
2898
2899         /* tcp flags bits setting. */
2900         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
2901                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
2902                         imir_ext |= E1000_IMIREXT_CTRL_URG;
2903                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
2904                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
2905                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
2906                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
2907                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
2908                         imir_ext |= E1000_IMIREXT_CTRL_RST;
2909                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
2910                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
2911                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
2912                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
2913         } else
2914                 imir_ext |= E1000_IMIREXT_CTRL_BP;
2915         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
2916         E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf);
2917         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
2918         return 0;
2919 }
2920
2921 /*
2922  * igb_remove_2tuple_filter - remove a 2tuple filter
2923  *
2924  * @param
2925  * dev: Pointer to struct rte_eth_dev.
2926  * ntuple_filter: ponter to the filter that will be removed.
2927  *
2928  * @return
2929  *    - On success, zero.
2930  *    - On failure, a negative value.
2931  */
2932 static int
2933 igb_remove_2tuple_filter(struct rte_eth_dev *dev,
2934                         struct rte_eth_ntuple_filter *ntuple_filter)
2935 {
2936         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2937         struct e1000_filter_info *filter_info =
2938                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
2939         struct e1000_2tuple_filter_info filter_2tuple;
2940         struct e1000_2tuple_filter *filter;
2941         int ret;
2942
2943         memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info));
2944         ret = ntuple_filter_to_2tuple(ntuple_filter,
2945                                       &filter_2tuple);
2946         if (ret < 0)
2947                 return ret;
2948
2949         filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list,
2950                                          &filter_2tuple);
2951         if (filter == NULL) {
2952                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
2953                 return -ENOENT;
2954         }
2955
2956         filter_info->twotuple_mask &= ~(1 << filter->index);
2957         TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries);
2958         rte_free(filter);
2959
2960         E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK);
2961         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
2962         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
2963         return 0;
2964 }
2965
2966 static inline struct e1000_flex_filter *
2967 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list,
2968                         struct e1000_flex_filter_info *key)
2969 {
2970         struct e1000_flex_filter *it;
2971
2972         TAILQ_FOREACH(it, filter_list, entries) {
2973                 if (memcmp(key, &it->filter_info,
2974                         sizeof(struct e1000_flex_filter_info)) == 0)
2975                         return it;
2976         }
2977
2978         return NULL;
2979 }
2980
2981 static int
2982 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
2983                         struct rte_eth_flex_filter *filter,
2984                         bool add)
2985 {
2986         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2987         struct e1000_filter_info *filter_info =
2988                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
2989         struct e1000_flex_filter *flex_filter, *it;
2990         uint32_t wufc, queueing, mask;
2991         uint32_t reg_off;
2992         uint8_t shift, i, j = 0;
2993
2994         flex_filter = rte_zmalloc("e1000_flex_filter",
2995                         sizeof(struct e1000_flex_filter), 0);
2996         if (flex_filter == NULL)
2997                 return -ENOMEM;
2998
2999         flex_filter->filter_info.len = filter->len;
3000         flex_filter->filter_info.priority = filter->priority;
3001         memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len);
3002         for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) {
3003                 mask = 0;
3004                 /* reverse bits in flex filter's mask*/
3005                 for (shift = 0; shift < CHAR_BIT; shift++) {
3006                         if (filter->mask[i] & (0x01 << shift))
3007                                 mask |= (0x80 >> shift);
3008                 }
3009                 flex_filter->filter_info.mask[i] = mask;
3010         }
3011
3012         wufc = E1000_READ_REG(hw, E1000_WUFC);
3013         if (flex_filter->index < E1000_MAX_FHFT)
3014                 reg_off = E1000_FHFT(flex_filter->index);
3015         else
3016                 reg_off = E1000_FHFT_EXT(flex_filter->index - E1000_MAX_FHFT);
3017
3018         if (add) {
3019                 if (eth_igb_flex_filter_lookup(&filter_info->flex_list,
3020                                 &flex_filter->filter_info) != NULL) {
3021                         PMD_DRV_LOG(ERR, "filter exists.");
3022                         rte_free(flex_filter);
3023                         return -EEXIST;
3024                 }
3025                 flex_filter->queue = filter->queue;
3026                 /*
3027                  * look for an unused flex filter index
3028                  * and insert the filter into the list.
3029                  */
3030                 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) {
3031                         if (!(filter_info->flex_mask & (1 << i))) {
3032                                 filter_info->flex_mask |= 1 << i;
3033                                 flex_filter->index = i;
3034                                 TAILQ_INSERT_TAIL(&filter_info->flex_list,
3035                                         flex_filter,
3036                                         entries);
3037                                 break;
3038                         }
3039                 }
3040                 if (i >= E1000_MAX_FLEX_FILTERS) {
3041                         PMD_DRV_LOG(ERR, "flex filters are full.");
3042                         rte_free(flex_filter);
3043                         return -ENOSYS;
3044                 }
3045
3046                 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ |
3047                                 (E1000_WUFC_FLX0 << flex_filter->index));
3048                 queueing = filter->len |
3049                         (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) |
3050                         (filter->priority << E1000_FHFT_QUEUEING_PRIO_SHIFT);
3051                 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET,
3052                                 queueing);
3053                 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) {
3054                         E1000_WRITE_REG(hw, reg_off,
3055                                         flex_filter->filter_info.dwords[j]);
3056                         reg_off += sizeof(uint32_t);
3057                         E1000_WRITE_REG(hw, reg_off,
3058                                         flex_filter->filter_info.dwords[++j]);
3059                         reg_off += sizeof(uint32_t);
3060                         E1000_WRITE_REG(hw, reg_off,
3061                                 (uint32_t)flex_filter->filter_info.mask[i]);
3062                         reg_off += sizeof(uint32_t) * 2;
3063                         ++j;
3064                 }
3065         } else {
3066                 it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
3067                                 &flex_filter->filter_info);
3068                 if (it == NULL) {
3069                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3070                         rte_free(flex_filter);
3071                         return -ENOENT;
3072                 }
3073
3074                 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++)
3075                         E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0);
3076                 E1000_WRITE_REG(hw, E1000_WUFC, wufc &
3077                         (~(E1000_WUFC_FLX0 << it->index)));
3078
3079                 filter_info->flex_mask &= ~(1 << it->index);
3080                 TAILQ_REMOVE(&filter_info->flex_list, it, entries);
3081                 rte_free(it);
3082                 rte_free(flex_filter);
3083         }
3084
3085         return 0;
3086 }
3087
3088 static int
3089 eth_igb_get_flex_filter(struct rte_eth_dev *dev,
3090                         struct rte_eth_flex_filter *filter)
3091 {
3092         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3093         struct e1000_filter_info *filter_info =
3094                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3095         struct e1000_flex_filter flex_filter, *it;
3096         uint32_t wufc, queueing, wufc_en = 0;
3097
3098         memset(&flex_filter, 0, sizeof(struct e1000_flex_filter));
3099         flex_filter.filter_info.len = filter->len;
3100         flex_filter.filter_info.priority = filter->priority;
3101         memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len);
3102         memcpy(flex_filter.filter_info.mask, filter->mask,
3103                         RTE_ALIGN(filter->len, sizeof(char)) / sizeof(char));
3104
3105         it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
3106                                 &flex_filter.filter_info);
3107         if (it == NULL) {
3108                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3109                 return -ENOENT;
3110         }
3111
3112         wufc = E1000_READ_REG(hw, E1000_WUFC);
3113         wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index);
3114
3115         if ((wufc & wufc_en) == wufc_en) {
3116                 uint32_t reg_off = 0;
3117                 if (it->index < E1000_MAX_FHFT)
3118                         reg_off = E1000_FHFT(it->index);
3119                 else
3120                         reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT);
3121
3122                 queueing = E1000_READ_REG(hw,
3123                                 reg_off + E1000_FHFT_QUEUEING_OFFSET);
3124                 filter->len = queueing & E1000_FHFT_QUEUEING_LEN;
3125                 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >>
3126                         E1000_FHFT_QUEUEING_PRIO_SHIFT;
3127                 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >>
3128                         E1000_FHFT_QUEUEING_QUEUE_SHIFT;
3129                 return 0;
3130         }
3131         return -ENOENT;
3132 }
3133
3134 static int
3135 eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
3136                         enum rte_filter_op filter_op,
3137                         void *arg)
3138 {
3139         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3140         struct rte_eth_flex_filter *filter;
3141         int ret = 0;
3142
3143         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
3144
3145         if (filter_op == RTE_ETH_FILTER_NOP)
3146                 return ret;
3147
3148         if (arg == NULL) {
3149                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
3150                             filter_op);
3151                 return -EINVAL;
3152         }
3153
3154         filter = (struct rte_eth_flex_filter *)arg;
3155         if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN
3156             || filter->len % sizeof(uint64_t) != 0) {
3157                 PMD_DRV_LOG(ERR, "filter's length is out of range");
3158                 return -EINVAL;
3159         }
3160         if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) {
3161                 PMD_DRV_LOG(ERR, "filter's priority is out of range");
3162                 return -EINVAL;
3163         }
3164
3165         switch (filter_op) {
3166         case RTE_ETH_FILTER_ADD:
3167                 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE);
3168                 break;
3169         case RTE_ETH_FILTER_DELETE:
3170                 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE);
3171                 break;
3172         case RTE_ETH_FILTER_GET:
3173                 ret = eth_igb_get_flex_filter(dev, filter);
3174                 break;
3175         default:
3176                 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op);
3177                 ret = -EINVAL;
3178                 break;
3179         }
3180
3181         return ret;
3182 }
3183
3184 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/
3185 static inline int
3186 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter,
3187                         struct e1000_5tuple_filter_info *filter_info)
3188 {
3189         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576)
3190                 return -EINVAL;
3191         if (filter->priority > E1000_2TUPLE_MAX_PRI)
3192                 return -EINVAL;  /* filter index is out of range. */
3193         if (filter->tcp_flags > TCP_FLAG_ALL)
3194                 return -EINVAL;  /* flags is invalid. */
3195
3196         switch (filter->dst_ip_mask) {
3197         case UINT32_MAX:
3198                 filter_info->dst_ip_mask = 0;
3199                 filter_info->dst_ip = filter->dst_ip;
3200                 break;
3201         case 0:
3202                 filter_info->dst_ip_mask = 1;
3203                 break;
3204         default:
3205                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
3206                 return -EINVAL;
3207         }
3208
3209         switch (filter->src_ip_mask) {
3210         case UINT32_MAX:
3211                 filter_info->src_ip_mask = 0;
3212                 filter_info->src_ip = filter->src_ip;
3213                 break;
3214         case 0:
3215                 filter_info->src_ip_mask = 1;
3216                 break;
3217         default:
3218                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
3219                 return -EINVAL;
3220         }
3221
3222         switch (filter->dst_port_mask) {
3223         case UINT16_MAX:
3224                 filter_info->dst_port_mask = 0;
3225                 filter_info->dst_port = filter->dst_port;
3226                 break;
3227         case 0:
3228                 filter_info->dst_port_mask = 1;
3229                 break;
3230         default:
3231                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
3232                 return -EINVAL;
3233         }
3234
3235         switch (filter->src_port_mask) {
3236         case UINT16_MAX:
3237                 filter_info->src_port_mask = 0;
3238                 filter_info->src_port = filter->src_port;
3239                 break;
3240         case 0:
3241                 filter_info->src_port_mask = 1;
3242                 break;
3243         default:
3244                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
3245                 return -EINVAL;
3246         }
3247
3248         switch (filter->proto_mask) {
3249         case UINT8_MAX:
3250                 filter_info->proto_mask = 0;
3251                 filter_info->proto = filter->proto;
3252                 break;
3253         case 0:
3254                 filter_info->proto_mask = 1;
3255                 break;
3256         default:
3257                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
3258                 return -EINVAL;
3259         }
3260
3261         filter_info->priority = (uint8_t)filter->priority;
3262         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
3263                 filter_info->tcp_flags = filter->tcp_flags;
3264         else
3265                 filter_info->tcp_flags = 0;
3266
3267         return 0;
3268 }
3269
3270 static inline struct e1000_5tuple_filter *
3271 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list,
3272                         struct e1000_5tuple_filter_info *key)
3273 {
3274         struct e1000_5tuple_filter *it;
3275
3276         TAILQ_FOREACH(it, filter_list, entries) {
3277                 if (memcmp(key, &it->filter_info,
3278                         sizeof(struct e1000_5tuple_filter_info)) == 0) {
3279                         return it;
3280                 }
3281         }
3282         return NULL;
3283 }
3284
3285 /*
3286  * igb_add_5tuple_filter_82576 - add a 5tuple filter
3287  *
3288  * @param
3289  * dev: Pointer to struct rte_eth_dev.
3290  * ntuple_filter: ponter to the filter that will be added.
3291  *
3292  * @return
3293  *    - On success, zero.
3294  *    - On failure, a negative value.
3295  */
3296 static int
3297 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
3298                         struct rte_eth_ntuple_filter *ntuple_filter)
3299 {
3300         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3301         struct e1000_filter_info *filter_info =
3302                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3303         struct e1000_5tuple_filter *filter;
3304         uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK;
3305         uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP;
3306         uint8_t i;
3307         int ret;
3308
3309         filter = rte_zmalloc("e1000_5tuple_filter",
3310                         sizeof(struct e1000_5tuple_filter), 0);
3311         if (filter == NULL)
3312                 return -ENOMEM;
3313
3314         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3315                                             &filter->filter_info);
3316         if (ret < 0) {
3317                 rte_free(filter);
3318                 return ret;
3319         }
3320
3321         if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
3322                                          &filter->filter_info) != NULL) {
3323                 PMD_DRV_LOG(ERR, "filter exists.");
3324                 rte_free(filter);
3325                 return -EEXIST;
3326         }
3327         filter->queue = ntuple_filter->queue;
3328
3329         /*
3330          * look for an unused 5tuple filter index,
3331          * and insert the filter to list.
3332          */
3333         for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) {
3334                 if (!(filter_info->fivetuple_mask & (1 << i))) {
3335                         filter_info->fivetuple_mask |= 1 << i;
3336                         filter->index = i;
3337                         TAILQ_INSERT_TAIL(&filter_info->fivetuple_list,
3338                                           filter,
3339                                           entries);
3340                         break;
3341                 }
3342         }
3343         if (i >= E1000_MAX_FTQF_FILTERS) {
3344                 PMD_DRV_LOG(ERR, "5tuple filters are full.");
3345                 rte_free(filter);
3346                 return -ENOSYS;
3347         }
3348
3349         ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK;
3350         if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */
3351                 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP;
3352         if (filter->filter_info.dst_ip_mask == 0)
3353                 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP;
3354         if (filter->filter_info.src_port_mask == 0)
3355                 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
3356         if (filter->filter_info.proto_mask == 0)
3357                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP;
3358         ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) &
3359                 E1000_FTQF_QUEUE_MASK;
3360         ftqf |= E1000_FTQF_QUEUE_ENABLE;
3361         E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf);
3362         E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip);
3363         E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip);
3364
3365         spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT;
3366         E1000_WRITE_REG(hw, E1000_SPQF(i), spqf);
3367
3368         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
3369         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
3370                 imir |= E1000_IMIR_PORT_BP;
3371         else
3372                 imir &= ~E1000_IMIR_PORT_BP;
3373         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
3374
3375         /* tcp flags bits setting. */
3376         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
3377                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
3378                         imir_ext |= E1000_IMIREXT_CTRL_URG;
3379                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
3380                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
3381                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
3382                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
3383                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
3384                         imir_ext |= E1000_IMIREXT_CTRL_RST;
3385                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
3386                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
3387                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
3388                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
3389         } else
3390                 imir_ext |= E1000_IMIREXT_CTRL_BP;
3391         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
3392         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
3393         return 0;
3394 }
3395
3396 /*
3397  * igb_remove_5tuple_filter_82576 - remove a 5tuple filter
3398  *
3399  * @param
3400  * dev: Pointer to struct rte_eth_dev.
3401  * ntuple_filter: ponter to the filter that will be removed.
3402  *
3403  * @return
3404  *    - On success, zero.
3405  *    - On failure, a negative value.
3406  */
3407 static int
3408 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
3409                                 struct rte_eth_ntuple_filter *ntuple_filter)
3410 {
3411         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3412         struct e1000_filter_info *filter_info =
3413                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3414         struct e1000_5tuple_filter_info filter_5tuple;
3415         struct e1000_5tuple_filter *filter;
3416         int ret;
3417
3418         memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info));
3419         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3420                                             &filter_5tuple);
3421         if (ret < 0)
3422                 return ret;
3423
3424         filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
3425                                          &filter_5tuple);
3426         if (filter == NULL) {
3427                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3428                 return -ENOENT;
3429         }
3430
3431         filter_info->fivetuple_mask &= ~(1 << filter->index);
3432         TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries);
3433         rte_free(filter);
3434
3435         E1000_WRITE_REG(hw, E1000_FTQF(filter->index),
3436                         E1000_FTQF_VF_BP | E1000_FTQF_MASK);
3437         E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0);
3438         E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0);
3439         E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0);
3440         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
3441         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
3442         return 0;
3443 }
3444
3445 static int
3446 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
3447 {
3448         uint32_t rctl;
3449         struct e1000_hw *hw;
3450         struct rte_eth_dev_info dev_info;
3451         uint32_t frame_size = mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN +
3452                                      VLAN_TAG_SIZE);
3453
3454         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3455
3456 #ifdef RTE_LIBRTE_82571_SUPPORT
3457         /* XXX: not bigger than max_rx_pktlen */
3458         if (hw->mac.type == e1000_82571)
3459                 return -ENOTSUP;
3460 #endif
3461         eth_igb_infos_get(dev, &dev_info);
3462
3463         /* check that mtu is within the allowed range */
3464         if ((mtu < ETHER_MIN_MTU) ||
3465             (frame_size > dev_info.max_rx_pktlen))
3466                 return -EINVAL;
3467
3468         /* refuse mtu that requires the support of scattered packets when this
3469          * feature has not been enabled before. */
3470         if (!dev->data->scattered_rx &&
3471             frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)
3472                 return -EINVAL;
3473
3474         rctl = E1000_READ_REG(hw, E1000_RCTL);
3475
3476         /* switch to jumbo mode if needed */
3477         if (frame_size > ETHER_MAX_LEN) {
3478                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
3479                 rctl |= E1000_RCTL_LPE;
3480         } else {
3481                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
3482                 rctl &= ~E1000_RCTL_LPE;
3483         }
3484         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3485
3486         /* update max frame size */
3487         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
3488
3489         E1000_WRITE_REG(hw, E1000_RLPML,
3490                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
3491
3492         return 0;
3493 }
3494
3495 /*
3496  * igb_add_del_ntuple_filter - add or delete a ntuple filter
3497  *
3498  * @param
3499  * dev: Pointer to struct rte_eth_dev.
3500  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
3501  * add: if true, add filter, if false, remove filter
3502  *
3503  * @return
3504  *    - On success, zero.
3505  *    - On failure, a negative value.
3506  */
3507 static int
3508 igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
3509                         struct rte_eth_ntuple_filter *ntuple_filter,
3510                         bool add)
3511 {
3512         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3513         int ret;
3514
3515         switch (ntuple_filter->flags) {
3516         case RTE_5TUPLE_FLAGS:
3517         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3518                 if (hw->mac.type != e1000_82576)
3519                         return -ENOTSUP;
3520                 if (add)
3521                         ret = igb_add_5tuple_filter_82576(dev,
3522                                                           ntuple_filter);
3523                 else
3524                         ret = igb_remove_5tuple_filter_82576(dev,
3525                                                              ntuple_filter);
3526                 break;
3527         case RTE_2TUPLE_FLAGS:
3528         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3529                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
3530                         return -ENOTSUP;
3531                 if (add)
3532                         ret = igb_add_2tuple_filter(dev, ntuple_filter);
3533                 else
3534                         ret = igb_remove_2tuple_filter(dev, ntuple_filter);
3535                 break;
3536         default:
3537                 ret = -EINVAL;
3538                 break;
3539         }
3540
3541         return ret;
3542 }
3543
3544 /*
3545  * igb_get_ntuple_filter - get a ntuple filter
3546  *
3547  * @param
3548  * dev: Pointer to struct rte_eth_dev.
3549  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
3550  *
3551  * @return
3552  *    - On success, zero.
3553  *    - On failure, a negative value.
3554  */
3555 static int
3556 igb_get_ntuple_filter(struct rte_eth_dev *dev,
3557                         struct rte_eth_ntuple_filter *ntuple_filter)
3558 {
3559         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3560         struct e1000_filter_info *filter_info =
3561                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3562         struct e1000_5tuple_filter_info filter_5tuple;
3563         struct e1000_2tuple_filter_info filter_2tuple;
3564         struct e1000_5tuple_filter *p_5tuple_filter;
3565         struct e1000_2tuple_filter *p_2tuple_filter;
3566         int ret;
3567
3568         switch (ntuple_filter->flags) {
3569         case RTE_5TUPLE_FLAGS:
3570         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3571                 if (hw->mac.type != e1000_82576)
3572                         return -ENOTSUP;
3573                 memset(&filter_5tuple,
3574                         0,
3575                         sizeof(struct e1000_5tuple_filter_info));
3576                 ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3577                                                     &filter_5tuple);
3578                 if (ret < 0)
3579                         return ret;
3580                 p_5tuple_filter = igb_5tuple_filter_lookup_82576(
3581                                         &filter_info->fivetuple_list,
3582                                         &filter_5tuple);
3583                 if (p_5tuple_filter == NULL) {
3584                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3585                         return -ENOENT;
3586                 }
3587                 ntuple_filter->queue = p_5tuple_filter->queue;
3588                 break;
3589         case RTE_2TUPLE_FLAGS:
3590         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3591                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
3592                         return -ENOTSUP;
3593                 memset(&filter_2tuple,
3594                         0,
3595                         sizeof(struct e1000_2tuple_filter_info));
3596                 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple);
3597                 if (ret < 0)
3598                         return ret;
3599                 p_2tuple_filter = igb_2tuple_filter_lookup(
3600                                         &filter_info->twotuple_list,
3601                                         &filter_2tuple);
3602                 if (p_2tuple_filter == NULL) {
3603                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3604                         return -ENOENT;
3605                 }
3606                 ntuple_filter->queue = p_2tuple_filter->queue;
3607                 break;
3608         default:
3609                 ret = -EINVAL;
3610                 break;
3611         }
3612
3613         return 0;
3614 }
3615
3616 /*
3617  * igb_ntuple_filter_handle - Handle operations for ntuple filter.
3618  * @dev: pointer to rte_eth_dev structure
3619  * @filter_op:operation will be taken.
3620  * @arg: a pointer to specific structure corresponding to the filter_op
3621  */
3622 static int
3623 igb_ntuple_filter_handle(struct rte_eth_dev *dev,
3624                                 enum rte_filter_op filter_op,
3625                                 void *arg)
3626 {
3627         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3628         int ret;
3629
3630         MAC_TYPE_FILTER_SUP(hw->mac.type);
3631
3632         if (filter_op == RTE_ETH_FILTER_NOP)
3633                 return 0;
3634
3635         if (arg == NULL) {
3636                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
3637                             filter_op);
3638                 return -EINVAL;
3639         }
3640
3641         switch (filter_op) {
3642         case RTE_ETH_FILTER_ADD:
3643                 ret = igb_add_del_ntuple_filter(dev,
3644                         (struct rte_eth_ntuple_filter *)arg,
3645                         TRUE);
3646                 break;
3647         case RTE_ETH_FILTER_DELETE:
3648                 ret = igb_add_del_ntuple_filter(dev,
3649                         (struct rte_eth_ntuple_filter *)arg,
3650                         FALSE);
3651                 break;
3652         case RTE_ETH_FILTER_GET:
3653                 ret = igb_get_ntuple_filter(dev,
3654                         (struct rte_eth_ntuple_filter *)arg);
3655                 break;
3656         default:
3657                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
3658                 ret = -EINVAL;
3659                 break;
3660         }
3661         return ret;
3662 }
3663
3664 static inline int
3665 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info,
3666                         uint16_t ethertype)
3667 {
3668         int i;
3669
3670         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
3671                 if (filter_info->ethertype_filters[i] == ethertype &&
3672                     (filter_info->ethertype_mask & (1 << i)))
3673                         return i;
3674         }
3675         return -1;
3676 }
3677
3678 static inline int
3679 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info,
3680                         uint16_t ethertype)
3681 {
3682         int i;
3683
3684         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
3685                 if (!(filter_info->ethertype_mask & (1 << i))) {
3686                         filter_info->ethertype_mask |= 1 << i;
3687                         filter_info->ethertype_filters[i] = ethertype;
3688                         return i;
3689                 }
3690         }
3691         return -1;
3692 }
3693
3694 static inline int
3695 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info,
3696                         uint8_t idx)
3697 {
3698         if (idx >= E1000_MAX_ETQF_FILTERS)
3699                 return -1;
3700         filter_info->ethertype_mask &= ~(1 << idx);
3701         filter_info->ethertype_filters[idx] = 0;
3702         return idx;
3703 }
3704
3705
3706 static int
3707 igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
3708                         struct rte_eth_ethertype_filter *filter,
3709                         bool add)
3710 {
3711         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3712         struct e1000_filter_info *filter_info =
3713                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3714         uint32_t etqf = 0;
3715         int ret;
3716
3717         if (filter->ether_type == ETHER_TYPE_IPv4 ||
3718                 filter->ether_type == ETHER_TYPE_IPv6) {
3719                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
3720                         " ethertype filter.", filter->ether_type);
3721                 return -EINVAL;
3722         }
3723
3724         if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) {
3725                 PMD_DRV_LOG(ERR, "mac compare is unsupported.");
3726                 return -EINVAL;
3727         }
3728         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) {
3729                 PMD_DRV_LOG(ERR, "drop option is unsupported.");
3730                 return -EINVAL;
3731         }
3732
3733         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
3734         if (ret >= 0 && add) {
3735                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.",
3736                             filter->ether_type);
3737                 return -EEXIST;
3738         }
3739         if (ret < 0 && !add) {
3740                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
3741                             filter->ether_type);
3742                 return -ENOENT;
3743         }
3744
3745         if (add) {
3746                 ret = igb_ethertype_filter_insert(filter_info,
3747                         filter->ether_type);
3748                 if (ret < 0) {
3749                         PMD_DRV_LOG(ERR, "ethertype filters are full.");
3750                         return -ENOSYS;
3751                 }
3752
3753                 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE;
3754                 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE);
3755                 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT;
3756         } else {
3757                 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret);
3758                 if (ret < 0)
3759                         return -ENOSYS;
3760         }
3761         E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf);
3762         E1000_WRITE_FLUSH(hw);
3763
3764         return 0;
3765 }
3766
3767 static int
3768 igb_get_ethertype_filter(struct rte_eth_dev *dev,
3769                         struct rte_eth_ethertype_filter *filter)
3770 {
3771         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3772         struct e1000_filter_info *filter_info =
3773                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3774         uint32_t etqf;
3775         int ret;
3776
3777         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
3778         if (ret < 0) {
3779                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
3780                             filter->ether_type);
3781                 return -ENOENT;
3782         }
3783
3784         etqf = E1000_READ_REG(hw, E1000_ETQF(ret));
3785         if (etqf & E1000_ETQF_FILTER_ENABLE) {
3786                 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE;
3787                 filter->flags = 0;
3788                 filter->queue = (etqf & E1000_ETQF_QUEUE) >>
3789                                 E1000_ETQF_QUEUE_SHIFT;
3790                 return 0;
3791         }
3792
3793         return -ENOENT;
3794 }
3795
3796 /*
3797  * igb_ethertype_filter_handle - Handle operations for ethertype filter.
3798  * @dev: pointer to rte_eth_dev structure
3799  * @filter_op:operation will be taken.
3800  * @arg: a pointer to specific structure corresponding to the filter_op
3801  */
3802 static int
3803 igb_ethertype_filter_handle(struct rte_eth_dev *dev,
3804                                 enum rte_filter_op filter_op,
3805                                 void *arg)
3806 {
3807         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3808         int ret;
3809
3810         MAC_TYPE_FILTER_SUP(hw->mac.type);
3811
3812         if (filter_op == RTE_ETH_FILTER_NOP)
3813                 return 0;
3814
3815         if (arg == NULL) {
3816                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
3817                             filter_op);
3818                 return -EINVAL;
3819         }
3820
3821         switch (filter_op) {
3822         case RTE_ETH_FILTER_ADD:
3823                 ret = igb_add_del_ethertype_filter(dev,
3824                         (struct rte_eth_ethertype_filter *)arg,
3825                         TRUE);
3826                 break;
3827         case RTE_ETH_FILTER_DELETE:
3828                 ret = igb_add_del_ethertype_filter(dev,
3829                         (struct rte_eth_ethertype_filter *)arg,
3830                         FALSE);
3831                 break;
3832         case RTE_ETH_FILTER_GET:
3833                 ret = igb_get_ethertype_filter(dev,
3834                         (struct rte_eth_ethertype_filter *)arg);
3835                 break;
3836         default:
3837                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
3838                 ret = -EINVAL;
3839                 break;
3840         }
3841         return ret;
3842 }
3843
3844 static int
3845 eth_igb_filter_ctrl(struct rte_eth_dev *dev,
3846                      enum rte_filter_type filter_type,
3847                      enum rte_filter_op filter_op,
3848                      void *arg)
3849 {
3850         int ret = -EINVAL;
3851
3852         switch (filter_type) {
3853         case RTE_ETH_FILTER_NTUPLE:
3854                 ret = igb_ntuple_filter_handle(dev, filter_op, arg);
3855                 break;
3856         case RTE_ETH_FILTER_ETHERTYPE:
3857                 ret = igb_ethertype_filter_handle(dev, filter_op, arg);
3858                 break;
3859         case RTE_ETH_FILTER_SYN:
3860                 ret = eth_igb_syn_filter_handle(dev, filter_op, arg);
3861                 break;
3862         case RTE_ETH_FILTER_FLEXIBLE:
3863                 ret = eth_igb_flex_filter_handle(dev, filter_op, arg);
3864                 break;
3865         default:
3866                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
3867                                                         filter_type);
3868                 break;
3869         }
3870
3871         return ret;
3872 }
3873
3874 static int
3875 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
3876                          struct ether_addr *mc_addr_set,
3877                          uint32_t nb_mc_addr)
3878 {
3879         struct e1000_hw *hw;
3880
3881         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3882         e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr);
3883         return 0;
3884 }
3885
3886 static int
3887 igb_timesync_enable(struct rte_eth_dev *dev)
3888 {
3889         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3890         uint32_t tsync_ctl;
3891         uint32_t tsauxc;
3892
3893         /* Enable system time for it isn't on by default. */
3894         tsauxc = E1000_READ_REG(hw, E1000_TSAUXC);
3895         tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME;
3896         E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc);
3897
3898         /* Start incrementing the register used to timestamp PTP packets. */
3899         E1000_WRITE_REG(hw, E1000_TIMINCA, E1000_TIMINCA_INIT);
3900
3901         /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
3902         E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588),
3903                         (ETHER_TYPE_1588 |
3904                          E1000_ETQF_FILTER_ENABLE |
3905                          E1000_ETQF_1588));
3906
3907         /* Enable timestamping of received PTP packets. */
3908         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
3909         tsync_ctl |= E1000_TSYNCRXCTL_ENABLED;
3910         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl);
3911
3912         /* Enable Timestamping of transmitted PTP packets. */
3913         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
3914         tsync_ctl |= E1000_TSYNCTXCTL_ENABLED;
3915         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl);
3916
3917         return 0;
3918 }
3919
3920 static int
3921 igb_timesync_disable(struct rte_eth_dev *dev)
3922 {
3923         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3924         uint32_t tsync_ctl;
3925
3926         /* Disable timestamping of transmitted PTP packets. */
3927         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
3928         tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED;
3929         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl);
3930
3931         /* Disable timestamping of received PTP packets. */
3932         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
3933         tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED;
3934         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl);
3935
3936         /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
3937         E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0);
3938
3939         /* Stop incrementating the System Time registers. */
3940         E1000_WRITE_REG(hw, E1000_TIMINCA, 0);
3941
3942         return 0;
3943 }
3944
3945 static int
3946 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
3947                                struct timespec *timestamp,
3948                                uint32_t flags __rte_unused)
3949 {
3950         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3951         uint32_t tsync_rxctl;
3952         uint32_t rx_stmpl;
3953         uint32_t rx_stmph;
3954
3955         tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
3956         if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0)
3957                 return -EINVAL;
3958
3959         rx_stmpl = E1000_READ_REG(hw, E1000_RXSTMPL);
3960         rx_stmph = E1000_READ_REG(hw, E1000_RXSTMPH);
3961
3962         timestamp->tv_sec = (uint64_t)(((uint64_t)rx_stmph << 32) | rx_stmpl);
3963         timestamp->tv_nsec = 0;
3964
3965         return  0;
3966 }
3967
3968 static int
3969 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
3970                                struct timespec *timestamp)
3971 {
3972         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3973         uint32_t tsync_txctl;
3974         uint32_t tx_stmpl;
3975         uint32_t tx_stmph;
3976
3977         tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
3978         if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0)
3979                 return -EINVAL;
3980
3981         tx_stmpl = E1000_READ_REG(hw, E1000_TXSTMPL);
3982         tx_stmph = E1000_READ_REG(hw, E1000_TXSTMPH);
3983
3984         timestamp->tv_sec = (uint64_t)(((uint64_t)tx_stmph << 32) | tx_stmpl);
3985         timestamp->tv_nsec = 0;
3986
3987         return  0;
3988 }
3989
3990 static int
3991 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused)
3992 {
3993         int count = 0;
3994         int g_ind = 0;
3995         const struct reg_info *reg_group;
3996
3997         while ((reg_group = igb_regs[g_ind++]))
3998                 count += igb_reg_group_count(reg_group);
3999
4000         return count;
4001 }
4002
4003 static int
4004 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused)
4005 {
4006         int count = 0;
4007         int g_ind = 0;
4008         const struct reg_info *reg_group;
4009
4010         while ((reg_group = igbvf_regs[g_ind++]))
4011                 count += igb_reg_group_count(reg_group);
4012
4013         return count;
4014 }
4015
4016 static int
4017 eth_igb_get_regs(struct rte_eth_dev *dev,
4018         struct rte_dev_reg_info *regs)
4019 {
4020         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4021         uint32_t *data = regs->data;
4022         int g_ind = 0;
4023         int count = 0;
4024         const struct reg_info *reg_group;
4025
4026         /* Support only full register dump */
4027         if ((regs->length == 0) ||
4028             (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) {
4029                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4030                         hw->device_id;
4031                 while ((reg_group = igb_regs[g_ind++]))
4032                         count += igb_read_regs_group(dev, &data[count],
4033                                                         reg_group);
4034                 return 0;
4035         }
4036
4037         return -ENOTSUP;
4038 }
4039
4040 static int
4041 igbvf_get_regs(struct rte_eth_dev *dev,
4042         struct rte_dev_reg_info *regs)
4043 {
4044         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4045         uint32_t *data = regs->data;
4046         int g_ind = 0;
4047         int count = 0;
4048         const struct reg_info *reg_group;
4049
4050         /* Support only full register dump */
4051         if ((regs->length == 0) ||
4052             (regs->length == (uint32_t)igbvf_get_reg_length(dev))) {
4053                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4054                         hw->device_id;
4055                 while ((reg_group = igbvf_regs[g_ind++]))
4056                         count += igb_read_regs_group(dev, &data[count],
4057                                                         reg_group);
4058                 return 0;
4059         }
4060
4061         return -ENOTSUP;
4062 }
4063
4064 static int
4065 eth_igb_get_eeprom_length(struct rte_eth_dev *dev)
4066 {
4067         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4068
4069         /* Return unit is byte count */
4070         return hw->nvm.word_size * 2;
4071 }
4072
4073 static int
4074 eth_igb_get_eeprom(struct rte_eth_dev *dev,
4075         struct rte_dev_eeprom_info *in_eeprom)
4076 {
4077         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4078         struct e1000_nvm_info *nvm = &hw->nvm;
4079         uint16_t *data = in_eeprom->data;
4080         int first, length;
4081
4082         first = in_eeprom->offset >> 1;
4083         length = in_eeprom->length >> 1;
4084         if ((first >= hw->nvm.word_size) ||
4085             ((first + length) >= hw->nvm.word_size))
4086                 return -EINVAL;
4087
4088         in_eeprom->magic = hw->vendor_id |
4089                 ((uint32_t)hw->device_id << 16);
4090
4091         if ((nvm->ops.read) == NULL)
4092                 return -ENOTSUP;
4093
4094         return nvm->ops.read(hw, first, length, data);
4095 }
4096
4097 static int
4098 eth_igb_set_eeprom(struct rte_eth_dev *dev,
4099         struct rte_dev_eeprom_info *in_eeprom)
4100 {
4101         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4102         struct e1000_nvm_info *nvm = &hw->nvm;
4103         uint16_t *data = in_eeprom->data;
4104         int first, length;
4105
4106         first = in_eeprom->offset >> 1;
4107         length = in_eeprom->length >> 1;
4108         if ((first >= hw->nvm.word_size) ||
4109             ((first + length) >= hw->nvm.word_size))
4110                 return -EINVAL;
4111
4112         in_eeprom->magic = (uint32_t)hw->vendor_id |
4113                 ((uint32_t)hw->device_id << 16);
4114
4115         if ((nvm->ops.write) == NULL)
4116                 return -ENOTSUP;
4117         return nvm->ops.write(hw,  first, length, data);
4118 }
4119
4120 static struct rte_driver pmd_igb_drv = {
4121         .type = PMD_PDEV,
4122         .init = rte_igb_pmd_init,
4123 };
4124
4125 static struct rte_driver pmd_igbvf_drv = {
4126         .type = PMD_PDEV,
4127         .init = rte_igbvf_pmd_init,
4128 };
4129
4130 static int
4131 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
4132 {
4133         struct e1000_hw *hw =
4134                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4135         uint32_t mask = 1 << queue_id;
4136
4137         E1000_WRITE_REG(hw, E1000_EIMC, mask);
4138         E1000_WRITE_FLUSH(hw);
4139
4140         return 0;
4141 }
4142
4143 static int
4144 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
4145 {
4146         struct e1000_hw *hw =
4147                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4148         uint32_t mask = 1 << queue_id;
4149         uint32_t regval;
4150
4151         regval = E1000_READ_REG(hw, E1000_EIMS);
4152         E1000_WRITE_REG(hw, E1000_EIMS, regval | mask);
4153         E1000_WRITE_FLUSH(hw);
4154
4155         rte_intr_enable(&dev->pci_dev->intr_handle);
4156
4157         return 0;
4158 }
4159
4160 static void
4161 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t  msix_vector,
4162                    uint8_t index, uint8_t offset)
4163 {
4164         uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4165
4166         /* clear bits */
4167         val &= ~((uint32_t)0xFF << offset);
4168
4169         /* write vector and valid bit */
4170         val |= (msix_vector | E1000_IVAR_VALID) << offset;
4171
4172         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val);
4173 }
4174
4175 static void
4176 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction,
4177                            uint8_t queue, uint8_t msix_vector)
4178 {
4179         uint32_t tmp = 0;
4180
4181         if (hw->mac.type == e1000_82575) {
4182                 if (direction == 0)
4183                         tmp = E1000_EICR_RX_QUEUE0 << queue;
4184                 else if (direction == 1)
4185                         tmp = E1000_EICR_TX_QUEUE0 << queue;
4186                 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp);
4187         } else if (hw->mac.type == e1000_82576) {
4188                 if ((direction == 0) || (direction == 1))
4189                         eth_igb_write_ivar(hw, msix_vector, queue & 0x7,
4190                                            ((queue & 0x8) << 1) +
4191                                            8 * direction);
4192         } else if ((hw->mac.type == e1000_82580) ||
4193                         (hw->mac.type == e1000_i350) ||
4194                         (hw->mac.type == e1000_i354) ||
4195                         (hw->mac.type == e1000_i210) ||
4196                         (hw->mac.type == e1000_i211)) {
4197                 if ((direction == 0) || (direction == 1))
4198                         eth_igb_write_ivar(hw, msix_vector,
4199                                            queue >> 1,
4200                                            ((queue & 0x1) << 4) +
4201                                            8 * direction);
4202         }
4203 }
4204
4205 /* Sets up the hardware to generate MSI-X interrupts properly
4206  * @hw
4207  *  board private structure
4208  */
4209 static void
4210 eth_igb_configure_msix_intr(struct rte_eth_dev *dev)
4211 {
4212         int queue_id;
4213         uint32_t tmpval, regval, intr_mask;
4214         struct e1000_hw *hw =
4215                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4216         uint32_t vec = 0;
4217         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
4218
4219         /* won't configure msix register if no mapping is done
4220          * between intr vector and event fd
4221          */
4222         if (!rte_intr_dp_is_en(intr_handle))
4223                 return;
4224
4225         /* set interrupt vector for other causes */
4226         if (hw->mac.type == e1000_82575) {
4227                 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT);
4228                 /* enable MSI-X PBA support */
4229                 tmpval |= E1000_CTRL_EXT_PBA_CLR;
4230
4231                 /* Auto-Mask interrupts upon ICR read */
4232                 tmpval |= E1000_CTRL_EXT_EIAME;
4233                 tmpval |= E1000_CTRL_EXT_IRCA;
4234
4235                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval);
4236
4237                 /* enable msix_other interrupt */
4238                 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER);
4239                 regval = E1000_READ_REG(hw, E1000_EIAC);
4240                 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER);
4241                 regval = E1000_READ_REG(hw, E1000_EIAM);
4242                 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER);
4243         } else if ((hw->mac.type == e1000_82576) ||
4244                         (hw->mac.type == e1000_82580) ||
4245                         (hw->mac.type == e1000_i350) ||
4246                         (hw->mac.type == e1000_i354) ||
4247                         (hw->mac.type == e1000_i210) ||
4248                         (hw->mac.type == e1000_i211)) {
4249                 /* turn on MSI-X capability first */
4250                 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE |
4251                                         E1000_GPIE_PBA | E1000_GPIE_EIAME |
4252                                         E1000_GPIE_NSICR);
4253
4254                 intr_mask = (1 << intr_handle->max_intr) - 1;
4255                 regval = E1000_READ_REG(hw, E1000_EIAC);
4256                 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask);
4257
4258                 /* enable msix_other interrupt */
4259                 regval = E1000_READ_REG(hw, E1000_EIMS);
4260                 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask);
4261                 tmpval = (dev->data->nb_rx_queues | E1000_IVAR_VALID) << 8;
4262                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval);
4263         }
4264
4265         /* use EIAM to auto-mask when MSI-X interrupt
4266          * is asserted, this saves a register write for every interrupt
4267          */
4268         intr_mask = (1 << intr_handle->nb_efd) - 1;
4269         regval = E1000_READ_REG(hw, E1000_EIAM);
4270         E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask);
4271
4272         for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) {
4273                 eth_igb_assign_msix_vector(hw, 0, queue_id, vec);
4274                 intr_handle->intr_vec[queue_id] = vec;
4275                 if (vec < intr_handle->nb_efd - 1)
4276                         vec++;
4277         }
4278
4279         E1000_WRITE_FLUSH(hw);
4280 }
4281
4282 PMD_REGISTER_DRIVER(pmd_igb_drv);
4283 PMD_REGISTER_DRIVER(pmd_igbvf_drv);