drivers/net/e1000/igb_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <stdarg.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_interrupts.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_log.h>
  44 #include <rte_debug.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_memory.h>
  49 #include <rte_memzone.h>
  50 #include <rte_eal.h>
  51 #include <rte_atomic.h>
  52 #include <rte_malloc.h>
  53 #include <rte_dev.h>
  54
  55 #include "e1000_logs.h"
  56 #include "base/e1000_api.h"
  57 #include "e1000_ethdev.h"
  58 #include "igb_regs.h"
  59
  60 /*
  61  * Default values for port configuration
  62  */
  63 #define IGB_DEFAULT_RX_FREE_THRESH  32
  64 #define IGB_DEFAULT_RX_PTHRESH      8
  65 #define IGB_DEFAULT_RX_HTHRESH      8
  66 #define IGB_DEFAULT_RX_WTHRESH      0
  67
  68 #define IGB_DEFAULT_TX_PTHRESH      32
  69 #define IGB_DEFAULT_TX_HTHRESH      0
  70 #define IGB_DEFAULT_TX_WTHRESH      0
  71
  72 #define IGB_HKEY_MAX_INDEX 10
  73
  74 /* Bit shift and mask */
  75 #define IGB_4_BIT_WIDTH  (CHAR_BIT / 2)
  76 #define IGB_4_BIT_MASK   RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t)
  77 #define IGB_8_BIT_WIDTH  CHAR_BIT
  78 #define IGB_8_BIT_MASK   UINT8_MAX
  79
  80 /* Additional timesync values. */
  81 #define E1000_ETQF_FILTER_1588 3
  82 #define E1000_TIMINCA_INCVALUE 16000000
  83 #define E1000_TIMINCA_INIT     ((0x02 << E1000_TIMINCA_16NS_SHIFT) \
  84                                 | E1000_TIMINCA_INCVALUE)
  85 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000
  86
  87 static int  eth_igb_configure(struct rte_eth_dev *dev);
  88 static int  eth_igb_start(struct rte_eth_dev *dev);
  89 static void eth_igb_stop(struct rte_eth_dev *dev);
  90 static void eth_igb_close(struct rte_eth_dev *dev);
  91 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev);
  92 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev);
  93 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev);
  94 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev);
  95 static int  eth_igb_link_update(struct rte_eth_dev *dev,
  96                                 int wait_to_complete);
  97 static void eth_igb_stats_get(struct rte_eth_dev *dev,
  98                                 struct rte_eth_stats *rte_stats);
  99 static void eth_igb_stats_reset(struct rte_eth_dev *dev);
 100 static void eth_igb_infos_get(struct rte_eth_dev *dev,
 101                               struct rte_eth_dev_info *dev_info);
 102 static void eth_igbvf_infos_get(struct rte_eth_dev *dev,
 103                                 struct rte_eth_dev_info *dev_info);
 104 static int  eth_igb_flow_ctrl_get(struct rte_eth_dev *dev,
 105                                 struct rte_eth_fc_conf *fc_conf);
 106 static int  eth_igb_flow_ctrl_set(struct rte_eth_dev *dev,
 107                                 struct rte_eth_fc_conf *fc_conf);
 108 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev);
 109 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev);
 110 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev);
 111 static int eth_igb_interrupt_action(struct rte_eth_dev *dev);
 112 static void eth_igb_interrupt_handler(struct rte_intr_handle *handle,
 113                                                         void *param);
 114 static int  igb_hardware_init(struct e1000_hw *hw);
 115 static void igb_hw_control_acquire(struct e1000_hw *hw);
 116 static void igb_hw_control_release(struct e1000_hw *hw);
 117 static void igb_init_manageability(struct e1000_hw *hw);
 118 static void igb_release_manageability(struct e1000_hw *hw);
 119
 120 static int  eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 121
 122 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev,
 123                 uint16_t vlan_id, int on);
 124 static void eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 125 static void eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 126
 127 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 128 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 129 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev);
 130 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev);
 131 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 132 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 133
 134 static int eth_igb_led_on(struct rte_eth_dev *dev);
 135 static int eth_igb_led_off(struct rte_eth_dev *dev);
 136
 137 static void igb_intr_disable(struct e1000_hw *hw);
 138 static int  igb_get_rx_buffer_size(struct e1000_hw *hw);
 139 static void eth_igb_rar_set(struct rte_eth_dev *dev,
 140                 struct ether_addr *mac_addr,
 141                 uint32_t index, uint32_t pool);
 142 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 143 static void eth_igb_default_mac_addr_set(struct rte_eth_dev *dev,
 144                 struct ether_addr *addr);
 145
 146 static void igbvf_intr_disable(struct e1000_hw *hw);
 147 static int igbvf_dev_configure(struct rte_eth_dev *dev);
 148 static int igbvf_dev_start(struct rte_eth_dev *dev);
 149 static void igbvf_dev_stop(struct rte_eth_dev *dev);
 150 static void igbvf_dev_close(struct rte_eth_dev *dev);
 151 static int eth_igbvf_link_update(struct e1000_hw *hw);
 152 static void eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats);
 153 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev);
 154 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev,
 155                 uint16_t vlan_id, int on);
 156 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on);
 157 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 158 static void igbvf_default_mac_addr_set(struct rte_eth_dev *dev,
 159                 struct ether_addr *addr);
 160 static int igbvf_get_reg_length(struct rte_eth_dev *dev);
 161 static int igbvf_get_regs(struct rte_eth_dev *dev,
 162                 struct rte_dev_reg_info *regs);
 163
 164 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev,
 165                                    struct rte_eth_rss_reta_entry64 *reta_conf,
 166                                    uint16_t reta_size);
 167 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev,
 168                                   struct rte_eth_rss_reta_entry64 *reta_conf,
 169                                   uint16_t reta_size);
 170
 171 static int eth_igb_syn_filter_set(struct rte_eth_dev *dev,
 172                         struct rte_eth_syn_filter *filter,
 173                         bool add);
 174 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev,
 175                         struct rte_eth_syn_filter *filter);
 176 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
 177                         enum rte_filter_op filter_op,
 178                         void *arg);
 179 static int igb_add_2tuple_filter(struct rte_eth_dev *dev,
 180                         struct rte_eth_ntuple_filter *ntuple_filter);
 181 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev,
 182                         struct rte_eth_ntuple_filter *ntuple_filter);
 183 static int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
 184                         struct rte_eth_flex_filter *filter,
 185                         bool add);
 186 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev,
 187                         struct rte_eth_flex_filter *filter);
 188 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
 189                         enum rte_filter_op filter_op,
 190                         void *arg);
 191 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
 192                         struct rte_eth_ntuple_filter *ntuple_filter);
 193 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
 194                         struct rte_eth_ntuple_filter *ntuple_filter);
 195 static int igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
 196                         struct rte_eth_ntuple_filter *filter,
 197                         bool add);
 198 static int igb_get_ntuple_filter(struct rte_eth_dev *dev,
 199                         struct rte_eth_ntuple_filter *filter);
 200 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev,
 201                                 enum rte_filter_op filter_op,
 202                                 void *arg);
 203 static int igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
 204                         struct rte_eth_ethertype_filter *filter,
 205                         bool add);
 206 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev,
 207                                 enum rte_filter_op filter_op,
 208                                 void *arg);
 209 static int igb_get_ethertype_filter(struct rte_eth_dev *dev,
 210                         struct rte_eth_ethertype_filter *filter);
 211 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev,
 212                      enum rte_filter_type filter_type,
 213                      enum rte_filter_op filter_op,
 214                      void *arg);
 215 static int eth_igb_get_reg_length(struct rte_eth_dev *dev);
 216 static int eth_igb_get_regs(struct rte_eth_dev *dev,
 217                 struct rte_dev_reg_info *regs);
 218 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev);
 219 static int eth_igb_get_eeprom(struct rte_eth_dev *dev,
 220                 struct rte_dev_eeprom_info *eeprom);
 221 static int eth_igb_set_eeprom(struct rte_eth_dev *dev,
 222                 struct rte_dev_eeprom_info *eeprom);
 223 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
 224                                     struct ether_addr *mc_addr_set,
 225                                     uint32_t nb_mc_addr);
 226 static int igb_timesync_enable(struct rte_eth_dev *dev);
 227 static int igb_timesync_disable(struct rte_eth_dev *dev);
 228 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 229                                           struct timespec *timestamp,
 230                                           uint32_t flags);
 231 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 232                                           struct timespec *timestamp);
 233 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev,
 234                                         uint16_t queue_id);
 235 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev,
 236                                          uint16_t queue_id);
 237 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction,
 238                                        uint8_t queue, uint8_t msix_vector);
 239 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector,
 240                                uint8_t index, uint8_t offset);
 241 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev);
 242
 243 /*
 244  * Define VF Stats MACRO for Non "cleared on read" register
 245  */
 246 #define UPDATE_VF_STAT(reg, last, cur)            \
 247 {                                                 \
 248         u32 latest = E1000_READ_REG(hw, reg);     \
 249         cur += (latest - last) & UINT_MAX;        \
 250         last = latest;                            \
 251 }
 252
 253 #define IGB_FC_PAUSE_TIME 0x0680
 254 #define IGB_LINK_UPDATE_CHECK_TIMEOUT  90  /* 9s */
 255 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */
 256
 257 #define IGBVF_PMD_NAME "rte_igbvf_pmd"     /* PMD name */
 258
 259 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full;
 260
 261 /*
 262  * The set of PCI devices this driver supports
 263  */
 264 static const struct rte_pci_id pci_id_igb_map[] = {
 265
 266 #define RTE_PCI_DEV_ID_DECL_IGB(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 267 #include "rte_pci_dev_ids.h"
 268
 269 {0},
 270 };
 271
 272 /*
 273  * The set of PCI devices this driver supports (for 82576&I350 VF)
 274  */
 275 static const struct rte_pci_id pci_id_igbvf_map[] = {
 276
 277 #define RTE_PCI_DEV_ID_DECL_IGBVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 278 #include "rte_pci_dev_ids.h"
 279
 280 {0},
 281 };
 282
 283 static const struct eth_dev_ops eth_igb_ops = {
 284         .dev_configure        = eth_igb_configure,
 285         .dev_start            = eth_igb_start,
 286         .dev_stop             = eth_igb_stop,
 287         .dev_close            = eth_igb_close,
 288         .promiscuous_enable   = eth_igb_promiscuous_enable,
 289         .promiscuous_disable  = eth_igb_promiscuous_disable,
 290         .allmulticast_enable  = eth_igb_allmulticast_enable,
 291         .allmulticast_disable = eth_igb_allmulticast_disable,
 292         .link_update          = eth_igb_link_update,
 293         .stats_get            = eth_igb_stats_get,
 294         .stats_reset          = eth_igb_stats_reset,
 295         .dev_infos_get        = eth_igb_infos_get,
 296         .mtu_set              = eth_igb_mtu_set,
 297         .vlan_filter_set      = eth_igb_vlan_filter_set,
 298         .vlan_tpid_set        = eth_igb_vlan_tpid_set,
 299         .vlan_offload_set     = eth_igb_vlan_offload_set,
 300         .rx_queue_setup       = eth_igb_rx_queue_setup,
 301         .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable,
 302         .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable,
 303         .rx_queue_release     = eth_igb_rx_queue_release,
 304         .rx_queue_count       = eth_igb_rx_queue_count,
 305         .rx_descriptor_done   = eth_igb_rx_descriptor_done,
 306         .tx_queue_setup       = eth_igb_tx_queue_setup,
 307         .tx_queue_release     = eth_igb_tx_queue_release,
 308         .dev_led_on           = eth_igb_led_on,
 309         .dev_led_off          = eth_igb_led_off,
 310         .flow_ctrl_get        = eth_igb_flow_ctrl_get,
 311         .flow_ctrl_set        = eth_igb_flow_ctrl_set,
 312         .mac_addr_add         = eth_igb_rar_set,
 313         .mac_addr_remove      = eth_igb_rar_clear,
 314         .mac_addr_set         = eth_igb_default_mac_addr_set,
 315         .reta_update          = eth_igb_rss_reta_update,
 316         .reta_query           = eth_igb_rss_reta_query,
 317         .rss_hash_update      = eth_igb_rss_hash_update,
 318         .rss_hash_conf_get    = eth_igb_rss_hash_conf_get,
 319         .filter_ctrl          = eth_igb_filter_ctrl,
 320         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 321         .timesync_enable      = igb_timesync_enable,
 322         .timesync_disable     = igb_timesync_disable,
 323         .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp,
 324         .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp,
 325         .get_reg_length       = eth_igb_get_reg_length,
 326         .get_reg              = eth_igb_get_regs,
 327         .get_eeprom_length    = eth_igb_get_eeprom_length,
 328         .get_eeprom           = eth_igb_get_eeprom,
 329         .set_eeprom           = eth_igb_set_eeprom,
 330 };
 331
 332 /*
 333  * dev_ops for virtual function, bare necessities for basic vf
 334  * operation have been implemented
 335  */
 336 static const struct eth_dev_ops igbvf_eth_dev_ops = {
 337         .dev_configure        = igbvf_dev_configure,
 338         .dev_start            = igbvf_dev_start,
 339         .dev_stop             = igbvf_dev_stop,
 340         .dev_close            = igbvf_dev_close,
 341         .link_update          = eth_igb_link_update,
 342         .stats_get            = eth_igbvf_stats_get,
 343         .stats_reset          = eth_igbvf_stats_reset,
 344         .vlan_filter_set      = igbvf_vlan_filter_set,
 345         .dev_infos_get        = eth_igbvf_infos_get,
 346         .rx_queue_setup       = eth_igb_rx_queue_setup,
 347         .rx_queue_release     = eth_igb_rx_queue_release,
 348         .tx_queue_setup       = eth_igb_tx_queue_setup,
 349         .tx_queue_release     = eth_igb_tx_queue_release,
 350         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 351         .mac_addr_set         = igbvf_default_mac_addr_set,
 352         .get_reg_length       = igbvf_get_reg_length,
 353         .get_reg              = igbvf_get_regs,
 354 };
 355
 356 /**
 357  * Atomically reads the link status information from global
 358  * structure rte_eth_dev.
 359  *
 360  * @param dev
 361  *   - Pointer to the structure rte_eth_dev to read from.
 362  *   - Pointer to the buffer to be saved with the link status.
 363  *
 364  * @return
 365  *   - On success, zero.
 366  *   - On failure, negative value.
 367  */
 368 static inline int
 369 rte_igb_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 370                                 struct rte_eth_link *link)
 371 {
 372         struct rte_eth_link *dst = link;
 373         struct rte_eth_link *src = &(dev->data->dev_link);
 374
 375         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 376                                         *(uint64_t *)src) == 0)
 377                 return -1;
 378
 379         return 0;
 380 }
 381
 382 /**
 383  * Atomically writes the link status information into global
 384  * structure rte_eth_dev.
 385  *
 386  * @param dev
 387  *   - Pointer to the structure rte_eth_dev to read from.
 388  *   - Pointer to the buffer to be saved with the link status.
 389  *
 390  * @return
 391  *   - On success, zero.
 392  *   - On failure, negative value.
 393  */
 394 static inline int
 395 rte_igb_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 396                                 struct rte_eth_link *link)
 397 {
 398         struct rte_eth_link *dst = &(dev->data->dev_link);
 399         struct rte_eth_link *src = link;
 400
 401         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 402                                         *(uint64_t *)src) == 0)
 403                 return -1;
 404
 405         return 0;
 406 }
 407
 408 static inline void
 409 igb_intr_enable(struct rte_eth_dev *dev)
 410 {
 411         struct e1000_interrupt *intr =
 412                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 413         struct e1000_hw *hw =
 414                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 415
 416         E1000_WRITE_REG(hw, E1000_IMS, intr->mask);
 417         E1000_WRITE_FLUSH(hw);
 418 }
 419
 420 static void
 421 igb_intr_disable(struct e1000_hw *hw)
 422 {
 423         E1000_WRITE_REG(hw, E1000_IMC, ~0);
 424         E1000_WRITE_FLUSH(hw);
 425 }
 426
 427 static inline int32_t
 428 igb_pf_reset_hw(struct e1000_hw *hw)
 429 {
 430         uint32_t ctrl_ext;
 431         int32_t status;
 432
 433         status = e1000_reset_hw(hw);
 434
 435         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 436         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 437         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 438         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 439         E1000_WRITE_FLUSH(hw);
 440
 441         return status;
 442 }
 443
 444 static void
 445 igb_identify_hardware(struct rte_eth_dev *dev)
 446 {
 447         struct e1000_hw *hw =
 448                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 449
 450         hw->vendor_id = dev->pci_dev->id.vendor_id;
 451         hw->device_id = dev->pci_dev->id.device_id;
 452         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
 453         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
 454
 455         e1000_set_mac_type(hw);
 456
 457         /* need to check if it is a vf device below */
 458 }
 459
 460 static int
 461 igb_reset_swfw_lock(struct e1000_hw *hw)
 462 {
 463         int ret_val;
 464
 465         /*
 466          * Do mac ops initialization manually here, since we will need
 467          * some function pointers set by this call.
 468          */
 469         ret_val = e1000_init_mac_params(hw);
 470         if (ret_val)
 471                 return ret_val;
 472
 473         /*
 474          * SMBI lock should not fail in this early stage. If this is the case,
 475          * it is due to an improper exit of the application.
 476          * So force the release of the faulty lock.
 477          */
 478         if (e1000_get_hw_semaphore_generic(hw) < 0) {
 479                 PMD_DRV_LOG(DEBUG, "SMBI lock released");
 480         }
 481         e1000_put_hw_semaphore_generic(hw);
 482
 483         if (hw->mac.ops.acquire_swfw_sync != NULL) {
 484                 uint16_t mask;
 485
 486                 /*
 487                  * Phy lock should not fail in this early stage. If this is the case,
 488                  * it is due to an improper exit of the application.
 489                  * So force the release of the faulty lock.
 490                  */
 491                 mask = E1000_SWFW_PHY0_SM << hw->bus.func;
 492                 if (hw->bus.func > E1000_FUNC_1)
 493                         mask <<= 2;
 494                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 495                         PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released",
 496                                     hw->bus.func);
 497                 }
 498                 hw->mac.ops.release_swfw_sync(hw, mask);
 499
 500                 /*
 501                  * This one is more tricky since it is common to all ports; but
 502                  * swfw_sync retries last long enough (1s) to be almost sure that if
 503                  * lock can not be taken it is due to an improper lock of the
 504                  * semaphore.
 505                  */
 506                 mask = E1000_SWFW_EEP_SM;
 507                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 508                         PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 509                 }
 510                 hw->mac.ops.release_swfw_sync(hw, mask);
 511         }
 512
 513         return E1000_SUCCESS;
 514 }
 515
 516 static int
 517 eth_igb_dev_init(struct rte_eth_dev *eth_dev)
 518 {
 519         int error = 0;
 520         struct rte_pci_device *pci_dev;
 521         struct e1000_hw *hw =
 522                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 523         struct e1000_vfta * shadow_vfta =
 524                 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 525         struct e1000_filter_info *filter_info =
 526                 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
 527         struct e1000_adapter *adapter =
 528                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 529
 530         uint32_t ctrl_ext;
 531
 532         pci_dev = eth_dev->pci_dev;
 533         eth_dev->dev_ops = &eth_igb_ops;
 534         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 535         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 536
 537         /* for secondary processes, we don't initialise any further as primary
 538          * has already done this work. Only check we don't need a different
 539          * RX function */
 540         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 541                 if (eth_dev->data->scattered_rx)
 542                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 543                 return 0;
 544         }
 545
 546         hw->hw_addr= (void *)pci_dev->mem_resource[0].addr;
 547
 548         igb_identify_hardware(eth_dev);
 549         if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) {
 550                 error = -EIO;
 551                 goto err_late;
 552         }
 553
 554         e1000_get_bus_info(hw);
 555
 556         /* Reset any pending lock */
 557         if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) {
 558                 error = -EIO;
 559                 goto err_late;
 560         }
 561
 562         /* Finish initialization */
 563         if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) {
 564                 error = -EIO;
 565                 goto err_late;
 566         }
 567
 568         hw->mac.autoneg = 1;
 569         hw->phy.autoneg_wait_to_complete = 0;
 570         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 571
 572         /* Copper options */
 573         if (hw->phy.media_type == e1000_media_type_copper) {
 574                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
 575                 hw->phy.disable_polarity_correction = 0;
 576                 hw->phy.ms_type = e1000_ms_hw_default;
 577         }
 578
 579         /*
 580          * Start from a known state, this is important in reading the nvm
 581          * and mac from that.
 582          */
 583         igb_pf_reset_hw(hw);
 584
 585         /* Make sure we have a good EEPROM before we read from it */
 586         if (e1000_validate_nvm_checksum(hw) < 0) {
 587                 /*
 588                  * Some PCI-E parts fail the first check due to
 589                  * the link being in sleep state, call it again,
 590                  * if it fails a second time its a real issue.
 591                  */
 592                 if (e1000_validate_nvm_checksum(hw) < 0) {
 593                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
 594                         error = -EIO;
 595                         goto err_late;
 596                 }
 597         }
 598
 599         /* Read the permanent MAC address out of the EEPROM */
 600         if (e1000_read_mac_addr(hw) != 0) {
 601                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
 602                 error = -EIO;
 603                 goto err_late;
 604         }
 605
 606         /* Allocate memory for storing MAC addresses */
 607         eth_dev->data->mac_addrs = rte_zmalloc("e1000",
 608                 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
 609         if (eth_dev->data->mac_addrs == NULL) {
 610                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to "
 611                                                 "store MAC addresses",
 612                                 ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 613                 error = -ENOMEM;
 614                 goto err_late;
 615         }
 616
 617         /* Copy the permanent MAC address */
 618         ether_addr_copy((struct ether_addr *)hw->mac.addr, &eth_dev->data->mac_addrs[0]);
 619
 620         /* initialize the vfta */
 621         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 622
 623         /* Now initialize the hardware */
 624         if (igb_hardware_init(hw) != 0) {
 625                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
 626                 rte_free(eth_dev->data->mac_addrs);
 627                 eth_dev->data->mac_addrs = NULL;
 628                 error = -ENODEV;
 629                 goto err_late;
 630         }
 631         hw->mac.get_link_status = 1;
 632         adapter->stopped = 0;
 633
 634         /* Indicate SOL/IDER usage */
 635         if (e1000_check_reset_block(hw) < 0) {
 636                 PMD_INIT_LOG(ERR, "PHY reset is blocked due to"
 637                                         "SOL/IDER session");
 638         }
 639
 640         /* initialize PF if max_vfs not zero */
 641         igb_pf_host_init(eth_dev);
 642
 643         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 644         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 645         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 646         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 647         E1000_WRITE_FLUSH(hw);
 648
 649         PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x",
 650                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 651                      pci_dev->id.device_id);
 652
 653         /* enable support intr */
 654         igb_intr_enable(eth_dev);
 655
 656         TAILQ_INIT(&filter_info->flex_list);
 657         filter_info->flex_mask = 0;
 658         TAILQ_INIT(&filter_info->twotuple_list);
 659         filter_info->twotuple_mask = 0;
 660         TAILQ_INIT(&filter_info->fivetuple_list);
 661         filter_info->fivetuple_mask = 0;
 662
 663         return 0;
 664
 665 err_late:
 666         igb_hw_control_release(hw);
 667
 668         return (error);
 669 }
 670
 671 static int
 672 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev)
 673 {
 674         struct rte_pci_device *pci_dev;
 675         struct e1000_hw *hw;
 676         struct e1000_adapter *adapter =
 677                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 678
 679         PMD_INIT_FUNC_TRACE();
 680
 681         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 682                 return -EPERM;
 683
 684         hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 685         pci_dev = eth_dev->pci_dev;
 686
 687         if (adapter->stopped == 0)
 688                 eth_igb_close(eth_dev);
 689
 690         eth_dev->dev_ops = NULL;
 691         eth_dev->rx_pkt_burst = NULL;
 692         eth_dev->tx_pkt_burst = NULL;
 693
 694         /* Reset any pending lock */
 695         igb_reset_swfw_lock(hw);
 696
 697         rte_free(eth_dev->data->mac_addrs);
 698         eth_dev->data->mac_addrs = NULL;
 699
 700         /* uninitialize PF if max_vfs not zero */
 701         igb_pf_host_uninit(eth_dev);
 702
 703         /* disable uio intr before callback unregister */
 704         rte_intr_disable(&(pci_dev->intr_handle));
 705         rte_intr_callback_unregister(&(pci_dev->intr_handle),
 706                 eth_igb_interrupt_handler, (void *)eth_dev);
 707
 708         return 0;
 709 }
 710
 711 /*
 712  * Virtual Function device init
 713  */
 714 static int
 715 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev)
 716 {
 717         struct rte_pci_device *pci_dev;
 718         struct e1000_adapter *adapter =
 719                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 720         struct e1000_hw *hw =
 721                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 722         int diag;
 723
 724         PMD_INIT_FUNC_TRACE();
 725
 726         eth_dev->dev_ops = &igbvf_eth_dev_ops;
 727         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 728         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 729
 730         /* for secondary processes, we don't initialise any further as primary
 731          * has already done this work. Only check we don't need a different
 732          * RX function */
 733         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 734                 if (eth_dev->data->scattered_rx)
 735                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 736                 return 0;
 737         }
 738
 739         pci_dev = eth_dev->pci_dev;
 740
 741         hw->device_id = pci_dev->id.device_id;
 742         hw->vendor_id = pci_dev->id.vendor_id;
 743         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 744         adapter->stopped = 0;
 745
 746         /* Initialize the shared code (base driver) */
 747         diag = e1000_setup_init_funcs(hw, TRUE);
 748         if (diag != 0) {
 749                 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d",
 750                         diag);
 751                 return -EIO;
 752         }
 753
 754         /* init_mailbox_params */
 755         hw->mbx.ops.init_params(hw);
 756
 757         /* Disable the interrupts for VF */
 758         igbvf_intr_disable(hw);
 759
 760         diag = hw->mac.ops.reset_hw(hw);
 761
 762         /* Allocate memory for storing MAC addresses */
 763         eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN *
 764                 hw->mac.rar_entry_count, 0);
 765         if (eth_dev->data->mac_addrs == NULL) {
 766                 PMD_INIT_LOG(ERR,
 767                         "Failed to allocate %d bytes needed to store MAC "
 768                         "addresses",
 769                         ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 770                 return -ENOMEM;
 771         }
 772
 773         /* Copy the permanent MAC address */
 774         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 775                         &eth_dev->data->mac_addrs[0]);
 776
 777         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x "
 778                      "mac.type=%s",
 779                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 780                      pci_dev->id.device_id, "igb_mac_82576_vf");
 781
 782         return 0;
 783 }
 784
 785 static int
 786 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev)
 787 {
 788         struct e1000_adapter *adapter =
 789                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 790
 791         PMD_INIT_FUNC_TRACE();
 792
 793         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 794                 return -EPERM;
 795
 796         if (adapter->stopped == 0)
 797                 igbvf_dev_close(eth_dev);
 798
 799         eth_dev->dev_ops = NULL;
 800         eth_dev->rx_pkt_burst = NULL;
 801         eth_dev->tx_pkt_burst = NULL;
 802
 803         rte_free(eth_dev->data->mac_addrs);
 804         eth_dev->data->mac_addrs = NULL;
 805
 806         return 0;
 807 }
 808
 809 static struct eth_driver rte_igb_pmd = {
 810         .pci_drv = {
 811                 .name = "rte_igb_pmd",
 812                 .id_table = pci_id_igb_map,
 813                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
 814                         RTE_PCI_DRV_DETACHABLE,
 815         },
 816         .eth_dev_init = eth_igb_dev_init,
 817         .eth_dev_uninit = eth_igb_dev_uninit,
 818         .dev_private_size = sizeof(struct e1000_adapter),
 819 };
 820
 821 /*
 822  * virtual function driver struct
 823  */
 824 static struct eth_driver rte_igbvf_pmd = {
 825         .pci_drv = {
 826                 .name = "rte_igbvf_pmd",
 827                 .id_table = pci_id_igbvf_map,
 828                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
 829         },
 830         .eth_dev_init = eth_igbvf_dev_init,
 831         .eth_dev_uninit = eth_igbvf_dev_uninit,
 832         .dev_private_size = sizeof(struct e1000_adapter),
 833 };
 834
 835 static int
 836 rte_igb_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 837 {
 838         rte_eth_driver_register(&rte_igb_pmd);
 839         return 0;
 840 }
 841
 842 static void
 843 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 844 {
 845         struct e1000_hw *hw =
 846                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 847         /* RCTL: enable VLAN filter since VMDq always use VLAN filter */
 848         uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL);
 849         rctl |= E1000_RCTL_VFE;
 850         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
 851 }
 852
 853 /*
 854  * VF Driver initialization routine.
 855  * Invoked one at EAL init time.
 856  * Register itself as the [Virtual Poll Mode] Driver of PCI IGB devices.
 857  */
 858 static int
 859 rte_igbvf_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 860 {
 861         PMD_INIT_FUNC_TRACE();
 862
 863         rte_eth_driver_register(&rte_igbvf_pmd);
 864         return (0);
 865 }
 866
 867 static int
 868 igb_check_mq_mode(struct rte_eth_dev *dev)
 869 {
 870         enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
 871         enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode;
 872         uint16_t nb_rx_q = dev->data->nb_rx_queues;
 873         uint16_t nb_tx_q = dev->data->nb_rx_queues;
 874
 875         if ((rx_mq_mode & ETH_MQ_RX_DCB_FLAG) ||
 876             tx_mq_mode == ETH_MQ_TX_DCB ||
 877             tx_mq_mode == ETH_MQ_TX_VMDQ_DCB) {
 878                 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
 879                 return -EINVAL;
 880         }
 881         if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
 882                 /* Check multi-queue mode.
 883                  * To no break software we accept ETH_MQ_RX_NONE as this might
 884                  * be used to turn off VLAN filter.
 885                  */
 886
 887                 if (rx_mq_mode == ETH_MQ_RX_NONE ||
 888                     rx_mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
 889                         dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY;
 890                         RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1;
 891                 } else {
 892                         /* Only support one queue on VFs.
 893                          * RSS together with SRIOV is not supported.
 894                          */
 895                         PMD_INIT_LOG(ERR, "SRIOV is active,"
 896                                         " wrong mq_mode rx %d.",
 897                                         rx_mq_mode);
 898                         return -EINVAL;
 899                 }
 900                 /* TX mode is not used here, so mode might be ignored.*/
 901                 if (tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) {
 902                         /* SRIOV only works in VMDq enable mode */
 903                         PMD_INIT_LOG(WARNING, "SRIOV is active,"
 904                                         " TX mode %d is not supported. "
 905                                         " Driver will behave as %d mode.",
 906                                         tx_mq_mode, ETH_MQ_TX_VMDQ_ONLY);
 907                 }
 908
 909                 /* check valid queue number */
 910                 if ((nb_rx_q > 1) || (nb_tx_q > 1)) {
 911                         PMD_INIT_LOG(ERR, "SRIOV is active,"
 912                                         " only support one queue on VFs.");
 913                         return -EINVAL;
 914                 }
 915         } else {
 916                 /* To no break software that set invalid mode, only display
 917                  * warning if invalid mode is used.
 918                  */
 919                 if (rx_mq_mode != ETH_MQ_RX_NONE &&
 920                     rx_mq_mode != ETH_MQ_RX_VMDQ_ONLY &&
 921                     rx_mq_mode != ETH_MQ_RX_RSS) {
 922                         /* RSS together with VMDq not supported*/
 923                         PMD_INIT_LOG(ERR, "RX mode %d is not supported.",
 924                                      rx_mq_mode);
 925                         return -EINVAL;
 926                 }
 927
 928                 if (tx_mq_mode != ETH_MQ_TX_NONE &&
 929                     tx_mq_mode != ETH_MQ_TX_VMDQ_ONLY) {
 930                         PMD_INIT_LOG(WARNING, "TX mode %d is not supported."
 931                                         " Due to txmode is meaningless in this"
 932                                         " driver, just ignore.",
 933                                         tx_mq_mode);
 934                 }
 935         }
 936         return 0;
 937 }
 938
 939 static int
 940 eth_igb_configure(struct rte_eth_dev *dev)
 941 {
 942         struct e1000_interrupt *intr =
 943                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 944         int ret;
 945
 946         PMD_INIT_FUNC_TRACE();
 947
 948         /* multipe queue mode checking */
 949         ret  = igb_check_mq_mode(dev);
 950         if (ret != 0) {
 951                 PMD_DRV_LOG(ERR, "igb_check_mq_mode fails with %d.",
 952                             ret);
 953                 return ret;
 954         }
 955
 956         intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
 957         PMD_INIT_FUNC_TRACE();
 958
 959         return 0;
 960 }
 961
 962 static int
 963 eth_igb_start(struct rte_eth_dev *dev)
 964 {
 965         struct e1000_hw *hw =
 966                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 967         struct e1000_adapter *adapter =
 968                 E1000_DEV_PRIVATE(dev->data->dev_private);
 969         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 970         int ret, mask;
 971         uint32_t intr_vector = 0;
 972         uint32_t ctrl_ext;
 973
 974         PMD_INIT_FUNC_TRACE();
 975
 976         /* Power up the phy. Needed to make the link go Up */
 977         e1000_power_up_phy(hw);
 978
 979         /*
 980          * Packet Buffer Allocation (PBA)
 981          * Writing PBA sets the receive portion of the buffer
 982          * the remainder is used for the transmit buffer.
 983          */
 984         if (hw->mac.type == e1000_82575) {
 985                 uint32_t pba;
 986
 987                 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
 988                 E1000_WRITE_REG(hw, E1000_PBA, pba);
 989         }
 990
 991         /* Put the address into the Receive Address Array */
 992         e1000_rar_set(hw, hw->mac.addr, 0);
 993
 994         /* Initialize the hardware */
 995         if (igb_hardware_init(hw)) {
 996                 PMD_INIT_LOG(ERR, "Unable to initialize the hardware");
 997                 return (-EIO);
 998         }
 999         adapter->stopped = 0;
1000
1001         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1002
1003         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1004         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
1005         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
1006         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
1007         E1000_WRITE_FLUSH(hw);
1008
1009         /* configure PF module if SRIOV enabled */
1010         igb_pf_host_configure(dev);
1011
1012         /* check and configure queue intr-vector mapping */
1013         if (dev->data->dev_conf.intr_conf.rxq != 0)
1014                 intr_vector = dev->data->nb_rx_queues;
1015
1016         if (rte_intr_efd_enable(intr_handle, intr_vector))
1017                 return -1;
1018
1019         if (rte_intr_dp_is_en(intr_handle)) {
1020                 intr_handle->intr_vec =
1021                         rte_zmalloc("intr_vec",
1022                                     dev->data->nb_rx_queues * sizeof(int), 0);
1023                 if (intr_handle->intr_vec == NULL) {
1024                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
1025                                      " intr_vec\n", dev->data->nb_rx_queues);
1026                         return -ENOMEM;
1027                 }
1028         }
1029
1030         /* confiugre msix for rx interrupt */
1031         eth_igb_configure_msix_intr(dev);
1032
1033         /* Configure for OS presence */
1034         igb_init_manageability(hw);
1035
1036         eth_igb_tx_init(dev);
1037
1038         /* This can fail when allocating mbufs for descriptor rings */
1039         ret = eth_igb_rx_init(dev);
1040         if (ret) {
1041                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
1042                 igb_dev_clear_queues(dev);
1043                 return ret;
1044         }
1045
1046         e1000_clear_hw_cntrs_base_generic(hw);
1047
1048         /*
1049          * VLAN Offload Settings
1050          */
1051         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
1052                         ETH_VLAN_EXTEND_MASK;
1053         eth_igb_vlan_offload_set(dev, mask);
1054
1055         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
1056                 /* Enable VLAN filter since VMDq always use VLAN filter */
1057                 igb_vmdq_vlan_hw_filter_enable(dev);
1058         }
1059
1060         if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) ||
1061                 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) ||
1062                 (hw->mac.type == e1000_i211)) {
1063                 /* Configure EITR with the maximum possible value (0xFFFF) */
1064                 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF);
1065         }
1066
1067         /* Setup link speed and duplex */
1068         switch (dev->data->dev_conf.link_speed) {
1069         case ETH_LINK_SPEED_AUTONEG:
1070                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
1071                         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
1072                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
1073                         hw->phy.autoneg_advertised = E1000_ALL_HALF_DUPLEX;
1074                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
1075                         hw->phy.autoneg_advertised = E1000_ALL_FULL_DUPLEX;
1076                 else
1077                         goto error_invalid_config;
1078                 break;
1079         case ETH_LINK_SPEED_10:
1080                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
1081                         hw->phy.autoneg_advertised = E1000_ALL_10_SPEED;
1082                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
1083                         hw->phy.autoneg_advertised = ADVERTISE_10_HALF;
1084                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
1085                         hw->phy.autoneg_advertised = ADVERTISE_10_FULL;
1086                 else
1087                         goto error_invalid_config;
1088                 break;
1089         case ETH_LINK_SPEED_100:
1090                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
1091                         hw->phy.autoneg_advertised = E1000_ALL_100_SPEED;
1092                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
1093                         hw->phy.autoneg_advertised = ADVERTISE_100_HALF;
1094                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
1095                         hw->phy.autoneg_advertised = ADVERTISE_100_FULL;
1096                 else
1097                         goto error_invalid_config;
1098                 break;
1099         case ETH_LINK_SPEED_1000:
1100                 if ((dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX) ||
1101                                 (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX))
1102                         hw->phy.autoneg_advertised = ADVERTISE_1000_FULL;
1103                 else
1104                         goto error_invalid_config;
1105                 break;
1106         case ETH_LINK_SPEED_10000:
1107         default:
1108                 goto error_invalid_config;
1109         }
1110         e1000_setup_link(hw);
1111
1112         /* check if lsc interrupt feature is enabled */
1113         if (dev->data->dev_conf.intr_conf.lsc != 0) {
1114                 if (rte_intr_allow_others(intr_handle)) {
1115                         rte_intr_callback_register(intr_handle,
1116                                                    eth_igb_interrupt_handler,
1117                                                    (void *)dev);
1118                         eth_igb_lsc_interrupt_setup(dev);
1119                 } else
1120                         PMD_INIT_LOG(INFO, "lsc won't enable because of"
1121                                      " no intr multiplex\n");
1122         }
1123
1124         /* check if rxq interrupt is enabled */
1125         if (dev->data->dev_conf.intr_conf.rxq != 0)
1126                 eth_igb_rxq_interrupt_setup(dev);
1127
1128         /* enable uio/vfio intr/eventfd mapping */
1129         rte_intr_enable(intr_handle);
1130
1131         /* resume enabled intr since hw reset */
1132         igb_intr_enable(dev);
1133
1134         PMD_INIT_LOG(DEBUG, "<<");
1135
1136         return (0);
1137
1138 error_invalid_config:
1139         PMD_INIT_LOG(ERR, "Invalid link_speed/link_duplex (%u/%u) for port %u",
1140                      dev->data->dev_conf.link_speed,
1141                      dev->data->dev_conf.link_duplex, dev->data->port_id);
1142         igb_dev_clear_queues(dev);
1143         return (-EINVAL);
1144 }
1145
1146 /*********************************************************************
1147  *
1148  *  This routine disables all traffic on the adapter by issuing a
1149  *  global reset on the MAC.
1150  *
1151  **********************************************************************/
1152 static void
1153 eth_igb_stop(struct rte_eth_dev *dev)
1154 {
1155         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1156         struct e1000_filter_info *filter_info =
1157                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
1158         struct rte_eth_link link;
1159         struct e1000_flex_filter *p_flex;
1160         struct e1000_5tuple_filter *p_5tuple, *p_5tuple_next;
1161         struct e1000_2tuple_filter *p_2tuple, *p_2tuple_next;
1162         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1163
1164         igb_intr_disable(hw);
1165
1166         /* disable intr eventfd mapping */
1167         rte_intr_disable(intr_handle);
1168
1169         igb_pf_reset_hw(hw);
1170         E1000_WRITE_REG(hw, E1000_WUC, 0);
1171
1172         /* Set bit for Go Link disconnect */
1173         if (hw->mac.type >= e1000_82580) {
1174                 uint32_t phpm_reg;
1175
1176                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1177                 phpm_reg |= E1000_82580_PM_GO_LINKD;
1178                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1179         }
1180
1181         /* Power down the phy. Needed to make the link go Down */
1182         if (hw->phy.media_type == e1000_media_type_copper)
1183                 e1000_power_down_phy(hw);
1184         else
1185                 e1000_shutdown_fiber_serdes_link(hw);
1186
1187         igb_dev_clear_queues(dev);
1188
1189         /* clear the recorded link status */
1190         memset(&link, 0, sizeof(link));
1191         rte_igb_dev_atomic_write_link_status(dev, &link);
1192
1193         /* Remove all flex filters of the device */
1194         while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) {
1195                 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries);
1196                 rte_free(p_flex);
1197         }
1198         filter_info->flex_mask = 0;
1199
1200         /* Remove all ntuple filters of the device */
1201         for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list);
1202              p_5tuple != NULL; p_5tuple = p_5tuple_next) {
1203                 p_5tuple_next = TAILQ_NEXT(p_5tuple, entries);
1204                 TAILQ_REMOVE(&filter_info->fivetuple_list,
1205                              p_5tuple, entries);
1206                 rte_free(p_5tuple);
1207         }
1208         filter_info->fivetuple_mask = 0;
1209         for (p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list);
1210              p_2tuple != NULL; p_2tuple = p_2tuple_next) {
1211                 p_2tuple_next = TAILQ_NEXT(p_2tuple, entries);
1212                 TAILQ_REMOVE(&filter_info->twotuple_list,
1213                              p_2tuple, entries);
1214                 rte_free(p_2tuple);
1215         }
1216         filter_info->twotuple_mask = 0;
1217
1218         /* Clean datapath event and queue/vec mapping */
1219         rte_intr_efd_disable(intr_handle);
1220         if (intr_handle->intr_vec != NULL) {
1221                 rte_free(intr_handle->intr_vec);
1222                 intr_handle->intr_vec = NULL;
1223         }
1224 }
1225
1226 static void
1227 eth_igb_close(struct rte_eth_dev *dev)
1228 {
1229         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1230         struct e1000_adapter *adapter =
1231                 E1000_DEV_PRIVATE(dev->data->dev_private);
1232         struct rte_eth_link link;
1233         struct rte_pci_device *pci_dev;
1234
1235         eth_igb_stop(dev);
1236         adapter->stopped = 1;
1237
1238         e1000_phy_hw_reset(hw);
1239         igb_release_manageability(hw);
1240         igb_hw_control_release(hw);
1241
1242         /* Clear bit for Go Link disconnect */
1243         if (hw->mac.type >= e1000_82580) {
1244                 uint32_t phpm_reg;
1245
1246                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1247                 phpm_reg &= ~E1000_82580_PM_GO_LINKD;
1248                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1249         }
1250
1251         igb_dev_free_queues(dev);
1252
1253         pci_dev = dev->pci_dev;
1254         if (pci_dev->intr_handle.intr_vec) {
1255                 rte_free(pci_dev->intr_handle.intr_vec);
1256                 pci_dev->intr_handle.intr_vec = NULL;
1257         }
1258
1259         memset(&link, 0, sizeof(link));
1260         rte_igb_dev_atomic_write_link_status(dev, &link);
1261 }
1262
1263 static int
1264 igb_get_rx_buffer_size(struct e1000_hw *hw)
1265 {
1266         uint32_t rx_buf_size;
1267         if (hw->mac.type == e1000_82576) {
1268                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10;
1269         } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) {
1270                 /* PBS needs to be translated according to a lookup table */
1271                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf);
1272                 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size);
1273                 rx_buf_size = (rx_buf_size << 10);
1274         } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) {
1275                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10;
1276         } else {
1277                 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10;
1278         }
1279
1280         return rx_buf_size;
1281 }
1282
1283 /*********************************************************************
1284  *
1285  *  Initialize the hardware
1286  *
1287  **********************************************************************/
1288 static int
1289 igb_hardware_init(struct e1000_hw *hw)
1290 {
1291         uint32_t rx_buf_size;
1292         int diag;
1293
1294         /* Let the firmware know the OS is in control */
1295         igb_hw_control_acquire(hw);
1296
1297         /*
1298          * These parameters control the automatic generation (Tx) and
1299          * response (Rx) to Ethernet PAUSE frames.
1300          * - High water mark should allow for at least two standard size (1518)
1301          *   frames to be received after sending an XOFF.
1302          * - Low water mark works best when it is very near the high water mark.
1303          *   This allows the receiver to restart by sending XON when it has
1304          *   drained a bit. Here we use an arbitrary value of 1500 which will
1305          *   restart after one full frame is pulled from the buffer. There
1306          *   could be several smaller frames in the buffer and if so they will
1307          *   not trigger the XON until their total number reduces the buffer
1308          *   by 1500.
1309          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
1310          */
1311         rx_buf_size = igb_get_rx_buffer_size(hw);
1312
1313         hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2);
1314         hw->fc.low_water = hw->fc.high_water - 1500;
1315         hw->fc.pause_time = IGB_FC_PAUSE_TIME;
1316         hw->fc.send_xon = 1;
1317
1318         /* Set Flow control, use the tunable location if sane */
1319         if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4))
1320                 hw->fc.requested_mode = igb_fc_setting;
1321         else
1322                 hw->fc.requested_mode = e1000_fc_none;
1323
1324         /* Issue a global reset */
1325         igb_pf_reset_hw(hw);
1326         E1000_WRITE_REG(hw, E1000_WUC, 0);
1327
1328         diag = e1000_init_hw(hw);
1329         if (diag < 0)
1330                 return (diag);
1331
1332         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1333         e1000_get_phy_info(hw);
1334         e1000_check_for_link(hw);
1335
1336         return (0);
1337 }
1338
1339 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */
1340 static void
1341 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1342 {
1343         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1344         struct e1000_hw_stats *stats =
1345                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1346         int pause_frames;
1347
1348         if(hw->phy.media_type == e1000_media_type_copper ||
1349             (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
1350                 stats->symerrs +=
1351                     E1000_READ_REG(hw,E1000_SYMERRS);
1352                 stats->sec += E1000_READ_REG(hw, E1000_SEC);
1353         }
1354
1355         stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
1356         stats->mpc += E1000_READ_REG(hw, E1000_MPC);
1357         stats->scc += E1000_READ_REG(hw, E1000_SCC);
1358         stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
1359
1360         stats->mcc += E1000_READ_REG(hw, E1000_MCC);
1361         stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
1362         stats->colc += E1000_READ_REG(hw, E1000_COLC);
1363         stats->dc += E1000_READ_REG(hw, E1000_DC);
1364         stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
1365         stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
1366         stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
1367         /*
1368         ** For watchdog management we need to know if we have been
1369         ** paused during the last interval, so capture that here.
1370         */
1371         pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC);
1372         stats->xoffrxc += pause_frames;
1373         stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
1374         stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
1375         stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
1376         stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
1377         stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
1378         stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
1379         stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
1380         stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
1381         stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
1382         stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
1383         stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
1384         stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
1385
1386         /* For the 64-bit byte counters the low dword must be read first. */
1387         /* Both registers clear on the read of the high dword */
1388
1389         stats->gorc += E1000_READ_REG(hw, E1000_GORCL);
1390         stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32);
1391         stats->gotc += E1000_READ_REG(hw, E1000_GOTCL);
1392         stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32);
1393
1394         stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
1395         stats->ruc += E1000_READ_REG(hw, E1000_RUC);
1396         stats->rfc += E1000_READ_REG(hw, E1000_RFC);
1397         stats->roc += E1000_READ_REG(hw, E1000_ROC);
1398         stats->rjc += E1000_READ_REG(hw, E1000_RJC);
1399
1400         stats->tor += E1000_READ_REG(hw, E1000_TORL);
1401         stats->tor += ((uint64_t)E1000_READ_REG(hw, E1000_TORH) << 32);
1402         stats->tot += E1000_READ_REG(hw, E1000_TOTL);
1403         stats->tot += ((uint64_t)E1000_READ_REG(hw, E1000_TOTH) << 32);
1404
1405         stats->tpr += E1000_READ_REG(hw, E1000_TPR);
1406         stats->tpt += E1000_READ_REG(hw, E1000_TPT);
1407         stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
1408         stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
1409         stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
1410         stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
1411         stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
1412         stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
1413         stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
1414         stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
1415
1416         /* Interrupt Counts */
1417
1418         stats->iac += E1000_READ_REG(hw, E1000_IAC);
1419         stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
1420         stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
1421         stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
1422         stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
1423         stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
1424         stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
1425         stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
1426         stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
1427
1428         /* Host to Card Statistics */
1429
1430         stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
1431         stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
1432         stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
1433         stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
1434         stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
1435         stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
1436         stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
1437         stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL);
1438         stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32);
1439         stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL);
1440         stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32);
1441         stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
1442         stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
1443         stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
1444
1445         stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
1446         stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
1447         stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
1448         stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
1449         stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
1450         stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
1451
1452         if (rte_stats == NULL)
1453                 return;
1454
1455         /* Rx Errors */
1456         rte_stats->ibadcrc = stats->crcerrs;
1457         rte_stats->ibadlen = stats->rlec + stats->ruc + stats->roc;
1458         rte_stats->imissed = stats->mpc;
1459         rte_stats->ierrors = rte_stats->ibadcrc +
1460                              rte_stats->ibadlen +
1461                              rte_stats->imissed +
1462                              stats->rxerrc + stats->algnerrc + stats->cexterr;
1463
1464         /* Tx Errors */
1465         rte_stats->oerrors = stats->ecol + stats->latecol;
1466
1467         /* XON/XOFF pause frames */
1468         rte_stats->tx_pause_xon  = stats->xontxc;
1469         rte_stats->rx_pause_xon  = stats->xonrxc;
1470         rte_stats->tx_pause_xoff = stats->xofftxc;
1471         rte_stats->rx_pause_xoff = stats->xoffrxc;
1472
1473         rte_stats->ipackets = stats->gprc;
1474         rte_stats->opackets = stats->gptc;
1475         rte_stats->ibytes   = stats->gorc;
1476         rte_stats->obytes   = stats->gotc;
1477 }
1478
1479 static void
1480 eth_igb_stats_reset(struct rte_eth_dev *dev)
1481 {
1482         struct e1000_hw_stats *hw_stats =
1483                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1484
1485         /* HW registers are cleared on read */
1486         eth_igb_stats_get(dev, NULL);
1487
1488         /* Reset software totals */
1489         memset(hw_stats, 0, sizeof(*hw_stats));
1490 }
1491
1492 static void
1493 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1494 {
1495         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1496         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1497                           E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1498
1499         /* Good Rx packets, include VF loopback */
1500         UPDATE_VF_STAT(E1000_VFGPRC,
1501             hw_stats->last_gprc, hw_stats->gprc);
1502
1503         /* Good Rx octets, include VF loopback */
1504         UPDATE_VF_STAT(E1000_VFGORC,
1505             hw_stats->last_gorc, hw_stats->gorc);
1506
1507         /* Good Tx packets, include VF loopback */
1508         UPDATE_VF_STAT(E1000_VFGPTC,
1509             hw_stats->last_gptc, hw_stats->gptc);
1510
1511         /* Good Tx octets, include VF loopback */
1512         UPDATE_VF_STAT(E1000_VFGOTC,
1513             hw_stats->last_gotc, hw_stats->gotc);
1514
1515         /* Rx Multicst packets */
1516         UPDATE_VF_STAT(E1000_VFMPRC,
1517             hw_stats->last_mprc, hw_stats->mprc);
1518
1519         /* Good Rx loopback packets */
1520         UPDATE_VF_STAT(E1000_VFGPRLBC,
1521             hw_stats->last_gprlbc, hw_stats->gprlbc);
1522
1523         /* Good Rx loopback octets */
1524         UPDATE_VF_STAT(E1000_VFGORLBC,
1525             hw_stats->last_gorlbc, hw_stats->gorlbc);
1526
1527         /* Good Tx loopback packets */
1528         UPDATE_VF_STAT(E1000_VFGPTLBC,
1529             hw_stats->last_gptlbc, hw_stats->gptlbc);
1530
1531         /* Good Tx loopback octets */
1532         UPDATE_VF_STAT(E1000_VFGOTLBC,
1533             hw_stats->last_gotlbc, hw_stats->gotlbc);
1534
1535         if (rte_stats == NULL)
1536                 return;
1537
1538         rte_stats->ipackets = hw_stats->gprc;
1539         rte_stats->ibytes = hw_stats->gorc;
1540         rte_stats->opackets = hw_stats->gptc;
1541         rte_stats->obytes = hw_stats->gotc;
1542         rte_stats->imcasts = hw_stats->mprc;
1543         rte_stats->ilbpackets = hw_stats->gprlbc;
1544         rte_stats->ilbbytes = hw_stats->gorlbc;
1545         rte_stats->olbpackets = hw_stats->gptlbc;
1546         rte_stats->olbbytes = hw_stats->gotlbc;
1547
1548 }
1549
1550 static void
1551 eth_igbvf_stats_reset(struct rte_eth_dev *dev)
1552 {
1553         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1554                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1555
1556         /* Sync HW register to the last stats */
1557         eth_igbvf_stats_get(dev, NULL);
1558
1559         /* reset HW current stats*/
1560         memset(&hw_stats->gprc, 0, sizeof(*hw_stats) -
1561                offsetof(struct e1000_vf_stats, gprc));
1562
1563 }
1564
1565 static void
1566 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1567 {
1568         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1569
1570         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1571         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1572         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1573         dev_info->rx_offload_capa =
1574                 DEV_RX_OFFLOAD_VLAN_STRIP |
1575                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1576                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1577                 DEV_RX_OFFLOAD_TCP_CKSUM;
1578         dev_info->tx_offload_capa =
1579                 DEV_TX_OFFLOAD_VLAN_INSERT |
1580                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1581                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1582                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1583                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
1584                 DEV_TX_OFFLOAD_TCP_TSO;
1585
1586         switch (hw->mac.type) {
1587         case e1000_82575:
1588                 dev_info->max_rx_queues = 4;
1589                 dev_info->max_tx_queues = 4;
1590                 dev_info->max_vmdq_pools = 0;
1591                 break;
1592
1593         case e1000_82576:
1594                 dev_info->max_rx_queues = 16;
1595                 dev_info->max_tx_queues = 16;
1596                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1597                 dev_info->vmdq_queue_num = 16;
1598                 break;
1599
1600         case e1000_82580:
1601                 dev_info->max_rx_queues = 8;
1602                 dev_info->max_tx_queues = 8;
1603                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1604                 dev_info->vmdq_queue_num = 8;
1605                 break;
1606
1607         case e1000_i350:
1608                 dev_info->max_rx_queues = 8;
1609                 dev_info->max_tx_queues = 8;
1610                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1611                 dev_info->vmdq_queue_num = 8;
1612                 break;
1613
1614         case e1000_i354:
1615                 dev_info->max_rx_queues = 8;
1616                 dev_info->max_tx_queues = 8;
1617                 break;
1618
1619         case e1000_i210:
1620                 dev_info->max_rx_queues = 4;
1621                 dev_info->max_tx_queues = 4;
1622                 dev_info->max_vmdq_pools = 0;
1623                 break;
1624
1625         case e1000_i211:
1626                 dev_info->max_rx_queues = 2;
1627                 dev_info->max_tx_queues = 2;
1628                 dev_info->max_vmdq_pools = 0;
1629                 break;
1630
1631         default:
1632                 /* Should not happen */
1633                 break;
1634         }
1635         dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t);
1636         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
1637         dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL;
1638
1639         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1640                 .rx_thresh = {
1641                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1642                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1643                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
1644                 },
1645                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
1646                 .rx_drop_en = 0,
1647         };
1648
1649         dev_info->default_txconf = (struct rte_eth_txconf) {
1650                 .tx_thresh = {
1651                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
1652                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
1653                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
1654                 },
1655                 .txq_flags = 0,
1656         };
1657 }
1658
1659 static void
1660 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1661 {
1662         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1663
1664         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1665         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1666         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1667         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
1668                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1669                                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1670                                 DEV_RX_OFFLOAD_TCP_CKSUM;
1671         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
1672                                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1673                                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1674                                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1675                                 DEV_TX_OFFLOAD_SCTP_CKSUM  |
1676                                 DEV_TX_OFFLOAD_TCP_TSO;
1677         switch (hw->mac.type) {
1678         case e1000_vfadapt:
1679                 dev_info->max_rx_queues = 2;
1680                 dev_info->max_tx_queues = 2;
1681                 break;
1682         case e1000_vfadapt_i350:
1683                 dev_info->max_rx_queues = 1;
1684                 dev_info->max_tx_queues = 1;
1685                 break;
1686         default:
1687                 /* Should not happen */
1688                 break;
1689         }
1690
1691         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1692                 .rx_thresh = {
1693                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1694                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1695                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
1696                 },
1697                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
1698                 .rx_drop_en = 0,
1699         };
1700
1701         dev_info->default_txconf = (struct rte_eth_txconf) {
1702                 .tx_thresh = {
1703                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
1704                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
1705                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
1706                 },
1707                 .txq_flags = 0,
1708         };
1709 }
1710
1711 /* return 0 means link status changed, -1 means not changed */
1712 static int
1713 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1714 {
1715         struct e1000_hw *hw =
1716                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1717         struct rte_eth_link link, old;
1718         int link_check, count;
1719
1720         link_check = 0;
1721         hw->mac.get_link_status = 1;
1722
1723         /* possible wait-to-complete in up to 9 seconds */
1724         for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) {
1725                 /* Read the real link status */
1726                 switch (hw->phy.media_type) {
1727                 case e1000_media_type_copper:
1728                         /* Do the work to read phy */
1729                         e1000_check_for_link(hw);
1730                         link_check = !hw->mac.get_link_status;
1731                         break;
1732
1733                 case e1000_media_type_fiber:
1734                         e1000_check_for_link(hw);
1735                         link_check = (E1000_READ_REG(hw, E1000_STATUS) &
1736                                       E1000_STATUS_LU);
1737                         break;
1738
1739                 case e1000_media_type_internal_serdes:
1740                         e1000_check_for_link(hw);
1741                         link_check = hw->mac.serdes_has_link;
1742                         break;
1743
1744                 /* VF device is type_unknown */
1745                 case e1000_media_type_unknown:
1746                         eth_igbvf_link_update(hw);
1747                         link_check = !hw->mac.get_link_status;
1748                         break;
1749
1750                 default:
1751                         break;
1752                 }
1753                 if (link_check || wait_to_complete == 0)
1754                         break;
1755                 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL);
1756         }
1757         memset(&link, 0, sizeof(link));
1758         rte_igb_dev_atomic_read_link_status(dev, &link);
1759         old = link;
1760
1761         /* Now we check if a transition has happened */
1762         if (link_check) {
1763                 hw->mac.ops.get_link_up_info(hw, &link.link_speed,
1764                                           &link.link_duplex);
1765                 link.link_status = 1;
1766         } else if (!link_check) {
1767                 link.link_speed = 0;
1768                 link.link_duplex = 0;
1769                 link.link_status = 0;
1770         }
1771         rte_igb_dev_atomic_write_link_status(dev, &link);
1772
1773         /* not changed */
1774         if (old.link_status == link.link_status)
1775                 return -1;
1776
1777         /* changed */
1778         return 0;
1779 }
1780
1781 /*
1782  * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
1783  * For ASF and Pass Through versions of f/w this means
1784  * that the driver is loaded.
1785  */
1786 static void
1787 igb_hw_control_acquire(struct e1000_hw *hw)
1788 {
1789         uint32_t ctrl_ext;
1790
1791         /* Let firmware know the driver has taken over */
1792         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1793         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1794 }
1795
1796 /*
1797  * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
1798  * For ASF and Pass Through versions of f/w this means that the
1799  * driver is no longer loaded.
1800  */
1801 static void
1802 igb_hw_control_release(struct e1000_hw *hw)
1803 {
1804         uint32_t ctrl_ext;
1805
1806         /* Let firmware taken over control of h/w */
1807         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1808         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1809                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1810 }
1811
1812 /*
1813  * Bit of a misnomer, what this really means is
1814  * to enable OS management of the system... aka
1815  * to disable special hardware management features.
1816  */
1817 static void
1818 igb_init_manageability(struct e1000_hw *hw)
1819 {
1820         if (e1000_enable_mng_pass_thru(hw)) {
1821                 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H);
1822                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
1823
1824                 /* disable hardware interception of ARP */
1825                 manc &= ~(E1000_MANC_ARP_EN);
1826
1827                 /* enable receiving management packets to the host */
1828                 manc |= E1000_MANC_EN_MNG2HOST;
1829                 manc2h |= 1 << 5;  /* Mng Port 623 */
1830                 manc2h |= 1 << 6;  /* Mng Port 664 */
1831                 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h);
1832                 E1000_WRITE_REG(hw, E1000_MANC, manc);
1833         }
1834 }
1835
1836 static void
1837 igb_release_manageability(struct e1000_hw *hw)
1838 {
1839         if (e1000_enable_mng_pass_thru(hw)) {
1840                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
1841
1842                 manc |= E1000_MANC_ARP_EN;
1843                 manc &= ~E1000_MANC_EN_MNG2HOST;
1844
1845                 E1000_WRITE_REG(hw, E1000_MANC, manc);
1846         }
1847 }
1848
1849 static void
1850 eth_igb_promiscuous_enable(struct rte_eth_dev *dev)
1851 {
1852         struct e1000_hw *hw =
1853                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1854         uint32_t rctl;
1855
1856         rctl = E1000_READ_REG(hw, E1000_RCTL);
1857         rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1858         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1859 }
1860
1861 static void
1862 eth_igb_promiscuous_disable(struct rte_eth_dev *dev)
1863 {
1864         struct e1000_hw *hw =
1865                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1866         uint32_t rctl;
1867
1868         rctl = E1000_READ_REG(hw, E1000_RCTL);
1869         rctl &= (~E1000_RCTL_UPE);
1870         if (dev->data->all_multicast == 1)
1871                 rctl |= E1000_RCTL_MPE;
1872         else
1873                 rctl &= (~E1000_RCTL_MPE);
1874         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1875 }
1876
1877 static void
1878 eth_igb_allmulticast_enable(struct rte_eth_dev *dev)
1879 {
1880         struct e1000_hw *hw =
1881                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1882         uint32_t rctl;
1883
1884         rctl = E1000_READ_REG(hw, E1000_RCTL);
1885         rctl |= E1000_RCTL_MPE;
1886         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1887 }
1888
1889 static void
1890 eth_igb_allmulticast_disable(struct rte_eth_dev *dev)
1891 {
1892         struct e1000_hw *hw =
1893                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1894         uint32_t rctl;
1895
1896         if (dev->data->promiscuous == 1)
1897                 return; /* must remain in all_multicast mode */
1898         rctl = E1000_READ_REG(hw, E1000_RCTL);
1899         rctl &= (~E1000_RCTL_MPE);
1900         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1901 }
1902
1903 static int
1904 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1905 {
1906         struct e1000_hw *hw =
1907                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1908         struct e1000_vfta * shadow_vfta =
1909                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1910         uint32_t vfta;
1911         uint32_t vid_idx;
1912         uint32_t vid_bit;
1913
1914         vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) &
1915                               E1000_VFTA_ENTRY_MASK);
1916         vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK));
1917         vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx);
1918         if (on)
1919                 vfta |= vid_bit;
1920         else
1921                 vfta &= ~vid_bit;
1922         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta);
1923
1924         /* update local VFTA copy */
1925         shadow_vfta->vfta[vid_idx] = vfta;
1926
1927         return 0;
1928 }
1929
1930 static void
1931 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1932 {
1933         struct e1000_hw *hw =
1934                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1935         uint32_t reg = ETHER_TYPE_VLAN ;
1936
1937         reg |= (tpid << 16);
1938         E1000_WRITE_REG(hw, E1000_VET, reg);
1939 }
1940
1941 static void
1942 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1943 {
1944         struct e1000_hw *hw =
1945                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1946         uint32_t reg;
1947
1948         /* Filter Table Disable */
1949         reg = E1000_READ_REG(hw, E1000_RCTL);
1950         reg &= ~E1000_RCTL_CFIEN;
1951         reg &= ~E1000_RCTL_VFE;
1952         E1000_WRITE_REG(hw, E1000_RCTL, reg);
1953 }
1954
1955 static void
1956 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1957 {
1958         struct e1000_hw *hw =
1959                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1960         struct e1000_vfta * shadow_vfta =
1961                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1962         uint32_t reg;
1963         int i;
1964
1965         /* Filter Table Enable, CFI not used for packet acceptance */
1966         reg = E1000_READ_REG(hw, E1000_RCTL);
1967         reg &= ~E1000_RCTL_CFIEN;
1968         reg |= E1000_RCTL_VFE;
1969         E1000_WRITE_REG(hw, E1000_RCTL, reg);
1970
1971         /* restore VFTA table */
1972         for (i = 0; i < IGB_VFTA_SIZE; i++)
1973                 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]);
1974 }
1975
1976 static void
1977 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev)
1978 {
1979         struct e1000_hw *hw =
1980                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1981         uint32_t reg;
1982
1983         /* VLAN Mode Disable */
1984         reg = E1000_READ_REG(hw, E1000_CTRL);
1985         reg &= ~E1000_CTRL_VME;
1986         E1000_WRITE_REG(hw, E1000_CTRL, reg);
1987 }
1988
1989 static void
1990 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev)
1991 {
1992         struct e1000_hw *hw =
1993                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1994         uint32_t reg;
1995
1996         /* VLAN Mode Enable */
1997         reg = E1000_READ_REG(hw, E1000_CTRL);
1998         reg |= E1000_CTRL_VME;
1999         E1000_WRITE_REG(hw, E1000_CTRL, reg);
2000 }
2001
2002 static void
2003 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev)
2004 {
2005         struct e1000_hw *hw =
2006                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2007         uint32_t reg;
2008
2009         /* CTRL_EXT: Extended VLAN */
2010         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
2011         reg &= ~E1000_CTRL_EXT_EXTEND_VLAN;
2012         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
2013
2014         /* Update maximum packet length */
2015         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
2016                 E1000_WRITE_REG(hw, E1000_RLPML,
2017                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
2018                                                 VLAN_TAG_SIZE);
2019 }
2020
2021 static void
2022 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev)
2023 {
2024         struct e1000_hw *hw =
2025                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2026         uint32_t reg;
2027
2028         /* CTRL_EXT: Extended VLAN */
2029         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
2030         reg |= E1000_CTRL_EXT_EXTEND_VLAN;
2031         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
2032
2033         /* Update maximum packet length */
2034         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
2035                 E1000_WRITE_REG(hw, E1000_RLPML,
2036                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
2037                                                 2 * VLAN_TAG_SIZE);
2038 }
2039
2040 static void
2041 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2042 {
2043         if(mask & ETH_VLAN_STRIP_MASK){
2044                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
2045                         igb_vlan_hw_strip_enable(dev);
2046                 else
2047                         igb_vlan_hw_strip_disable(dev);
2048         }
2049
2050         if(mask & ETH_VLAN_FILTER_MASK){
2051                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
2052                         igb_vlan_hw_filter_enable(dev);
2053                 else
2054                         igb_vlan_hw_filter_disable(dev);
2055         }
2056
2057         if(mask & ETH_VLAN_EXTEND_MASK){
2058                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
2059                         igb_vlan_hw_extend_enable(dev);
2060                 else
2061                         igb_vlan_hw_extend_disable(dev);
2062         }
2063 }
2064
2065
2066 /**
2067  * It enables the interrupt mask and then enable the interrupt.
2068  *
2069  * @param dev
2070  *  Pointer to struct rte_eth_dev.
2071  *
2072  * @return
2073  *  - On success, zero.
2074  *  - On failure, a negative value.
2075  */
2076 static int
2077 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev)
2078 {
2079         struct e1000_interrupt *intr =
2080                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2081
2082         intr->mask |= E1000_ICR_LSC;
2083
2084         return 0;
2085 }
2086
2087 /* It clears the interrupt causes and enables the interrupt.
2088  * It will be called once only during nic initialized.
2089  *
2090  * @param dev
2091  *  Pointer to struct rte_eth_dev.
2092  *
2093  * @return
2094  *  - On success, zero.
2095  *  - On failure, a negative value.
2096  */
2097 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev)
2098 {
2099         uint32_t mask, regval;
2100         struct e1000_hw *hw =
2101                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2102         struct rte_eth_dev_info dev_info;
2103
2104         memset(&dev_info, 0, sizeof(dev_info));
2105         eth_igb_infos_get(dev, &dev_info);
2106
2107         mask = 0xFFFFFFFF >> (32 - dev_info.max_rx_queues);
2108         regval = E1000_READ_REG(hw, E1000_EIMS);
2109         E1000_WRITE_REG(hw, E1000_EIMS, regval | mask);
2110
2111         return 0;
2112 }
2113
2114 /*
2115  * It reads ICR and gets interrupt causes, check it and set a bit flag
2116  * to update link status.
2117  *
2118  * @param dev
2119  *  Pointer to struct rte_eth_dev.
2120  *
2121  * @return
2122  *  - On success, zero.
2123  *  - On failure, a negative value.
2124  */
2125 static int
2126 eth_igb_interrupt_get_status(struct rte_eth_dev *dev)
2127 {
2128         uint32_t icr;
2129         struct e1000_hw *hw =
2130                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2131         struct e1000_interrupt *intr =
2132                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2133
2134         igb_intr_disable(hw);
2135
2136         /* read-on-clear nic registers here */
2137         icr = E1000_READ_REG(hw, E1000_ICR);
2138
2139         intr->flags = 0;
2140         if (icr & E1000_ICR_LSC) {
2141                 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
2142         }
2143
2144         if (icr & E1000_ICR_VMMB)
2145                 intr->flags |= E1000_FLAG_MAILBOX;
2146
2147         return 0;
2148 }
2149
2150 /*
2151  * It executes link_update after knowing an interrupt is prsent.
2152  *
2153  * @param dev
2154  *  Pointer to struct rte_eth_dev.
2155  *
2156  * @return
2157  *  - On success, zero.
2158  *  - On failure, a negative value.
2159  */
2160 static int
2161 eth_igb_interrupt_action(struct rte_eth_dev *dev)
2162 {
2163         struct e1000_hw *hw =
2164                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2165         struct e1000_interrupt *intr =
2166                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2167         uint32_t tctl, rctl;
2168         struct rte_eth_link link;
2169         int ret;
2170
2171         if (intr->flags & E1000_FLAG_MAILBOX) {
2172                 igb_pf_mbx_process(dev);
2173                 intr->flags &= ~E1000_FLAG_MAILBOX;
2174         }
2175
2176         igb_intr_enable(dev);
2177         rte_intr_enable(&(dev->pci_dev->intr_handle));
2178
2179         if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) {
2180                 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE;
2181
2182                 /* set get_link_status to check register later */
2183                 hw->mac.get_link_status = 1;
2184                 ret = eth_igb_link_update(dev, 0);
2185
2186                 /* check if link has changed */
2187                 if (ret < 0)
2188                         return 0;
2189
2190                 memset(&link, 0, sizeof(link));
2191                 rte_igb_dev_atomic_read_link_status(dev, &link);
2192                 if (link.link_status) {
2193                         PMD_INIT_LOG(INFO,
2194                                      " Port %d: Link Up - speed %u Mbps - %s",
2195                                      dev->data->port_id,
2196                                      (unsigned)link.link_speed,
2197                                      link.link_duplex == ETH_LINK_FULL_DUPLEX ?
2198                                      "full-duplex" : "half-duplex");
2199                 } else {
2200                         PMD_INIT_LOG(INFO, " Port %d: Link Down",
2201                                      dev->data->port_id);
2202                 }
2203
2204                 PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d",
2205                              dev->pci_dev->addr.domain,
2206                              dev->pci_dev->addr.bus,
2207                              dev->pci_dev->addr.devid,
2208                              dev->pci_dev->addr.function);
2209                 tctl = E1000_READ_REG(hw, E1000_TCTL);
2210                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2211                 if (link.link_status) {
2212                         /* enable Tx/Rx */
2213                         tctl |= E1000_TCTL_EN;
2214                         rctl |= E1000_RCTL_EN;
2215                 } else {
2216                         /* disable Tx/Rx */
2217                         tctl &= ~E1000_TCTL_EN;
2218                         rctl &= ~E1000_RCTL_EN;
2219                 }
2220                 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
2221                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2222                 E1000_WRITE_FLUSH(hw);
2223                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
2224         }
2225
2226         return 0;
2227 }
2228
2229 /**
2230  * Interrupt handler which shall be registered at first.
2231  *
2232  * @param handle
2233  *  Pointer to interrupt handle.
2234  * @param param
2235  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2236  *
2237  * @return
2238  *  void
2239  */
2240 static void
2241 eth_igb_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
2242                                                         void *param)
2243 {
2244         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2245
2246         eth_igb_interrupt_get_status(dev);
2247         eth_igb_interrupt_action(dev);
2248 }
2249
2250 static int
2251 eth_igb_led_on(struct rte_eth_dev *dev)
2252 {
2253         struct e1000_hw *hw;
2254
2255         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2256         return (e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
2257 }
2258
2259 static int
2260 eth_igb_led_off(struct rte_eth_dev *dev)
2261 {
2262         struct e1000_hw *hw;
2263
2264         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2265         return (e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
2266 }
2267
2268 static int
2269 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2270 {
2271         struct e1000_hw *hw;
2272         uint32_t ctrl;
2273         int tx_pause;
2274         int rx_pause;
2275
2276         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2277         fc_conf->pause_time = hw->fc.pause_time;
2278         fc_conf->high_water = hw->fc.high_water;
2279         fc_conf->low_water = hw->fc.low_water;
2280         fc_conf->send_xon = hw->fc.send_xon;
2281         fc_conf->autoneg = hw->mac.autoneg;
2282
2283         /*
2284          * Return rx_pause and tx_pause status according to actual setting of
2285          * the TFCE and RFCE bits in the CTRL register.
2286          */
2287         ctrl = E1000_READ_REG(hw, E1000_CTRL);
2288         if (ctrl & E1000_CTRL_TFCE)
2289                 tx_pause = 1;
2290         else
2291                 tx_pause = 0;
2292
2293         if (ctrl & E1000_CTRL_RFCE)
2294                 rx_pause = 1;
2295         else
2296                 rx_pause = 0;
2297
2298         if (rx_pause && tx_pause)
2299                 fc_conf->mode = RTE_FC_FULL;
2300         else if (rx_pause)
2301                 fc_conf->mode = RTE_FC_RX_PAUSE;
2302         else if (tx_pause)
2303                 fc_conf->mode = RTE_FC_TX_PAUSE;
2304         else
2305                 fc_conf->mode = RTE_FC_NONE;
2306
2307         return 0;
2308 }
2309
2310 static int
2311 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2312 {
2313         struct e1000_hw *hw;
2314         int err;
2315         enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = {
2316                 e1000_fc_none,
2317                 e1000_fc_rx_pause,
2318                 e1000_fc_tx_pause,
2319                 e1000_fc_full
2320         };
2321         uint32_t rx_buf_size;
2322         uint32_t max_high_water;
2323         uint32_t rctl;
2324
2325         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2326         if (fc_conf->autoneg != hw->mac.autoneg)
2327                 return -ENOTSUP;
2328         rx_buf_size = igb_get_rx_buffer_size(hw);
2329         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2330
2331         /* At least reserve one Ethernet frame for watermark */
2332         max_high_water = rx_buf_size - ETHER_MAX_LEN;
2333         if ((fc_conf->high_water > max_high_water) ||
2334             (fc_conf->high_water < fc_conf->low_water)) {
2335                 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value");
2336                 PMD_INIT_LOG(ERR, "high water must <=  0x%x", max_high_water);
2337                 return (-EINVAL);
2338         }
2339
2340         hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode];
2341         hw->fc.pause_time     = fc_conf->pause_time;
2342         hw->fc.high_water     = fc_conf->high_water;
2343         hw->fc.low_water      = fc_conf->low_water;
2344         hw->fc.send_xon       = fc_conf->send_xon;
2345
2346         err = e1000_setup_link_generic(hw);
2347         if (err == E1000_SUCCESS) {
2348
2349                 /* check if we want to forward MAC frames - driver doesn't have native
2350                  * capability to do that, so we'll write the registers ourselves */
2351
2352                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2353
2354                 /* set or clear MFLCN.PMCF bit depending on configuration */
2355                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2356                         rctl |= E1000_RCTL_PMCF;
2357                 else
2358                         rctl &= ~E1000_RCTL_PMCF;
2359
2360                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2361                 E1000_WRITE_FLUSH(hw);
2362
2363                 return 0;
2364         }
2365
2366         PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err);
2367         return (-EIO);
2368 }
2369
2370 #define E1000_RAH_POOLSEL_SHIFT      (18)
2371 static void
2372 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2373                 uint32_t index, __rte_unused uint32_t pool)
2374 {
2375         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2376         uint32_t rah;
2377
2378         e1000_rar_set(hw, mac_addr->addr_bytes, index);
2379         rah = E1000_READ_REG(hw, E1000_RAH(index));
2380         rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool));
2381         E1000_WRITE_REG(hw, E1000_RAH(index), rah);
2382 }
2383
2384 static void
2385 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index)
2386 {
2387         uint8_t addr[ETHER_ADDR_LEN];
2388         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2389
2390         memset(addr, 0, sizeof(addr));
2391
2392         e1000_rar_set(hw, addr, index);
2393 }
2394
2395 static void
2396 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev,
2397                                 struct ether_addr *addr)
2398 {
2399         eth_igb_rar_clear(dev, 0);
2400
2401         eth_igb_rar_set(dev, (void *)addr, 0, 0);
2402 }
2403 /*
2404  * Virtual Function operations
2405  */
2406 static void
2407 igbvf_intr_disable(struct e1000_hw *hw)
2408 {
2409         PMD_INIT_FUNC_TRACE();
2410
2411         /* Clear interrupt mask to stop from interrupts being generated */
2412         E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF);
2413
2414         E1000_WRITE_FLUSH(hw);
2415 }
2416
2417 static void
2418 igbvf_stop_adapter(struct rte_eth_dev *dev)
2419 {
2420         u32 reg_val;
2421         u16 i;
2422         struct rte_eth_dev_info dev_info;
2423         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2424
2425         memset(&dev_info, 0, sizeof(dev_info));
2426         eth_igbvf_infos_get(dev, &dev_info);
2427
2428         /* Clear interrupt mask to stop from interrupts being generated */
2429         igbvf_intr_disable(hw);
2430
2431         /* Clear any pending interrupts, flush previous writes */
2432         E1000_READ_REG(hw, E1000_EICR);
2433
2434         /* Disable the transmit unit.  Each queue must be disabled. */
2435         for (i = 0; i < dev_info.max_tx_queues; i++)
2436                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH);
2437
2438         /* Disable the receive unit by stopping each queue */
2439         for (i = 0; i < dev_info.max_rx_queues; i++) {
2440                 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i));
2441                 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE;
2442                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val);
2443                 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE)
2444                         ;
2445         }
2446
2447         /* flush all queues disables */
2448         E1000_WRITE_FLUSH(hw);
2449         msec_delay(2);
2450 }
2451
2452 static int eth_igbvf_link_update(struct e1000_hw *hw)
2453 {
2454         struct e1000_mbx_info *mbx = &hw->mbx;
2455         struct e1000_mac_info *mac = &hw->mac;
2456         int ret_val = E1000_SUCCESS;
2457
2458         PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf");
2459
2460         /*
2461          * We only want to run this if there has been a rst asserted.
2462          * in this case that could mean a link change, device reset,
2463          * or a virtual function reset
2464          */
2465
2466         /* If we were hit with a reset or timeout drop the link */
2467         if (!e1000_check_for_rst(hw, 0) || !mbx->timeout)
2468                 mac->get_link_status = TRUE;
2469
2470         if (!mac->get_link_status)
2471                 goto out;
2472
2473         /* if link status is down no point in checking to see if pf is up */
2474         if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU))
2475                 goto out;
2476
2477         /* if we passed all the tests above then the link is up and we no
2478          * longer need to check for link */
2479         mac->get_link_status = FALSE;
2480
2481 out:
2482         return ret_val;
2483 }
2484
2485
2486 static int
2487 igbvf_dev_configure(struct rte_eth_dev *dev)
2488 {
2489         struct rte_eth_conf* conf = &dev->data->dev_conf;
2490
2491         PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
2492                      dev->data->port_id);
2493
2494         /*
2495          * VF has no ability to enable/disable HW CRC
2496          * Keep the persistent behavior the same as Host PF
2497          */
2498 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC
2499         if (!conf->rxmode.hw_strip_crc) {
2500                 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip");
2501                 conf->rxmode.hw_strip_crc = 1;
2502         }
2503 #else
2504         if (conf->rxmode.hw_strip_crc) {
2505                 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip");
2506                 conf->rxmode.hw_strip_crc = 0;
2507         }
2508 #endif
2509
2510         return 0;
2511 }
2512
2513 static int
2514 igbvf_dev_start(struct rte_eth_dev *dev)
2515 {
2516         struct e1000_hw *hw =
2517                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2518         struct e1000_adapter *adapter =
2519                 E1000_DEV_PRIVATE(dev->data->dev_private);
2520         int ret;
2521
2522         PMD_INIT_FUNC_TRACE();
2523
2524         hw->mac.ops.reset_hw(hw);
2525         adapter->stopped = 0;
2526
2527         /* Set all vfta */
2528         igbvf_set_vfta_all(dev,1);
2529
2530         eth_igbvf_tx_init(dev);
2531
2532         /* This can fail when allocating mbufs for descriptor rings */
2533         ret = eth_igbvf_rx_init(dev);
2534         if (ret) {
2535                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
2536                 igb_dev_clear_queues(dev);
2537                 return ret;
2538         }
2539
2540         return 0;
2541 }
2542
2543 static void
2544 igbvf_dev_stop(struct rte_eth_dev *dev)
2545 {
2546         PMD_INIT_FUNC_TRACE();
2547
2548         igbvf_stop_adapter(dev);
2549
2550         /*
2551           * Clear what we set, but we still keep shadow_vfta to
2552           * restore after device starts
2553           */
2554         igbvf_set_vfta_all(dev,0);
2555
2556         igb_dev_clear_queues(dev);
2557 }
2558
2559 static void
2560 igbvf_dev_close(struct rte_eth_dev *dev)
2561 {
2562         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2563         struct e1000_adapter *adapter =
2564                 E1000_DEV_PRIVATE(dev->data->dev_private);
2565
2566         PMD_INIT_FUNC_TRACE();
2567
2568         e1000_reset_hw(hw);
2569
2570         igbvf_dev_stop(dev);
2571         adapter->stopped = 1;
2572         igb_dev_free_queues(dev);
2573 }
2574
2575 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on)
2576 {
2577         struct e1000_mbx_info *mbx = &hw->mbx;
2578         uint32_t msgbuf[2];
2579
2580         /* After set vlan, vlan strip will also be enabled in igb driver*/
2581         msgbuf[0] = E1000_VF_SET_VLAN;
2582         msgbuf[1] = vid;
2583         /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */
2584         if (on)
2585                 msgbuf[0] |= E1000_VF_SET_VLAN_ADD;
2586
2587         return (mbx->ops.write_posted(hw, msgbuf, 2, 0));
2588 }
2589
2590 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2591 {
2592         struct e1000_hw *hw =
2593                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2594         struct e1000_vfta * shadow_vfta =
2595                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2596         int i = 0, j = 0, vfta = 0, mask = 1;
2597
2598         for (i = 0; i < IGB_VFTA_SIZE; i++){
2599                 vfta = shadow_vfta->vfta[i];
2600                 if(vfta){
2601                         mask = 1;
2602                         for (j = 0; j < 32; j++){
2603                                 if(vfta & mask)
2604                                         igbvf_set_vfta(hw,
2605                                                 (uint16_t)((i<<5)+j), on);
2606                                 mask<<=1;
2607                         }
2608                 }
2609         }
2610
2611 }
2612
2613 static int
2614 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2615 {
2616         struct e1000_hw *hw =
2617                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2618         struct e1000_vfta * shadow_vfta =
2619                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2620         uint32_t vid_idx = 0;
2621         uint32_t vid_bit = 0;
2622         int ret = 0;
2623
2624         PMD_INIT_FUNC_TRACE();
2625
2626         /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/
2627         ret = igbvf_set_vfta(hw, vlan_id, !!on);
2628         if(ret){
2629                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2630                 return ret;
2631         }
2632         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2633         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2634
2635         /*Save what we set and retore it after device reset*/
2636         if (on)
2637                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2638         else
2639                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2640
2641         return 0;
2642 }
2643
2644 static void
2645 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr)
2646 {
2647         struct e1000_hw *hw =
2648                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2649
2650         /* index is not used by rar_set() */
2651         hw->mac.ops.rar_set(hw, (void *)addr, 0);
2652 }
2653
2654
2655 static int
2656 eth_igb_rss_reta_update(struct rte_eth_dev *dev,
2657                         struct rte_eth_rss_reta_entry64 *reta_conf,
2658                         uint16_t reta_size)
2659 {
2660         uint8_t i, j, mask;
2661         uint32_t reta, r;
2662         uint16_t idx, shift;
2663         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2664
2665         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2666                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2667                         "(%d) doesn't match the number hardware can supported "
2668                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2669                 return -EINVAL;
2670         }
2671
2672         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
2673                 idx = i / RTE_RETA_GROUP_SIZE;
2674                 shift = i % RTE_RETA_GROUP_SIZE;
2675                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2676                                                 IGB_4_BIT_MASK);
2677                 if (!mask)
2678                         continue;
2679                 if (mask == IGB_4_BIT_MASK)
2680                         r = 0;
2681                 else
2682                         r = E1000_READ_REG(hw, E1000_RETA(i >> 2));
2683                 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) {
2684                         if (mask & (0x1 << j))
2685                                 reta |= reta_conf[idx].reta[shift + j] <<
2686                                                         (CHAR_BIT * j);
2687                         else
2688                                 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j));
2689                 }
2690                 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta);
2691         }
2692
2693         return 0;
2694 }
2695
2696 static int
2697 eth_igb_rss_reta_query(struct rte_eth_dev *dev,
2698                        struct rte_eth_rss_reta_entry64 *reta_conf,
2699                        uint16_t reta_size)
2700 {
2701         uint8_t i, j, mask;
2702         uint32_t reta;
2703         uint16_t idx, shift;
2704         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2705
2706         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2707                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2708                         "(%d) doesn't match the number hardware can supported "
2709                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2710                 return -EINVAL;
2711         }
2712
2713         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
2714                 idx = i / RTE_RETA_GROUP_SIZE;
2715                 shift = i % RTE_RETA_GROUP_SIZE;
2716                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2717                                                 IGB_4_BIT_MASK);
2718                 if (!mask)
2719                         continue;
2720                 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2));
2721                 for (j = 0; j < IGB_4_BIT_WIDTH; j++) {
2722                         if (mask & (0x1 << j))
2723                                 reta_conf[idx].reta[shift + j] =
2724                                         ((reta >> (CHAR_BIT * j)) &
2725                                                 IGB_8_BIT_MASK);
2726                 }
2727         }
2728
2729         return 0;
2730 }
2731
2732 #define MAC_TYPE_FILTER_SUP(type)    do {\
2733         if ((type) != e1000_82580 && (type) != e1000_i350 &&\
2734                 (type) != e1000_82576)\
2735                 return -ENOTSUP;\
2736 } while (0)
2737
2738 static int
2739 eth_igb_syn_filter_set(struct rte_eth_dev *dev,
2740                         struct rte_eth_syn_filter *filter,
2741                         bool add)
2742 {
2743         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2744         uint32_t synqf, rfctl;
2745
2746         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
2747                 return -EINVAL;
2748
2749         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
2750
2751         if (add) {
2752                 if (synqf & E1000_SYN_FILTER_ENABLE)
2753                         return -EINVAL;
2754
2755                 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) &
2756                         E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE);
2757
2758                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
2759                 if (filter->hig_pri)
2760                         rfctl |= E1000_RFCTL_SYNQFP;
2761                 else
2762                         rfctl &= ~E1000_RFCTL_SYNQFP;
2763
2764                 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl);
2765         } else {
2766                 if (!(synqf & E1000_SYN_FILTER_ENABLE))
2767                         return -ENOENT;
2768                 synqf = 0;
2769         }
2770
2771         E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf);
2772         E1000_WRITE_FLUSH(hw);
2773         return 0;
2774 }
2775
2776 static int
2777 eth_igb_syn_filter_get(struct rte_eth_dev *dev,
2778                         struct rte_eth_syn_filter *filter)
2779 {
2780         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2781         uint32_t synqf, rfctl;
2782
2783         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
2784         if (synqf & E1000_SYN_FILTER_ENABLE) {
2785                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
2786                 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0;
2787                 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >>
2788                                 E1000_SYN_FILTER_QUEUE_SHIFT);
2789                 return 0;
2790         }
2791
2792         return -ENOENT;
2793 }
2794
2795 static int
2796 eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
2797                         enum rte_filter_op filter_op,
2798                         void *arg)
2799 {
2800         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2801         int ret;
2802
2803         MAC_TYPE_FILTER_SUP(hw->mac.type);
2804
2805         if (filter_op == RTE_ETH_FILTER_NOP)
2806                 return 0;
2807
2808         if (arg == NULL) {
2809                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
2810                             filter_op);
2811                 return -EINVAL;
2812         }
2813
2814         switch (filter_op) {
2815         case RTE_ETH_FILTER_ADD:
2816                 ret = eth_igb_syn_filter_set(dev,
2817                                 (struct rte_eth_syn_filter *)arg,
2818                                 TRUE);
2819                 break;
2820         case RTE_ETH_FILTER_DELETE:
2821                 ret = eth_igb_syn_filter_set(dev,
2822                                 (struct rte_eth_syn_filter *)arg,
2823                                 FALSE);
2824                 break;
2825         case RTE_ETH_FILTER_GET:
2826                 ret = eth_igb_syn_filter_get(dev,
2827                                 (struct rte_eth_syn_filter *)arg);
2828                 break;
2829         default:
2830                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
2831                 ret = -EINVAL;
2832                 break;
2833         }
2834
2835         return ret;
2836 }
2837
2838 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
2839         if ((type) != e1000_82580 && (type) != e1000_i350)\
2840                 return -ENOSYS; \
2841 } while (0)
2842
2843 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/
2844 static inline int
2845 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter,
2846                         struct e1000_2tuple_filter_info *filter_info)
2847 {
2848         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
2849                 return -EINVAL;
2850         if (filter->priority > E1000_2TUPLE_MAX_PRI)
2851                 return -EINVAL;  /* filter index is out of range. */
2852         if (filter->tcp_flags > TCP_FLAG_ALL)
2853                 return -EINVAL;  /* flags is invalid. */
2854
2855         switch (filter->dst_port_mask) {
2856         case UINT16_MAX:
2857                 filter_info->dst_port_mask = 0;
2858                 filter_info->dst_port = filter->dst_port;
2859                 break;
2860         case 0:
2861                 filter_info->dst_port_mask = 1;
2862                 break;
2863         default:
2864                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
2865                 return -EINVAL;
2866         }
2867
2868         switch (filter->proto_mask) {
2869         case UINT8_MAX:
2870                 filter_info->proto_mask = 0;
2871                 filter_info->proto = filter->proto;
2872                 break;
2873         case 0:
2874                 filter_info->proto_mask = 1;
2875                 break;
2876         default:
2877                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
2878                 return -EINVAL;
2879         }
2880
2881         filter_info->priority = (uint8_t)filter->priority;
2882         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
2883                 filter_info->tcp_flags = filter->tcp_flags;
2884         else
2885                 filter_info->tcp_flags = 0;
2886
2887         return 0;
2888 }
2889
2890 static inline struct e1000_2tuple_filter *
2891 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list,
2892                         struct e1000_2tuple_filter_info *key)
2893 {
2894         struct e1000_2tuple_filter *it;
2895
2896         TAILQ_FOREACH(it, filter_list, entries) {
2897                 if (memcmp(key, &it->filter_info,
2898                         sizeof(struct e1000_2tuple_filter_info)) == 0) {
2899                         return it;
2900                 }
2901         }
2902         return NULL;
2903 }
2904
2905 /*
2906  * igb_add_2tuple_filter - add a 2tuple filter
2907  *
2908  * @param
2909  * dev: Pointer to struct rte_eth_dev.
2910  * ntuple_filter: ponter to the filter that will be added.
2911  *
2912  * @return
2913  *    - On success, zero.
2914  *    - On failure, a negative value.
2915  */
2916 static int
2917 igb_add_2tuple_filter(struct rte_eth_dev *dev,
2918                         struct rte_eth_ntuple_filter *ntuple_filter)
2919 {
2920         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2921         struct e1000_filter_info *filter_info =
2922                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
2923         struct e1000_2tuple_filter *filter;
2924         uint32_t ttqf = E1000_TTQF_DISABLE_MASK;
2925         uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP;
2926         int i, ret;
2927
2928         filter = rte_zmalloc("e1000_2tuple_filter",
2929                         sizeof(struct e1000_2tuple_filter), 0);
2930         if (filter == NULL)
2931                 return -ENOMEM;
2932
2933         ret = ntuple_filter_to_2tuple(ntuple_filter,
2934                                       &filter->filter_info);
2935         if (ret < 0) {
2936                 rte_free(filter);
2937                 return ret;
2938         }
2939         if (igb_2tuple_filter_lookup(&filter_info->twotuple_list,
2940                                          &filter->filter_info) != NULL) {
2941                 PMD_DRV_LOG(ERR, "filter exists.");
2942                 rte_free(filter);
2943                 return -EEXIST;
2944         }
2945         filter->queue = ntuple_filter->queue;
2946
2947         /*
2948          * look for an unused 2tuple filter index,
2949          * and insert the filter to list.
2950          */
2951         for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) {
2952                 if (!(filter_info->twotuple_mask & (1 << i))) {
2953                         filter_info->twotuple_mask |= 1 << i;
2954                         filter->index = i;
2955                         TAILQ_INSERT_TAIL(&filter_info->twotuple_list,
2956                                           filter,
2957                                           entries);
2958                         break;
2959                 }
2960         }
2961         if (i >= E1000_MAX_TTQF_FILTERS) {
2962                 PMD_DRV_LOG(ERR, "2tuple filters are full.");
2963                 rte_free(filter);
2964                 return -ENOSYS;
2965         }
2966
2967         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
2968         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
2969                 imir |= E1000_IMIR_PORT_BP;
2970         else
2971                 imir &= ~E1000_IMIR_PORT_BP;
2972
2973         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
2974
2975         ttqf |= E1000_TTQF_QUEUE_ENABLE;
2976         ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT);
2977         ttqf |= (uint32_t)(filter->filter_info.proto & E1000_TTQF_PROTOCOL_MASK);
2978         if (filter->filter_info.proto_mask == 0)
2979                 ttqf &= ~E1000_TTQF_MASK_ENABLE;
2980
2981         /* tcp flags bits setting. */
2982         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
2983                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
2984                         imir_ext |= E1000_IMIREXT_CTRL_URG;
2985                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
2986                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
2987                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
2988                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
2989                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
2990                         imir_ext |= E1000_IMIREXT_CTRL_RST;
2991                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
2992                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
2993                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
2994                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
2995         } else
2996                 imir_ext |= E1000_IMIREXT_CTRL_BP;
2997         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
2998         E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf);
2999         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
3000         return 0;
3001 }
3002
3003 /*
3004  * igb_remove_2tuple_filter - remove a 2tuple filter
3005  *
3006  * @param
3007  * dev: Pointer to struct rte_eth_dev.
3008  * ntuple_filter: ponter to the filter that will be removed.
3009  *
3010  * @return
3011  *    - On success, zero.
3012  *    - On failure, a negative value.
3013  */
3014 static int
3015 igb_remove_2tuple_filter(struct rte_eth_dev *dev,
3016                         struct rte_eth_ntuple_filter *ntuple_filter)
3017 {
3018         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3019         struct e1000_filter_info *filter_info =
3020                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3021         struct e1000_2tuple_filter_info filter_2tuple;
3022         struct e1000_2tuple_filter *filter;
3023         int ret;
3024
3025         memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info));
3026         ret = ntuple_filter_to_2tuple(ntuple_filter,
3027                                       &filter_2tuple);
3028         if (ret < 0)
3029                 return ret;
3030
3031         filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list,
3032                                          &filter_2tuple);
3033         if (filter == NULL) {
3034                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3035                 return -ENOENT;
3036         }
3037
3038         filter_info->twotuple_mask &= ~(1 << filter->index);
3039         TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries);
3040         rte_free(filter);
3041
3042         E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK);
3043         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
3044         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
3045         return 0;
3046 }
3047
3048 static inline struct e1000_flex_filter *
3049 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list,
3050                         struct e1000_flex_filter_info *key)
3051 {
3052         struct e1000_flex_filter *it;
3053
3054         TAILQ_FOREACH(it, filter_list, entries) {
3055                 if (memcmp(key, &it->filter_info,
3056                         sizeof(struct e1000_flex_filter_info)) == 0)
3057                         return it;
3058         }
3059
3060         return NULL;
3061 }
3062
3063 static int
3064 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
3065                         struct rte_eth_flex_filter *filter,
3066                         bool add)
3067 {
3068         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3069         struct e1000_filter_info *filter_info =
3070                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3071         struct e1000_flex_filter *flex_filter, *it;
3072         uint32_t wufc, queueing, mask;
3073         uint32_t reg_off;
3074         uint8_t shift, i, j = 0;
3075
3076         flex_filter = rte_zmalloc("e1000_flex_filter",
3077                         sizeof(struct e1000_flex_filter), 0);
3078         if (flex_filter == NULL)
3079                 return -ENOMEM;
3080
3081         flex_filter->filter_info.len = filter->len;
3082         flex_filter->filter_info.priority = filter->priority;
3083         memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len);
3084         for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) {
3085                 mask = 0;
3086                 /* reverse bits in flex filter's mask*/
3087                 for (shift = 0; shift < CHAR_BIT; shift++) {
3088                         if (filter->mask[i] & (0x01 << shift))
3089                                 mask |= (0x80 >> shift);
3090                 }
3091                 flex_filter->filter_info.mask[i] = mask;
3092         }
3093
3094         wufc = E1000_READ_REG(hw, E1000_WUFC);
3095         if (flex_filter->index < E1000_MAX_FHFT)
3096                 reg_off = E1000_FHFT(flex_filter->index);
3097         else
3098                 reg_off = E1000_FHFT_EXT(flex_filter->index - E1000_MAX_FHFT);
3099
3100         if (add) {
3101                 if (eth_igb_flex_filter_lookup(&filter_info->flex_list,
3102                                 &flex_filter->filter_info) != NULL) {
3103                         PMD_DRV_LOG(ERR, "filter exists.");
3104                         rte_free(flex_filter);
3105                         return -EEXIST;
3106                 }
3107                 flex_filter->queue = filter->queue;
3108                 /*
3109                  * look for an unused flex filter index
3110                  * and insert the filter into the list.
3111                  */
3112                 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) {
3113                         if (!(filter_info->flex_mask & (1 << i))) {
3114                                 filter_info->flex_mask |= 1 << i;
3115                                 flex_filter->index = i;
3116                                 TAILQ_INSERT_TAIL(&filter_info->flex_list,
3117                                         flex_filter,
3118                                         entries);
3119                                 break;
3120                         }
3121                 }
3122                 if (i >= E1000_MAX_FLEX_FILTERS) {
3123                         PMD_DRV_LOG(ERR, "flex filters are full.");
3124                         rte_free(flex_filter);
3125                         return -ENOSYS;
3126                 }
3127
3128                 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ |
3129                                 (E1000_WUFC_FLX0 << flex_filter->index));
3130                 queueing = filter->len |
3131                         (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) |
3132                         (filter->priority << E1000_FHFT_QUEUEING_PRIO_SHIFT);
3133                 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET,
3134                                 queueing);
3135                 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) {
3136                         E1000_WRITE_REG(hw, reg_off,
3137                                         flex_filter->filter_info.dwords[j]);
3138                         reg_off += sizeof(uint32_t);
3139                         E1000_WRITE_REG(hw, reg_off,
3140                                         flex_filter->filter_info.dwords[++j]);
3141                         reg_off += sizeof(uint32_t);
3142                         E1000_WRITE_REG(hw, reg_off,
3143                                 (uint32_t)flex_filter->filter_info.mask[i]);
3144                         reg_off += sizeof(uint32_t) * 2;
3145                         ++j;
3146                 }
3147         } else {
3148                 it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
3149                                 &flex_filter->filter_info);
3150                 if (it == NULL) {
3151                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3152                         rte_free(flex_filter);
3153                         return -ENOENT;
3154                 }
3155
3156                 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++)
3157                         E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0);
3158                 E1000_WRITE_REG(hw, E1000_WUFC, wufc &
3159                         (~(E1000_WUFC_FLX0 << it->index)));
3160
3161                 filter_info->flex_mask &= ~(1 << it->index);
3162                 TAILQ_REMOVE(&filter_info->flex_list, it, entries);
3163                 rte_free(it);
3164                 rte_free(flex_filter);
3165         }
3166
3167         return 0;
3168 }
3169
3170 static int
3171 eth_igb_get_flex_filter(struct rte_eth_dev *dev,
3172                         struct rte_eth_flex_filter *filter)
3173 {
3174         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3175         struct e1000_filter_info *filter_info =
3176                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3177         struct e1000_flex_filter flex_filter, *it;
3178         uint32_t wufc, queueing, wufc_en = 0;
3179
3180         memset(&flex_filter, 0, sizeof(struct e1000_flex_filter));
3181         flex_filter.filter_info.len = filter->len;
3182         flex_filter.filter_info.priority = filter->priority;
3183         memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len);
3184         memcpy(flex_filter.filter_info.mask, filter->mask,
3185                         RTE_ALIGN(filter->len, sizeof(char)) / sizeof(char));
3186
3187         it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
3188                                 &flex_filter.filter_info);
3189         if (it == NULL) {
3190                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3191                 return -ENOENT;
3192         }
3193
3194         wufc = E1000_READ_REG(hw, E1000_WUFC);
3195         wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index);
3196
3197         if ((wufc & wufc_en) == wufc_en) {
3198                 uint32_t reg_off = 0;
3199                 if (it->index < E1000_MAX_FHFT)
3200                         reg_off = E1000_FHFT(it->index);
3201                 else
3202                         reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT);
3203
3204                 queueing = E1000_READ_REG(hw,
3205                                 reg_off + E1000_FHFT_QUEUEING_OFFSET);
3206                 filter->len = queueing & E1000_FHFT_QUEUEING_LEN;
3207                 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >>
3208                         E1000_FHFT_QUEUEING_PRIO_SHIFT;
3209                 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >>
3210                         E1000_FHFT_QUEUEING_QUEUE_SHIFT;
3211                 return 0;
3212         }
3213         return -ENOENT;
3214 }
3215
3216 static int
3217 eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
3218                         enum rte_filter_op filter_op,
3219                         void *arg)
3220 {
3221         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3222         struct rte_eth_flex_filter *filter;
3223         int ret = 0;
3224
3225         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
3226
3227         if (filter_op == RTE_ETH_FILTER_NOP)
3228                 return ret;
3229
3230         if (arg == NULL) {
3231                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
3232                             filter_op);
3233                 return -EINVAL;
3234         }
3235
3236         filter = (struct rte_eth_flex_filter *)arg;
3237         if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN
3238             || filter->len % sizeof(uint64_t) != 0) {
3239                 PMD_DRV_LOG(ERR, "filter's length is out of range");
3240                 return -EINVAL;
3241         }
3242         if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) {
3243                 PMD_DRV_LOG(ERR, "filter's priority is out of range");
3244                 return -EINVAL;
3245         }
3246
3247         switch (filter_op) {
3248         case RTE_ETH_FILTER_ADD:
3249                 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE);
3250                 break;
3251         case RTE_ETH_FILTER_DELETE:
3252                 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE);
3253                 break;
3254         case RTE_ETH_FILTER_GET:
3255                 ret = eth_igb_get_flex_filter(dev, filter);
3256                 break;
3257         default:
3258                 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op);
3259                 ret = -EINVAL;
3260                 break;
3261         }
3262
3263         return ret;
3264 }
3265
3266 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/
3267 static inline int
3268 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter,
3269                         struct e1000_5tuple_filter_info *filter_info)
3270 {
3271         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576)
3272                 return -EINVAL;
3273         if (filter->priority > E1000_2TUPLE_MAX_PRI)
3274                 return -EINVAL;  /* filter index is out of range. */
3275         if (filter->tcp_flags > TCP_FLAG_ALL)
3276                 return -EINVAL;  /* flags is invalid. */
3277
3278         switch (filter->dst_ip_mask) {
3279         case UINT32_MAX:
3280                 filter_info->dst_ip_mask = 0;
3281                 filter_info->dst_ip = filter->dst_ip;
3282                 break;
3283         case 0:
3284                 filter_info->dst_ip_mask = 1;
3285                 break;
3286         default:
3287                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
3288                 return -EINVAL;
3289         }
3290
3291         switch (filter->src_ip_mask) {
3292         case UINT32_MAX:
3293                 filter_info->src_ip_mask = 0;
3294                 filter_info->src_ip = filter->src_ip;
3295                 break;
3296         case 0:
3297                 filter_info->src_ip_mask = 1;
3298                 break;
3299         default:
3300                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
3301                 return -EINVAL;
3302         }
3303
3304         switch (filter->dst_port_mask) {
3305         case UINT16_MAX:
3306                 filter_info->dst_port_mask = 0;
3307                 filter_info->dst_port = filter->dst_port;
3308                 break;
3309         case 0:
3310                 filter_info->dst_port_mask = 1;
3311                 break;
3312         default:
3313                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
3314                 return -EINVAL;
3315         }
3316
3317         switch (filter->src_port_mask) {
3318         case UINT16_MAX:
3319                 filter_info->src_port_mask = 0;
3320                 filter_info->src_port = filter->src_port;
3321                 break;
3322         case 0:
3323                 filter_info->src_port_mask = 1;
3324                 break;
3325         default:
3326                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
3327                 return -EINVAL;
3328         }
3329
3330         switch (filter->proto_mask) {
3331         case UINT8_MAX:
3332                 filter_info->proto_mask = 0;
3333                 filter_info->proto = filter->proto;
3334                 break;
3335         case 0:
3336                 filter_info->proto_mask = 1;
3337                 break;
3338         default:
3339                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
3340                 return -EINVAL;
3341         }
3342
3343         filter_info->priority = (uint8_t)filter->priority;
3344         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
3345                 filter_info->tcp_flags = filter->tcp_flags;
3346         else
3347                 filter_info->tcp_flags = 0;
3348
3349         return 0;
3350 }
3351
3352 static inline struct e1000_5tuple_filter *
3353 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list,
3354                         struct e1000_5tuple_filter_info *key)
3355 {
3356         struct e1000_5tuple_filter *it;
3357
3358         TAILQ_FOREACH(it, filter_list, entries) {
3359                 if (memcmp(key, &it->filter_info,
3360                         sizeof(struct e1000_5tuple_filter_info)) == 0) {
3361                         return it;
3362                 }
3363         }
3364         return NULL;
3365 }
3366
3367 /*
3368  * igb_add_5tuple_filter_82576 - add a 5tuple filter
3369  *
3370  * @param
3371  * dev: Pointer to struct rte_eth_dev.
3372  * ntuple_filter: ponter to the filter that will be added.
3373  *
3374  * @return
3375  *    - On success, zero.
3376  *    - On failure, a negative value.
3377  */
3378 static int
3379 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
3380                         struct rte_eth_ntuple_filter *ntuple_filter)
3381 {
3382         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3383         struct e1000_filter_info *filter_info =
3384                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3385         struct e1000_5tuple_filter *filter;
3386         uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK;
3387         uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP;
3388         uint8_t i;
3389         int ret;
3390
3391         filter = rte_zmalloc("e1000_5tuple_filter",
3392                         sizeof(struct e1000_5tuple_filter), 0);
3393         if (filter == NULL)
3394                 return -ENOMEM;
3395
3396         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3397                                             &filter->filter_info);
3398         if (ret < 0) {
3399                 rte_free(filter);
3400                 return ret;
3401         }
3402
3403         if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
3404                                          &filter->filter_info) != NULL) {
3405                 PMD_DRV_LOG(ERR, "filter exists.");
3406                 rte_free(filter);
3407                 return -EEXIST;
3408         }
3409         filter->queue = ntuple_filter->queue;
3410
3411         /*
3412          * look for an unused 5tuple filter index,
3413          * and insert the filter to list.
3414          */
3415         for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) {
3416                 if (!(filter_info->fivetuple_mask & (1 << i))) {
3417                         filter_info->fivetuple_mask |= 1 << i;
3418                         filter->index = i;
3419                         TAILQ_INSERT_TAIL(&filter_info->fivetuple_list,
3420                                           filter,
3421                                           entries);
3422                         break;
3423                 }
3424         }
3425         if (i >= E1000_MAX_FTQF_FILTERS) {
3426                 PMD_DRV_LOG(ERR, "5tuple filters are full.");
3427                 rte_free(filter);
3428                 return -ENOSYS;
3429         }
3430
3431         ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK;
3432         if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */
3433                 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP;
3434         if (filter->filter_info.dst_ip_mask == 0)
3435                 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP;
3436         if (filter->filter_info.src_port_mask == 0)
3437                 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
3438         if (filter->filter_info.proto_mask == 0)
3439                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP;
3440         ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) &
3441                 E1000_FTQF_QUEUE_MASK;
3442         ftqf |= E1000_FTQF_QUEUE_ENABLE;
3443         E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf);
3444         E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip);
3445         E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip);
3446
3447         spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT;
3448         E1000_WRITE_REG(hw, E1000_SPQF(i), spqf);
3449
3450         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
3451         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
3452                 imir |= E1000_IMIR_PORT_BP;
3453         else
3454                 imir &= ~E1000_IMIR_PORT_BP;
3455         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
3456
3457         /* tcp flags bits setting. */
3458         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
3459                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
3460                         imir_ext |= E1000_IMIREXT_CTRL_URG;
3461                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
3462                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
3463                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
3464                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
3465                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
3466                         imir_ext |= E1000_IMIREXT_CTRL_RST;
3467                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
3468                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
3469                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
3470                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
3471         } else
3472                 imir_ext |= E1000_IMIREXT_CTRL_BP;
3473         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
3474         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
3475         return 0;
3476 }
3477
3478 /*
3479  * igb_remove_5tuple_filter_82576 - remove a 5tuple filter
3480  *
3481  * @param
3482  * dev: Pointer to struct rte_eth_dev.
3483  * ntuple_filter: ponter to the filter that will be removed.
3484  *
3485  * @return
3486  *    - On success, zero.
3487  *    - On failure, a negative value.
3488  */
3489 static int
3490 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
3491                                 struct rte_eth_ntuple_filter *ntuple_filter)
3492 {
3493         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3494         struct e1000_filter_info *filter_info =
3495                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3496         struct e1000_5tuple_filter_info filter_5tuple;
3497         struct e1000_5tuple_filter *filter;
3498         int ret;
3499
3500         memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info));
3501         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3502                                             &filter_5tuple);
3503         if (ret < 0)
3504                 return ret;
3505
3506         filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
3507                                          &filter_5tuple);
3508         if (filter == NULL) {
3509                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3510                 return -ENOENT;
3511         }
3512
3513         filter_info->fivetuple_mask &= ~(1 << filter->index);
3514         TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries);
3515         rte_free(filter);
3516
3517         E1000_WRITE_REG(hw, E1000_FTQF(filter->index),
3518                         E1000_FTQF_VF_BP | E1000_FTQF_MASK);
3519         E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0);
3520         E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0);
3521         E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0);
3522         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
3523         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
3524         return 0;
3525 }
3526
3527 static int
3528 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
3529 {
3530         uint32_t rctl;
3531         struct e1000_hw *hw;
3532         struct rte_eth_dev_info dev_info;
3533         uint32_t frame_size = mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN +
3534                                      VLAN_TAG_SIZE);
3535
3536         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3537
3538 #ifdef RTE_LIBRTE_82571_SUPPORT
3539         /* XXX: not bigger than max_rx_pktlen */
3540         if (hw->mac.type == e1000_82571)
3541                 return -ENOTSUP;
3542 #endif
3543         eth_igb_infos_get(dev, &dev_info);
3544
3545         /* check that mtu is within the allowed range */
3546         if ((mtu < ETHER_MIN_MTU) ||
3547             (frame_size > dev_info.max_rx_pktlen))
3548                 return -EINVAL;
3549
3550         /* refuse mtu that requires the support of scattered packets when this
3551          * feature has not been enabled before. */
3552         if (!dev->data->scattered_rx &&
3553             frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)
3554                 return -EINVAL;
3555
3556         rctl = E1000_READ_REG(hw, E1000_RCTL);
3557
3558         /* switch to jumbo mode if needed */
3559         if (frame_size > ETHER_MAX_LEN) {
3560                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
3561                 rctl |= E1000_RCTL_LPE;
3562         } else {
3563                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
3564                 rctl &= ~E1000_RCTL_LPE;
3565         }
3566         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3567
3568         /* update max frame size */
3569         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
3570
3571         E1000_WRITE_REG(hw, E1000_RLPML,
3572                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
3573
3574         return 0;
3575 }
3576
3577 /*
3578  * igb_add_del_ntuple_filter - add or delete a ntuple filter
3579  *
3580  * @param
3581  * dev: Pointer to struct rte_eth_dev.
3582  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
3583  * add: if true, add filter, if false, remove filter
3584  *
3585  * @return
3586  *    - On success, zero.
3587  *    - On failure, a negative value.
3588  */
3589 static int
3590 igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
3591                         struct rte_eth_ntuple_filter *ntuple_filter,
3592                         bool add)
3593 {
3594         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3595         int ret;
3596
3597         switch (ntuple_filter->flags) {
3598         case RTE_5TUPLE_FLAGS:
3599         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3600                 if (hw->mac.type != e1000_82576)
3601                         return -ENOTSUP;
3602                 if (add)
3603                         ret = igb_add_5tuple_filter_82576(dev,
3604                                                           ntuple_filter);
3605                 else
3606                         ret = igb_remove_5tuple_filter_82576(dev,
3607                                                              ntuple_filter);
3608                 break;
3609         case RTE_2TUPLE_FLAGS:
3610         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3611                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
3612                         return -ENOTSUP;
3613                 if (add)
3614                         ret = igb_add_2tuple_filter(dev, ntuple_filter);
3615                 else
3616                         ret = igb_remove_2tuple_filter(dev, ntuple_filter);
3617                 break;
3618         default:
3619                 ret = -EINVAL;
3620                 break;
3621         }
3622
3623         return ret;
3624 }
3625
3626 /*
3627  * igb_get_ntuple_filter - get a ntuple filter
3628  *
3629  * @param
3630  * dev: Pointer to struct rte_eth_dev.
3631  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
3632  *
3633  * @return
3634  *    - On success, zero.
3635  *    - On failure, a negative value.
3636  */
3637 static int
3638 igb_get_ntuple_filter(struct rte_eth_dev *dev,
3639                         struct rte_eth_ntuple_filter *ntuple_filter)
3640 {
3641         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3642         struct e1000_filter_info *filter_info =
3643                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3644         struct e1000_5tuple_filter_info filter_5tuple;
3645         struct e1000_2tuple_filter_info filter_2tuple;
3646         struct e1000_5tuple_filter *p_5tuple_filter;
3647         struct e1000_2tuple_filter *p_2tuple_filter;
3648         int ret;
3649
3650         switch (ntuple_filter->flags) {
3651         case RTE_5TUPLE_FLAGS:
3652         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3653                 if (hw->mac.type != e1000_82576)
3654                         return -ENOTSUP;
3655                 memset(&filter_5tuple,
3656                         0,
3657                         sizeof(struct e1000_5tuple_filter_info));
3658                 ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3659                                                     &filter_5tuple);
3660                 if (ret < 0)
3661                         return ret;
3662                 p_5tuple_filter = igb_5tuple_filter_lookup_82576(
3663                                         &filter_info->fivetuple_list,
3664                                         &filter_5tuple);
3665                 if (p_5tuple_filter == NULL) {
3666                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3667                         return -ENOENT;
3668                 }
3669                 ntuple_filter->queue = p_5tuple_filter->queue;
3670                 break;
3671         case RTE_2TUPLE_FLAGS:
3672         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3673                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
3674                         return -ENOTSUP;
3675                 memset(&filter_2tuple,
3676                         0,
3677                         sizeof(struct e1000_2tuple_filter_info));
3678                 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple);
3679                 if (ret < 0)
3680                         return ret;
3681                 p_2tuple_filter = igb_2tuple_filter_lookup(
3682                                         &filter_info->twotuple_list,
3683                                         &filter_2tuple);
3684                 if (p_2tuple_filter == NULL) {
3685                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3686                         return -ENOENT;
3687                 }
3688                 ntuple_filter->queue = p_2tuple_filter->queue;
3689                 break;
3690         default:
3691                 ret = -EINVAL;
3692                 break;
3693         }
3694
3695         return 0;
3696 }
3697
3698 /*
3699  * igb_ntuple_filter_handle - Handle operations for ntuple filter.
3700  * @dev: pointer to rte_eth_dev structure
3701  * @filter_op:operation will be taken.
3702  * @arg: a pointer to specific structure corresponding to the filter_op
3703  */
3704 static int
3705 igb_ntuple_filter_handle(struct rte_eth_dev *dev,
3706                                 enum rte_filter_op filter_op,
3707                                 void *arg)
3708 {
3709         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3710         int ret;
3711
3712         MAC_TYPE_FILTER_SUP(hw->mac.type);
3713
3714         if (filter_op == RTE_ETH_FILTER_NOP)
3715                 return 0;
3716
3717         if (arg == NULL) {
3718                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
3719                             filter_op);
3720                 return -EINVAL;
3721         }
3722
3723         switch (filter_op) {
3724         case RTE_ETH_FILTER_ADD:
3725                 ret = igb_add_del_ntuple_filter(dev,
3726                         (struct rte_eth_ntuple_filter *)arg,
3727                         TRUE);
3728                 break;
3729         case RTE_ETH_FILTER_DELETE:
3730                 ret = igb_add_del_ntuple_filter(dev,
3731                         (struct rte_eth_ntuple_filter *)arg,
3732                         FALSE);
3733                 break;
3734         case RTE_ETH_FILTER_GET:
3735                 ret = igb_get_ntuple_filter(dev,
3736                         (struct rte_eth_ntuple_filter *)arg);
3737                 break;
3738         default:
3739                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
3740                 ret = -EINVAL;
3741                 break;
3742         }
3743         return ret;
3744 }
3745
3746 static inline int
3747 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info,
3748                         uint16_t ethertype)
3749 {
3750         int i;
3751
3752         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
3753                 if (filter_info->ethertype_filters[i] == ethertype &&
3754                     (filter_info->ethertype_mask & (1 << i)))
3755                         return i;
3756         }
3757         return -1;
3758 }
3759
3760 static inline int
3761 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info,
3762                         uint16_t ethertype)
3763 {
3764         int i;
3765
3766         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
3767                 if (!(filter_info->ethertype_mask & (1 << i))) {
3768                         filter_info->ethertype_mask |= 1 << i;
3769                         filter_info->ethertype_filters[i] = ethertype;
3770                         return i;
3771                 }
3772         }
3773         return -1;
3774 }
3775
3776 static inline int
3777 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info,
3778                         uint8_t idx)
3779 {
3780         if (idx >= E1000_MAX_ETQF_FILTERS)
3781                 return -1;
3782         filter_info->ethertype_mask &= ~(1 << idx);
3783         filter_info->ethertype_filters[idx] = 0;
3784         return idx;
3785 }
3786
3787
3788 static int
3789 igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
3790                         struct rte_eth_ethertype_filter *filter,
3791                         bool add)
3792 {
3793         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3794         struct e1000_filter_info *filter_info =
3795                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3796         uint32_t etqf = 0;
3797         int ret;
3798
3799         if (filter->ether_type == ETHER_TYPE_IPv4 ||
3800                 filter->ether_type == ETHER_TYPE_IPv6) {
3801                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
3802                         " ethertype filter.", filter->ether_type);
3803                 return -EINVAL;
3804         }
3805
3806         if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) {
3807                 PMD_DRV_LOG(ERR, "mac compare is unsupported.");
3808                 return -EINVAL;
3809         }
3810         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) {
3811                 PMD_DRV_LOG(ERR, "drop option is unsupported.");
3812                 return -EINVAL;
3813         }
3814
3815         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
3816         if (ret >= 0 && add) {
3817                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.",
3818                             filter->ether_type);
3819                 return -EEXIST;
3820         }
3821         if (ret < 0 && !add) {
3822                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
3823                             filter->ether_type);
3824                 return -ENOENT;
3825         }
3826
3827         if (add) {
3828                 ret = igb_ethertype_filter_insert(filter_info,
3829                         filter->ether_type);
3830                 if (ret < 0) {
3831                         PMD_DRV_LOG(ERR, "ethertype filters are full.");
3832                         return -ENOSYS;
3833                 }
3834
3835                 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE;
3836                 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE);
3837                 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT;
3838         } else {
3839                 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret);
3840                 if (ret < 0)
3841                         return -ENOSYS;
3842         }
3843         E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf);
3844         E1000_WRITE_FLUSH(hw);
3845
3846         return 0;
3847 }
3848
3849 static int
3850 igb_get_ethertype_filter(struct rte_eth_dev *dev,
3851                         struct rte_eth_ethertype_filter *filter)
3852 {
3853         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3854         struct e1000_filter_info *filter_info =
3855                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3856         uint32_t etqf;
3857         int ret;
3858
3859         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
3860         if (ret < 0) {
3861                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
3862                             filter->ether_type);
3863                 return -ENOENT;
3864         }
3865
3866         etqf = E1000_READ_REG(hw, E1000_ETQF(ret));
3867         if (etqf & E1000_ETQF_FILTER_ENABLE) {
3868                 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE;
3869                 filter->flags = 0;
3870                 filter->queue = (etqf & E1000_ETQF_QUEUE) >>
3871                                 E1000_ETQF_QUEUE_SHIFT;
3872                 return 0;
3873         }
3874
3875         return -ENOENT;
3876 }
3877
3878 /*
3879  * igb_ethertype_filter_handle - Handle operations for ethertype filter.
3880  * @dev: pointer to rte_eth_dev structure
3881  * @filter_op:operation will be taken.
3882  * @arg: a pointer to specific structure corresponding to the filter_op
3883  */
3884 static int
3885 igb_ethertype_filter_handle(struct rte_eth_dev *dev,
3886                                 enum rte_filter_op filter_op,
3887                                 void *arg)
3888 {
3889         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3890         int ret;
3891
3892         MAC_TYPE_FILTER_SUP(hw->mac.type);
3893
3894         if (filter_op == RTE_ETH_FILTER_NOP)
3895                 return 0;
3896
3897         if (arg == NULL) {
3898                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
3899                             filter_op);
3900                 return -EINVAL;
3901         }
3902
3903         switch (filter_op) {
3904         case RTE_ETH_FILTER_ADD:
3905                 ret = igb_add_del_ethertype_filter(dev,
3906                         (struct rte_eth_ethertype_filter *)arg,
3907                         TRUE);
3908                 break;
3909         case RTE_ETH_FILTER_DELETE:
3910                 ret = igb_add_del_ethertype_filter(dev,
3911                         (struct rte_eth_ethertype_filter *)arg,
3912                         FALSE);
3913                 break;
3914         case RTE_ETH_FILTER_GET:
3915                 ret = igb_get_ethertype_filter(dev,
3916                         (struct rte_eth_ethertype_filter *)arg);
3917                 break;
3918         default:
3919                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
3920                 ret = -EINVAL;
3921                 break;
3922         }
3923         return ret;
3924 }
3925
3926 static int
3927 eth_igb_filter_ctrl(struct rte_eth_dev *dev,
3928                      enum rte_filter_type filter_type,
3929                      enum rte_filter_op filter_op,
3930                      void *arg)
3931 {
3932         int ret = -EINVAL;
3933
3934         switch (filter_type) {
3935         case RTE_ETH_FILTER_NTUPLE:
3936                 ret = igb_ntuple_filter_handle(dev, filter_op, arg);
3937                 break;
3938         case RTE_ETH_FILTER_ETHERTYPE:
3939                 ret = igb_ethertype_filter_handle(dev, filter_op, arg);
3940                 break;
3941         case RTE_ETH_FILTER_SYN:
3942                 ret = eth_igb_syn_filter_handle(dev, filter_op, arg);
3943                 break;
3944         case RTE_ETH_FILTER_FLEXIBLE:
3945                 ret = eth_igb_flex_filter_handle(dev, filter_op, arg);
3946                 break;
3947         default:
3948                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
3949                                                         filter_type);
3950                 break;
3951         }
3952
3953         return ret;
3954 }
3955
3956 static int
3957 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
3958                          struct ether_addr *mc_addr_set,
3959                          uint32_t nb_mc_addr)
3960 {
3961         struct e1000_hw *hw;
3962
3963         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3964         e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr);
3965         return 0;
3966 }
3967
3968 static int
3969 igb_timesync_enable(struct rte_eth_dev *dev)
3970 {
3971         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3972         uint32_t tsync_ctl;
3973         uint32_t tsauxc;
3974
3975         /* Enable system time for it isn't on by default. */
3976         tsauxc = E1000_READ_REG(hw, E1000_TSAUXC);
3977         tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME;
3978         E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc);
3979
3980         /* Start incrementing the register used to timestamp PTP packets. */
3981         E1000_WRITE_REG(hw, E1000_TIMINCA, E1000_TIMINCA_INIT);
3982
3983         /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
3984         E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588),
3985                         (ETHER_TYPE_1588 |
3986                          E1000_ETQF_FILTER_ENABLE |
3987                          E1000_ETQF_1588));
3988
3989         /* Enable timestamping of received PTP packets. */
3990         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
3991         tsync_ctl |= E1000_TSYNCRXCTL_ENABLED;
3992         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl);
3993
3994         /* Enable Timestamping of transmitted PTP packets. */
3995         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
3996         tsync_ctl |= E1000_TSYNCTXCTL_ENABLED;
3997         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl);
3998
3999         return 0;
4000 }
4001
4002 static int
4003 igb_timesync_disable(struct rte_eth_dev *dev)
4004 {
4005         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4006         uint32_t tsync_ctl;
4007
4008         /* Disable timestamping of transmitted PTP packets. */
4009         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
4010         tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED;
4011         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl);
4012
4013         /* Disable timestamping of received PTP packets. */
4014         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
4015         tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED;
4016         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl);
4017
4018         /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
4019         E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0);
4020
4021         /* Stop incrementating the System Time registers. */
4022         E1000_WRITE_REG(hw, E1000_TIMINCA, 0);
4023
4024         return 0;
4025 }
4026
4027 static int
4028 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
4029                                struct timespec *timestamp,
4030                                uint32_t flags __rte_unused)
4031 {
4032         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4033         uint32_t tsync_rxctl;
4034         uint32_t rx_stmpl;
4035         uint32_t rx_stmph;
4036
4037         tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
4038         if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0)
4039                 return -EINVAL;
4040
4041         rx_stmpl = E1000_READ_REG(hw, E1000_RXSTMPL);
4042         rx_stmph = E1000_READ_REG(hw, E1000_RXSTMPH);
4043
4044         timestamp->tv_sec = (uint64_t)(((uint64_t)rx_stmph << 32) | rx_stmpl);
4045         timestamp->tv_nsec = 0;
4046
4047         return  0;
4048 }
4049
4050 static int
4051 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
4052                                struct timespec *timestamp)
4053 {
4054         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4055         uint32_t tsync_txctl;
4056         uint32_t tx_stmpl;
4057         uint32_t tx_stmph;
4058
4059         tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
4060         if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0)
4061                 return -EINVAL;
4062
4063         tx_stmpl = E1000_READ_REG(hw, E1000_TXSTMPL);
4064         tx_stmph = E1000_READ_REG(hw, E1000_TXSTMPH);
4065
4066         timestamp->tv_sec = (uint64_t)(((uint64_t)tx_stmph << 32) | tx_stmpl);
4067         timestamp->tv_nsec = 0;
4068
4069         return  0;
4070 }
4071
4072 static int
4073 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused)
4074 {
4075         int count = 0;
4076         int g_ind = 0;
4077         const struct reg_info *reg_group;
4078
4079         while ((reg_group = igb_regs[g_ind++]))
4080                 count += igb_reg_group_count(reg_group);
4081
4082         return count;
4083 }
4084
4085 static int
4086 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused)
4087 {
4088         int count = 0;
4089         int g_ind = 0;
4090         const struct reg_info *reg_group;
4091
4092         while ((reg_group = igbvf_regs[g_ind++]))
4093                 count += igb_reg_group_count(reg_group);
4094
4095         return count;
4096 }
4097
4098 static int
4099 eth_igb_get_regs(struct rte_eth_dev *dev,
4100         struct rte_dev_reg_info *regs)
4101 {
4102         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4103         uint32_t *data = regs->data;
4104         int g_ind = 0;
4105         int count = 0;
4106         const struct reg_info *reg_group;
4107
4108         /* Support only full register dump */
4109         if ((regs->length == 0) ||
4110             (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) {
4111                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4112                         hw->device_id;
4113                 while ((reg_group = igb_regs[g_ind++]))
4114                         count += igb_read_regs_group(dev, &data[count],
4115                                                         reg_group);
4116                 return 0;
4117         }
4118
4119         return -ENOTSUP;
4120 }
4121
4122 static int
4123 igbvf_get_regs(struct rte_eth_dev *dev,
4124         struct rte_dev_reg_info *regs)
4125 {
4126         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4127         uint32_t *data = regs->data;
4128         int g_ind = 0;
4129         int count = 0;
4130         const struct reg_info *reg_group;
4131
4132         /* Support only full register dump */
4133         if ((regs->length == 0) ||
4134             (regs->length == (uint32_t)igbvf_get_reg_length(dev))) {
4135                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4136                         hw->device_id;
4137                 while ((reg_group = igbvf_regs[g_ind++]))
4138                         count += igb_read_regs_group(dev, &data[count],
4139                                                         reg_group);
4140                 return 0;
4141         }
4142
4143         return -ENOTSUP;
4144 }
4145
4146 static int
4147 eth_igb_get_eeprom_length(struct rte_eth_dev *dev)
4148 {
4149         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4150
4151         /* Return unit is byte count */
4152         return hw->nvm.word_size * 2;
4153 }
4154
4155 static int
4156 eth_igb_get_eeprom(struct rte_eth_dev *dev,
4157         struct rte_dev_eeprom_info *in_eeprom)
4158 {
4159         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4160         struct e1000_nvm_info *nvm = &hw->nvm;
4161         uint16_t *data = in_eeprom->data;
4162         int first, length;
4163
4164         first = in_eeprom->offset >> 1;
4165         length = in_eeprom->length >> 1;
4166         if ((first >= hw->nvm.word_size) ||
4167             ((first + length) >= hw->nvm.word_size))
4168                 return -EINVAL;
4169
4170         in_eeprom->magic = hw->vendor_id |
4171                 ((uint32_t)hw->device_id << 16);
4172
4173         if ((nvm->ops.read) == NULL)
4174                 return -ENOTSUP;
4175
4176         return nvm->ops.read(hw, first, length, data);
4177 }
4178
4179 static int
4180 eth_igb_set_eeprom(struct rte_eth_dev *dev,
4181         struct rte_dev_eeprom_info *in_eeprom)
4182 {
4183         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4184         struct e1000_nvm_info *nvm = &hw->nvm;
4185         uint16_t *data = in_eeprom->data;
4186         int first, length;
4187
4188         first = in_eeprom->offset >> 1;
4189         length = in_eeprom->length >> 1;
4190         if ((first >= hw->nvm.word_size) ||
4191             ((first + length) >= hw->nvm.word_size))
4192                 return -EINVAL;
4193
4194         in_eeprom->magic = (uint32_t)hw->vendor_id |
4195                 ((uint32_t)hw->device_id << 16);
4196
4197         if ((nvm->ops.write) == NULL)
4198                 return -ENOTSUP;
4199         return nvm->ops.write(hw,  first, length, data);
4200 }
4201
4202 static struct rte_driver pmd_igb_drv = {
4203         .type = PMD_PDEV,
4204         .init = rte_igb_pmd_init,
4205 };
4206
4207 static struct rte_driver pmd_igbvf_drv = {
4208         .type = PMD_PDEV,
4209         .init = rte_igbvf_pmd_init,
4210 };
4211
4212 static int
4213 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
4214 {
4215         struct e1000_hw *hw =
4216                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4217         uint32_t mask = 1 << queue_id;
4218
4219         E1000_WRITE_REG(hw, E1000_EIMC, mask);
4220         E1000_WRITE_FLUSH(hw);
4221
4222         return 0;
4223 }
4224
4225 static int
4226 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
4227 {
4228         struct e1000_hw *hw =
4229                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4230         uint32_t mask = 1 << queue_id;
4231         uint32_t regval;
4232
4233         regval = E1000_READ_REG(hw, E1000_EIMS);
4234         E1000_WRITE_REG(hw, E1000_EIMS, regval | mask);
4235         E1000_WRITE_FLUSH(hw);
4236
4237         rte_intr_enable(&dev->pci_dev->intr_handle);
4238
4239         return 0;
4240 }
4241
4242 static void
4243 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t  msix_vector,
4244                    uint8_t index, uint8_t offset)
4245 {
4246         uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4247
4248         /* clear bits */
4249         val &= ~((uint32_t)0xFF << offset);
4250
4251         /* write vector and valid bit */
4252         val |= (msix_vector | E1000_IVAR_VALID) << offset;
4253
4254         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val);
4255 }
4256
4257 static void
4258 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction,
4259                            uint8_t queue, uint8_t msix_vector)
4260 {
4261         uint32_t tmp = 0;
4262
4263         if (hw->mac.type == e1000_82575) {
4264                 if (direction == 0)
4265                         tmp = E1000_EICR_RX_QUEUE0 << queue;
4266                 else if (direction == 1)
4267                         tmp = E1000_EICR_TX_QUEUE0 << queue;
4268                 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp);
4269         } else if (hw->mac.type == e1000_82576) {
4270                 if ((direction == 0) || (direction == 1))
4271                         eth_igb_write_ivar(hw, msix_vector, queue & 0x7,
4272                                            ((queue & 0x8) << 1) +
4273                                            8 * direction);
4274         } else if ((hw->mac.type == e1000_82580) ||
4275                         (hw->mac.type == e1000_i350) ||
4276                         (hw->mac.type == e1000_i354) ||
4277                         (hw->mac.type == e1000_i210) ||
4278                         (hw->mac.type == e1000_i211)) {
4279                 if ((direction == 0) || (direction == 1))
4280                         eth_igb_write_ivar(hw, msix_vector,
4281                                            queue >> 1,
4282                                            ((queue & 0x1) << 4) +
4283                                            8 * direction);
4284         }
4285 }
4286
4287 /* Sets up the hardware to generate MSI-X interrupts properly
4288  * @hw
4289  *  board private structure
4290  */
4291 static void
4292 eth_igb_configure_msix_intr(struct rte_eth_dev *dev)
4293 {
4294         int queue_id;
4295         uint32_t tmpval, regval, intr_mask;
4296         struct e1000_hw *hw =
4297                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4298         uint32_t vec = 0;
4299         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
4300
4301         /* won't configure msix register if no mapping is done
4302          * between intr vector and event fd
4303          */
4304         if (!rte_intr_dp_is_en(intr_handle))
4305                 return;
4306
4307         /* set interrupt vector for other causes */
4308         if (hw->mac.type == e1000_82575) {
4309                 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT);
4310                 /* enable MSI-X PBA support */
4311                 tmpval |= E1000_CTRL_EXT_PBA_CLR;
4312
4313                 /* Auto-Mask interrupts upon ICR read */
4314                 tmpval |= E1000_CTRL_EXT_EIAME;
4315                 tmpval |= E1000_CTRL_EXT_IRCA;
4316
4317                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval);
4318
4319                 /* enable msix_other interrupt */
4320                 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER);
4321                 regval = E1000_READ_REG(hw, E1000_EIAC);
4322                 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER);
4323                 regval = E1000_READ_REG(hw, E1000_EIAM);
4324                 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER);
4325         } else if ((hw->mac.type == e1000_82576) ||
4326                         (hw->mac.type == e1000_82580) ||
4327                         (hw->mac.type == e1000_i350) ||
4328                         (hw->mac.type == e1000_i354) ||
4329                         (hw->mac.type == e1000_i210) ||
4330                         (hw->mac.type == e1000_i211)) {
4331                 /* turn on MSI-X capability first */
4332                 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE |
4333                                         E1000_GPIE_PBA | E1000_GPIE_EIAME |
4334                                         E1000_GPIE_NSICR);
4335
4336                 intr_mask = (1 << intr_handle->max_intr) - 1;
4337                 regval = E1000_READ_REG(hw, E1000_EIAC);
4338                 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask);
4339
4340                 /* enable msix_other interrupt */
4341                 regval = E1000_READ_REG(hw, E1000_EIMS);
4342                 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask);
4343                 tmpval = (dev->data->nb_rx_queues | E1000_IVAR_VALID) << 8;
4344                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval);
4345         }
4346
4347         /* use EIAM to auto-mask when MSI-X interrupt
4348          * is asserted, this saves a register write for every interrupt
4349          */
4350         intr_mask = (1 << intr_handle->nb_efd) - 1;
4351         regval = E1000_READ_REG(hw, E1000_EIAM);
4352         E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask);
4353
4354         for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) {
4355                 eth_igb_assign_msix_vector(hw, 0, queue_id, vec);
4356                 intr_handle->intr_vec[queue_id] = vec;
4357                 if (vec < intr_handle->nb_efd - 1)
4358                         vec++;
4359         }
4360
4361         E1000_WRITE_FLUSH(hw);
4362 }
4363
4364 PMD_REGISTER_DRIVER(pmd_igb_drv);
4365 PMD_REGISTER_DRIVER(pmd_igbvf_drv);