drivers/net/e1000/igb_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <stdarg.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_interrupts.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_log.h>
  44 #include <rte_debug.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_memory.h>
  49 #include <rte_memzone.h>
  50 #include <rte_eal.h>
  51 #include <rte_atomic.h>
  52 #include <rte_malloc.h>
  53 #include <rte_dev.h>
  54
  55 #include "e1000_logs.h"
  56 #include "base/e1000_api.h"
  57 #include "e1000_ethdev.h"
  58 #include "igb_regs.h"
  59
  60 /*
  61  * Default values for port configuration
  62  */
  63 #define IGB_DEFAULT_RX_FREE_THRESH  32
  64 #define IGB_DEFAULT_RX_PTHRESH      8
  65 #define IGB_DEFAULT_RX_HTHRESH      8
  66 #define IGB_DEFAULT_RX_WTHRESH      0
  67
  68 #define IGB_DEFAULT_TX_PTHRESH      32
  69 #define IGB_DEFAULT_TX_HTHRESH      0
  70 #define IGB_DEFAULT_TX_WTHRESH      0
  71
  72 #define IGB_HKEY_MAX_INDEX 10
  73
  74 /* Bit shift and mask */
  75 #define IGB_4_BIT_WIDTH  (CHAR_BIT / 2)
  76 #define IGB_4_BIT_MASK   RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t)
  77 #define IGB_8_BIT_WIDTH  CHAR_BIT
  78 #define IGB_8_BIT_MASK   UINT8_MAX
  79
  80 /* Additional timesync values. */
  81 #define E1000_ETQF_FILTER_1588 3
  82 #define E1000_TIMINCA_INCVALUE 16000000
  83 #define E1000_TIMINCA_INIT     ((0x02 << E1000_TIMINCA_16NS_SHIFT) \
  84                                 | E1000_TIMINCA_INCVALUE)
  85 #define E1000_TSAUXC_DISABLE_SYSTIME 0x80000000
  86
  87 static int  eth_igb_configure(struct rte_eth_dev *dev);
  88 static int  eth_igb_start(struct rte_eth_dev *dev);
  89 static void eth_igb_stop(struct rte_eth_dev *dev);
  90 static void eth_igb_close(struct rte_eth_dev *dev);
  91 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev);
  92 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev);
  93 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev);
  94 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev);
  95 static int  eth_igb_link_update(struct rte_eth_dev *dev,
  96                                 int wait_to_complete);
  97 static void eth_igb_stats_get(struct rte_eth_dev *dev,
  98                                 struct rte_eth_stats *rte_stats);
  99 static void eth_igb_stats_reset(struct rte_eth_dev *dev);
 100 static void eth_igb_infos_get(struct rte_eth_dev *dev,
 101                               struct rte_eth_dev_info *dev_info);
 102 static void eth_igbvf_infos_get(struct rte_eth_dev *dev,
 103                                 struct rte_eth_dev_info *dev_info);
 104 static int  eth_igb_flow_ctrl_get(struct rte_eth_dev *dev,
 105                                 struct rte_eth_fc_conf *fc_conf);
 106 static int  eth_igb_flow_ctrl_set(struct rte_eth_dev *dev,
 107                                 struct rte_eth_fc_conf *fc_conf);
 108 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev);
 109 #ifdef RTE_NEXT_ABI
 110 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev);
 111 #endif
 112 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev);
 113 static int eth_igb_interrupt_action(struct rte_eth_dev *dev);
 114 static void eth_igb_interrupt_handler(struct rte_intr_handle *handle,
 115                                                         void *param);
 116 static int  igb_hardware_init(struct e1000_hw *hw);
 117 static void igb_hw_control_acquire(struct e1000_hw *hw);
 118 static void igb_hw_control_release(struct e1000_hw *hw);
 119 static void igb_init_manageability(struct e1000_hw *hw);
 120 static void igb_release_manageability(struct e1000_hw *hw);
 121
 122 static int  eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 123
 124 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev,
 125                 uint16_t vlan_id, int on);
 126 static void eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 127 static void eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 128
 129 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 130 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 131 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev);
 132 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev);
 133 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 134 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 135
 136 static int eth_igb_led_on(struct rte_eth_dev *dev);
 137 static int eth_igb_led_off(struct rte_eth_dev *dev);
 138
 139 static void igb_intr_disable(struct e1000_hw *hw);
 140 static int  igb_get_rx_buffer_size(struct e1000_hw *hw);
 141 static void eth_igb_rar_set(struct rte_eth_dev *dev,
 142                 struct ether_addr *mac_addr,
 143                 uint32_t index, uint32_t pool);
 144 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 145 static void eth_igb_default_mac_addr_set(struct rte_eth_dev *dev,
 146                 struct ether_addr *addr);
 147
 148 static void igbvf_intr_disable(struct e1000_hw *hw);
 149 static int igbvf_dev_configure(struct rte_eth_dev *dev);
 150 static int igbvf_dev_start(struct rte_eth_dev *dev);
 151 static void igbvf_dev_stop(struct rte_eth_dev *dev);
 152 static void igbvf_dev_close(struct rte_eth_dev *dev);
 153 static int eth_igbvf_link_update(struct e1000_hw *hw);
 154 static void eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats);
 155 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev);
 156 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev,
 157                 uint16_t vlan_id, int on);
 158 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on);
 159 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 160 static void igbvf_default_mac_addr_set(struct rte_eth_dev *dev,
 161                 struct ether_addr *addr);
 162 static int igbvf_get_reg_length(struct rte_eth_dev *dev);
 163 static int igbvf_get_regs(struct rte_eth_dev *dev,
 164                 struct rte_dev_reg_info *regs);
 165
 166 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev,
 167                                    struct rte_eth_rss_reta_entry64 *reta_conf,
 168                                    uint16_t reta_size);
 169 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev,
 170                                   struct rte_eth_rss_reta_entry64 *reta_conf,
 171                                   uint16_t reta_size);
 172
 173 static int eth_igb_syn_filter_set(struct rte_eth_dev *dev,
 174                         struct rte_eth_syn_filter *filter,
 175                         bool add);
 176 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev,
 177                         struct rte_eth_syn_filter *filter);
 178 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
 179                         enum rte_filter_op filter_op,
 180                         void *arg);
 181 static int igb_add_2tuple_filter(struct rte_eth_dev *dev,
 182                         struct rte_eth_ntuple_filter *ntuple_filter);
 183 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev,
 184                         struct rte_eth_ntuple_filter *ntuple_filter);
 185 static int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
 186                         struct rte_eth_flex_filter *filter,
 187                         bool add);
 188 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev,
 189                         struct rte_eth_flex_filter *filter);
 190 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
 191                         enum rte_filter_op filter_op,
 192                         void *arg);
 193 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
 194                         struct rte_eth_ntuple_filter *ntuple_filter);
 195 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
 196                         struct rte_eth_ntuple_filter *ntuple_filter);
 197 static int igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
 198                         struct rte_eth_ntuple_filter *filter,
 199                         bool add);
 200 static int igb_get_ntuple_filter(struct rte_eth_dev *dev,
 201                         struct rte_eth_ntuple_filter *filter);
 202 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev,
 203                                 enum rte_filter_op filter_op,
 204                                 void *arg);
 205 static int igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
 206                         struct rte_eth_ethertype_filter *filter,
 207                         bool add);
 208 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev,
 209                                 enum rte_filter_op filter_op,
 210                                 void *arg);
 211 static int igb_get_ethertype_filter(struct rte_eth_dev *dev,
 212                         struct rte_eth_ethertype_filter *filter);
 213 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev,
 214                      enum rte_filter_type filter_type,
 215                      enum rte_filter_op filter_op,
 216                      void *arg);
 217 static int eth_igb_get_reg_length(struct rte_eth_dev *dev);
 218 static int eth_igb_get_regs(struct rte_eth_dev *dev,
 219                 struct rte_dev_reg_info *regs);
 220 static int eth_igb_get_eeprom_length(struct rte_eth_dev *dev);
 221 static int eth_igb_get_eeprom(struct rte_eth_dev *dev,
 222                 struct rte_dev_eeprom_info *eeprom);
 223 static int eth_igb_set_eeprom(struct rte_eth_dev *dev,
 224                 struct rte_dev_eeprom_info *eeprom);
 225 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
 226                                     struct ether_addr *mc_addr_set,
 227                                     uint32_t nb_mc_addr);
 228 static int igb_timesync_enable(struct rte_eth_dev *dev);
 229 static int igb_timesync_disable(struct rte_eth_dev *dev);
 230 static int igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
 231                                           struct timespec *timestamp,
 232                                           uint32_t flags);
 233 static int igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
 234                                           struct timespec *timestamp);
 235 #ifdef RTE_NEXT_ABI
 236 static int eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev,
 237                                         uint16_t queue_id);
 238 static int eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev,
 239                                          uint16_t queue_id);
 240 static void eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction,
 241                                        uint8_t queue, uint8_t msix_vector);
 242 static void eth_igb_write_ivar(struct e1000_hw *hw, uint8_t msix_vector,
 243                                uint8_t index, uint8_t offset);
 244 #endif
 245 static void eth_igb_configure_msix_intr(struct rte_eth_dev *dev);
 246
 247 /*
 248  * Define VF Stats MACRO for Non "cleared on read" register
 249  */
 250 #define UPDATE_VF_STAT(reg, last, cur)            \
 251 {                                                 \
 252         u32 latest = E1000_READ_REG(hw, reg);     \
 253         cur += latest - last;                     \
 254         last = latest;                            \
 255 }
 256
 257
 258 #define IGB_FC_PAUSE_TIME 0x0680
 259 #define IGB_LINK_UPDATE_CHECK_TIMEOUT  90  /* 9s */
 260 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */
 261
 262 #define IGBVF_PMD_NAME "rte_igbvf_pmd"     /* PMD name */
 263
 264 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full;
 265
 266 /*
 267  * The set of PCI devices this driver supports
 268  */
 269 static const struct rte_pci_id pci_id_igb_map[] = {
 270
 271 #define RTE_PCI_DEV_ID_DECL_IGB(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 272 #include "rte_pci_dev_ids.h"
 273
 274 {0},
 275 };
 276
 277 /*
 278  * The set of PCI devices this driver supports (for 82576&I350 VF)
 279  */
 280 static const struct rte_pci_id pci_id_igbvf_map[] = {
 281
 282 #define RTE_PCI_DEV_ID_DECL_IGBVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 283 #include "rte_pci_dev_ids.h"
 284
 285 {0},
 286 };
 287
 288 static const struct eth_dev_ops eth_igb_ops = {
 289         .dev_configure        = eth_igb_configure,
 290         .dev_start            = eth_igb_start,
 291         .dev_stop             = eth_igb_stop,
 292         .dev_close            = eth_igb_close,
 293         .promiscuous_enable   = eth_igb_promiscuous_enable,
 294         .promiscuous_disable  = eth_igb_promiscuous_disable,
 295         .allmulticast_enable  = eth_igb_allmulticast_enable,
 296         .allmulticast_disable = eth_igb_allmulticast_disable,
 297         .link_update          = eth_igb_link_update,
 298         .stats_get            = eth_igb_stats_get,
 299         .stats_reset          = eth_igb_stats_reset,
 300         .dev_infos_get        = eth_igb_infos_get,
 301         .mtu_set              = eth_igb_mtu_set,
 302         .vlan_filter_set      = eth_igb_vlan_filter_set,
 303         .vlan_tpid_set        = eth_igb_vlan_tpid_set,
 304         .vlan_offload_set     = eth_igb_vlan_offload_set,
 305         .rx_queue_setup       = eth_igb_rx_queue_setup,
 306 #ifdef RTE_NEXT_ABI
 307         .rx_queue_intr_enable = eth_igb_rx_queue_intr_enable,
 308         .rx_queue_intr_disable = eth_igb_rx_queue_intr_disable,
 309 #endif
 310         .rx_queue_release     = eth_igb_rx_queue_release,
 311         .rx_queue_count       = eth_igb_rx_queue_count,
 312         .rx_descriptor_done   = eth_igb_rx_descriptor_done,
 313         .tx_queue_setup       = eth_igb_tx_queue_setup,
 314         .tx_queue_release     = eth_igb_tx_queue_release,
 315         .dev_led_on           = eth_igb_led_on,
 316         .dev_led_off          = eth_igb_led_off,
 317         .flow_ctrl_get        = eth_igb_flow_ctrl_get,
 318         .flow_ctrl_set        = eth_igb_flow_ctrl_set,
 319         .mac_addr_add         = eth_igb_rar_set,
 320         .mac_addr_remove      = eth_igb_rar_clear,
 321         .mac_addr_set         = eth_igb_default_mac_addr_set,
 322         .reta_update          = eth_igb_rss_reta_update,
 323         .reta_query           = eth_igb_rss_reta_query,
 324         .rss_hash_update      = eth_igb_rss_hash_update,
 325         .rss_hash_conf_get    = eth_igb_rss_hash_conf_get,
 326         .filter_ctrl          = eth_igb_filter_ctrl,
 327         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 328         .timesync_enable      = igb_timesync_enable,
 329         .timesync_disable     = igb_timesync_disable,
 330         .timesync_read_rx_timestamp = igb_timesync_read_rx_timestamp,
 331         .timesync_read_tx_timestamp = igb_timesync_read_tx_timestamp,
 332         .get_reg_length       = eth_igb_get_reg_length,
 333         .get_reg              = eth_igb_get_regs,
 334         .get_eeprom_length    = eth_igb_get_eeprom_length,
 335         .get_eeprom           = eth_igb_get_eeprom,
 336         .set_eeprom           = eth_igb_set_eeprom,
 337 };
 338
 339 /*
 340  * dev_ops for virtual function, bare necessities for basic vf
 341  * operation have been implemented
 342  */
 343 static const struct eth_dev_ops igbvf_eth_dev_ops = {
 344         .dev_configure        = igbvf_dev_configure,
 345         .dev_start            = igbvf_dev_start,
 346         .dev_stop             = igbvf_dev_stop,
 347         .dev_close            = igbvf_dev_close,
 348         .link_update          = eth_igb_link_update,
 349         .stats_get            = eth_igbvf_stats_get,
 350         .stats_reset          = eth_igbvf_stats_reset,
 351         .vlan_filter_set      = igbvf_vlan_filter_set,
 352         .dev_infos_get        = eth_igbvf_infos_get,
 353         .rx_queue_setup       = eth_igb_rx_queue_setup,
 354         .rx_queue_release     = eth_igb_rx_queue_release,
 355         .tx_queue_setup       = eth_igb_tx_queue_setup,
 356         .tx_queue_release     = eth_igb_tx_queue_release,
 357         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 358         .mac_addr_set         = igbvf_default_mac_addr_set,
 359         .get_reg_length       = igbvf_get_reg_length,
 360         .get_reg              = igbvf_get_regs,
 361 };
 362
 363 /**
 364  * Atomically reads the link status information from global
 365  * structure rte_eth_dev.
 366  *
 367  * @param dev
 368  *   - Pointer to the structure rte_eth_dev to read from.
 369  *   - Pointer to the buffer to be saved with the link status.
 370  *
 371  * @return
 372  *   - On success, zero.
 373  *   - On failure, negative value.
 374  */
 375 static inline int
 376 rte_igb_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 377                                 struct rte_eth_link *link)
 378 {
 379         struct rte_eth_link *dst = link;
 380         struct rte_eth_link *src = &(dev->data->dev_link);
 381
 382         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 383                                         *(uint64_t *)src) == 0)
 384                 return -1;
 385
 386         return 0;
 387 }
 388
 389 /**
 390  * Atomically writes the link status information into global
 391  * structure rte_eth_dev.
 392  *
 393  * @param dev
 394  *   - Pointer to the structure rte_eth_dev to read from.
 395  *   - Pointer to the buffer to be saved with the link status.
 396  *
 397  * @return
 398  *   - On success, zero.
 399  *   - On failure, negative value.
 400  */
 401 static inline int
 402 rte_igb_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 403                                 struct rte_eth_link *link)
 404 {
 405         struct rte_eth_link *dst = &(dev->data->dev_link);
 406         struct rte_eth_link *src = link;
 407
 408         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 409                                         *(uint64_t *)src) == 0)
 410                 return -1;
 411
 412         return 0;
 413 }
 414
 415 static inline void
 416 igb_intr_enable(struct rte_eth_dev *dev)
 417 {
 418         struct e1000_interrupt *intr =
 419                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 420         struct e1000_hw *hw =
 421                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 422
 423         E1000_WRITE_REG(hw, E1000_IMS, intr->mask);
 424         E1000_WRITE_FLUSH(hw);
 425 }
 426
 427 static void
 428 igb_intr_disable(struct e1000_hw *hw)
 429 {
 430         E1000_WRITE_REG(hw, E1000_IMC, ~0);
 431         E1000_WRITE_FLUSH(hw);
 432 }
 433
 434 static inline int32_t
 435 igb_pf_reset_hw(struct e1000_hw *hw)
 436 {
 437         uint32_t ctrl_ext;
 438         int32_t status;
 439
 440         status = e1000_reset_hw(hw);
 441
 442         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 443         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 444         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 445         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 446         E1000_WRITE_FLUSH(hw);
 447
 448         return status;
 449 }
 450
 451 static void
 452 igb_identify_hardware(struct rte_eth_dev *dev)
 453 {
 454         struct e1000_hw *hw =
 455                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 456
 457         hw->vendor_id = dev->pci_dev->id.vendor_id;
 458         hw->device_id = dev->pci_dev->id.device_id;
 459         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
 460         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
 461
 462         e1000_set_mac_type(hw);
 463
 464         /* need to check if it is a vf device below */
 465 }
 466
 467 static int
 468 igb_reset_swfw_lock(struct e1000_hw *hw)
 469 {
 470         int ret_val;
 471
 472         /*
 473          * Do mac ops initialization manually here, since we will need
 474          * some function pointers set by this call.
 475          */
 476         ret_val = e1000_init_mac_params(hw);
 477         if (ret_val)
 478                 return ret_val;
 479
 480         /*
 481          * SMBI lock should not fail in this early stage. If this is the case,
 482          * it is due to an improper exit of the application.
 483          * So force the release of the faulty lock.
 484          */
 485         if (e1000_get_hw_semaphore_generic(hw) < 0) {
 486                 PMD_DRV_LOG(DEBUG, "SMBI lock released");
 487         }
 488         e1000_put_hw_semaphore_generic(hw);
 489
 490         if (hw->mac.ops.acquire_swfw_sync != NULL) {
 491                 uint16_t mask;
 492
 493                 /*
 494                  * Phy lock should not fail in this early stage. If this is the case,
 495                  * it is due to an improper exit of the application.
 496                  * So force the release of the faulty lock.
 497                  */
 498                 mask = E1000_SWFW_PHY0_SM << hw->bus.func;
 499                 if (hw->bus.func > E1000_FUNC_1)
 500                         mask <<= 2;
 501                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 502                         PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released",
 503                                     hw->bus.func);
 504                 }
 505                 hw->mac.ops.release_swfw_sync(hw, mask);
 506
 507                 /*
 508                  * This one is more tricky since it is common to all ports; but
 509                  * swfw_sync retries last long enough (1s) to be almost sure that if
 510                  * lock can not be taken it is due to an improper lock of the
 511                  * semaphore.
 512                  */
 513                 mask = E1000_SWFW_EEP_SM;
 514                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 515                         PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 516                 }
 517                 hw->mac.ops.release_swfw_sync(hw, mask);
 518         }
 519
 520         return E1000_SUCCESS;
 521 }
 522
 523 static int
 524 eth_igb_dev_init(struct rte_eth_dev *eth_dev)
 525 {
 526         int error = 0;
 527         struct rte_pci_device *pci_dev;
 528         struct e1000_hw *hw =
 529                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 530         struct e1000_vfta * shadow_vfta =
 531                 E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 532         struct e1000_filter_info *filter_info =
 533                 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
 534         struct e1000_adapter *adapter =
 535                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 536
 537         uint32_t ctrl_ext;
 538
 539         pci_dev = eth_dev->pci_dev;
 540         eth_dev->dev_ops = &eth_igb_ops;
 541         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 542         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 543
 544         /* for secondary processes, we don't initialise any further as primary
 545          * has already done this work. Only check we don't need a different
 546          * RX function */
 547         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 548                 if (eth_dev->data->scattered_rx)
 549                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 550                 return 0;
 551         }
 552
 553         hw->hw_addr= (void *)pci_dev->mem_resource[0].addr;
 554
 555         igb_identify_hardware(eth_dev);
 556         if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) {
 557                 error = -EIO;
 558                 goto err_late;
 559         }
 560
 561         e1000_get_bus_info(hw);
 562
 563         /* Reset any pending lock */
 564         if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) {
 565                 error = -EIO;
 566                 goto err_late;
 567         }
 568
 569         /* Finish initialization */
 570         if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) {
 571                 error = -EIO;
 572                 goto err_late;
 573         }
 574
 575         hw->mac.autoneg = 1;
 576         hw->phy.autoneg_wait_to_complete = 0;
 577         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 578
 579         /* Copper options */
 580         if (hw->phy.media_type == e1000_media_type_copper) {
 581                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
 582                 hw->phy.disable_polarity_correction = 0;
 583                 hw->phy.ms_type = e1000_ms_hw_default;
 584         }
 585
 586         /*
 587          * Start from a known state, this is important in reading the nvm
 588          * and mac from that.
 589          */
 590         igb_pf_reset_hw(hw);
 591
 592         /* Make sure we have a good EEPROM before we read from it */
 593         if (e1000_validate_nvm_checksum(hw) < 0) {
 594                 /*
 595                  * Some PCI-E parts fail the first check due to
 596                  * the link being in sleep state, call it again,
 597                  * if it fails a second time its a real issue.
 598                  */
 599                 if (e1000_validate_nvm_checksum(hw) < 0) {
 600                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
 601                         error = -EIO;
 602                         goto err_late;
 603                 }
 604         }
 605
 606         /* Read the permanent MAC address out of the EEPROM */
 607         if (e1000_read_mac_addr(hw) != 0) {
 608                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
 609                 error = -EIO;
 610                 goto err_late;
 611         }
 612
 613         /* Allocate memory for storing MAC addresses */
 614         eth_dev->data->mac_addrs = rte_zmalloc("e1000",
 615                 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
 616         if (eth_dev->data->mac_addrs == NULL) {
 617                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to "
 618                                                 "store MAC addresses",
 619                                 ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 620                 error = -ENOMEM;
 621                 goto err_late;
 622         }
 623
 624         /* Copy the permanent MAC address */
 625         ether_addr_copy((struct ether_addr *)hw->mac.addr, &eth_dev->data->mac_addrs[0]);
 626
 627         /* initialize the vfta */
 628         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 629
 630         /* Now initialize the hardware */
 631         if (igb_hardware_init(hw) != 0) {
 632                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
 633                 rte_free(eth_dev->data->mac_addrs);
 634                 eth_dev->data->mac_addrs = NULL;
 635                 error = -ENODEV;
 636                 goto err_late;
 637         }
 638         hw->mac.get_link_status = 1;
 639         adapter->stopped = 0;
 640
 641         /* Indicate SOL/IDER usage */
 642         if (e1000_check_reset_block(hw) < 0) {
 643                 PMD_INIT_LOG(ERR, "PHY reset is blocked due to"
 644                                         "SOL/IDER session");
 645         }
 646
 647         /* initialize PF if max_vfs not zero */
 648         igb_pf_host_init(eth_dev);
 649
 650         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 651         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 652         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 653         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 654         E1000_WRITE_FLUSH(hw);
 655
 656         PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x",
 657                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 658                      pci_dev->id.device_id);
 659
 660         /* enable support intr */
 661         igb_intr_enable(eth_dev);
 662
 663         TAILQ_INIT(&filter_info->flex_list);
 664         filter_info->flex_mask = 0;
 665         TAILQ_INIT(&filter_info->twotuple_list);
 666         filter_info->twotuple_mask = 0;
 667         TAILQ_INIT(&filter_info->fivetuple_list);
 668         filter_info->fivetuple_mask = 0;
 669
 670         return 0;
 671
 672 err_late:
 673         igb_hw_control_release(hw);
 674
 675         return (error);
 676 }
 677
 678 static int
 679 eth_igb_dev_uninit(struct rte_eth_dev *eth_dev)
 680 {
 681         struct rte_pci_device *pci_dev;
 682         struct e1000_hw *hw;
 683         struct e1000_adapter *adapter =
 684                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 685
 686         PMD_INIT_FUNC_TRACE();
 687
 688         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 689                 return -EPERM;
 690
 691         hw = E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 692         pci_dev = eth_dev->pci_dev;
 693
 694         if (adapter->stopped == 0)
 695                 eth_igb_close(eth_dev);
 696
 697         eth_dev->dev_ops = NULL;
 698         eth_dev->rx_pkt_burst = NULL;
 699         eth_dev->tx_pkt_burst = NULL;
 700
 701         /* Reset any pending lock */
 702         igb_reset_swfw_lock(hw);
 703
 704         rte_free(eth_dev->data->mac_addrs);
 705         eth_dev->data->mac_addrs = NULL;
 706
 707         /* uninitialize PF if max_vfs not zero */
 708         igb_pf_host_uninit(eth_dev);
 709
 710         /* disable uio intr before callback unregister */
 711         rte_intr_disable(&(pci_dev->intr_handle));
 712         rte_intr_callback_unregister(&(pci_dev->intr_handle),
 713                 eth_igb_interrupt_handler, (void *)eth_dev);
 714
 715         return 0;
 716 }
 717
 718 /*
 719  * Virtual Function device init
 720  */
 721 static int
 722 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev)
 723 {
 724         struct rte_pci_device *pci_dev;
 725         struct e1000_adapter *adapter =
 726                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 727         struct e1000_hw *hw =
 728                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 729         int diag;
 730
 731         PMD_INIT_FUNC_TRACE();
 732
 733         eth_dev->dev_ops = &igbvf_eth_dev_ops;
 734         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 735         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 736
 737         /* for secondary processes, we don't initialise any further as primary
 738          * has already done this work. Only check we don't need a different
 739          * RX function */
 740         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 741                 if (eth_dev->data->scattered_rx)
 742                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 743                 return 0;
 744         }
 745
 746         pci_dev = eth_dev->pci_dev;
 747
 748         hw->device_id = pci_dev->id.device_id;
 749         hw->vendor_id = pci_dev->id.vendor_id;
 750         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 751         adapter->stopped = 0;
 752
 753         /* Initialize the shared code (base driver) */
 754         diag = e1000_setup_init_funcs(hw, TRUE);
 755         if (diag != 0) {
 756                 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d",
 757                         diag);
 758                 return -EIO;
 759         }
 760
 761         /* init_mailbox_params */
 762         hw->mbx.ops.init_params(hw);
 763
 764         /* Disable the interrupts for VF */
 765         igbvf_intr_disable(hw);
 766
 767         diag = hw->mac.ops.reset_hw(hw);
 768
 769         /* Allocate memory for storing MAC addresses */
 770         eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN *
 771                 hw->mac.rar_entry_count, 0);
 772         if (eth_dev->data->mac_addrs == NULL) {
 773                 PMD_INIT_LOG(ERR,
 774                         "Failed to allocate %d bytes needed to store MAC "
 775                         "addresses",
 776                         ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 777                 return -ENOMEM;
 778         }
 779
 780         /* Copy the permanent MAC address */
 781         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 782                         &eth_dev->data->mac_addrs[0]);
 783
 784         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x "
 785                      "mac.type=%s",
 786                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 787                      pci_dev->id.device_id, "igb_mac_82576_vf");
 788
 789         return 0;
 790 }
 791
 792 static int
 793 eth_igbvf_dev_uninit(struct rte_eth_dev *eth_dev)
 794 {
 795         struct e1000_adapter *adapter =
 796                 E1000_DEV_PRIVATE(eth_dev->data->dev_private);
 797
 798         PMD_INIT_FUNC_TRACE();
 799
 800         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 801                 return -EPERM;
 802
 803         if (adapter->stopped == 0)
 804                 igbvf_dev_close(eth_dev);
 805
 806         eth_dev->dev_ops = NULL;
 807         eth_dev->rx_pkt_burst = NULL;
 808         eth_dev->tx_pkt_burst = NULL;
 809
 810         rte_free(eth_dev->data->mac_addrs);
 811         eth_dev->data->mac_addrs = NULL;
 812
 813         return 0;
 814 }
 815
 816 static struct eth_driver rte_igb_pmd = {
 817         .pci_drv = {
 818                 .name = "rte_igb_pmd",
 819                 .id_table = pci_id_igb_map,
 820                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
 821                         RTE_PCI_DRV_DETACHABLE,
 822         },
 823         .eth_dev_init = eth_igb_dev_init,
 824         .eth_dev_uninit = eth_igb_dev_uninit,
 825         .dev_private_size = sizeof(struct e1000_adapter),
 826 };
 827
 828 /*
 829  * virtual function driver struct
 830  */
 831 static struct eth_driver rte_igbvf_pmd = {
 832         .pci_drv = {
 833                 .name = "rte_igbvf_pmd",
 834                 .id_table = pci_id_igbvf_map,
 835                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
 836         },
 837         .eth_dev_init = eth_igbvf_dev_init,
 838         .eth_dev_uninit = eth_igbvf_dev_uninit,
 839         .dev_private_size = sizeof(struct e1000_adapter),
 840 };
 841
 842 static int
 843 rte_igb_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 844 {
 845         rte_eth_driver_register(&rte_igb_pmd);
 846         return 0;
 847 }
 848
 849 static void
 850 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 851 {
 852         struct e1000_hw *hw =
 853                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 854         /* RCTL: enable VLAN filter since VMDq always use VLAN filter */
 855         uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL);
 856         rctl |= E1000_RCTL_VFE;
 857         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
 858 }
 859
 860 /*
 861  * VF Driver initialization routine.
 862  * Invoked one at EAL init time.
 863  * Register itself as the [Virtual Poll Mode] Driver of PCI IGB devices.
 864  */
 865 static int
 866 rte_igbvf_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 867 {
 868         PMD_INIT_FUNC_TRACE();
 869
 870         rte_eth_driver_register(&rte_igbvf_pmd);
 871         return (0);
 872 }
 873
 874 static int
 875 eth_igb_configure(struct rte_eth_dev *dev)
 876 {
 877         struct e1000_interrupt *intr =
 878                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 879
 880         PMD_INIT_FUNC_TRACE();
 881         intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
 882         PMD_INIT_FUNC_TRACE();
 883
 884         return (0);
 885 }
 886
 887 static int
 888 eth_igb_start(struct rte_eth_dev *dev)
 889 {
 890         struct e1000_hw *hw =
 891                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 892         struct e1000_adapter *adapter =
 893                 E1000_DEV_PRIVATE(dev->data->dev_private);
 894         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
 895         int ret, mask;
 896 #ifdef RTE_NEXT_ABI
 897         uint32_t intr_vector = 0;
 898 #endif
 899         uint32_t ctrl_ext;
 900
 901         PMD_INIT_FUNC_TRACE();
 902
 903         /* Power up the phy. Needed to make the link go Up */
 904         e1000_power_up_phy(hw);
 905
 906         /*
 907          * Packet Buffer Allocation (PBA)
 908          * Writing PBA sets the receive portion of the buffer
 909          * the remainder is used for the transmit buffer.
 910          */
 911         if (hw->mac.type == e1000_82575) {
 912                 uint32_t pba;
 913
 914                 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
 915                 E1000_WRITE_REG(hw, E1000_PBA, pba);
 916         }
 917
 918         /* Put the address into the Receive Address Array */
 919         e1000_rar_set(hw, hw->mac.addr, 0);
 920
 921         /* Initialize the hardware */
 922         if (igb_hardware_init(hw)) {
 923                 PMD_INIT_LOG(ERR, "Unable to initialize the hardware");
 924                 return (-EIO);
 925         }
 926         adapter->stopped = 0;
 927
 928         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
 929
 930         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 931         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 932         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 933         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 934         E1000_WRITE_FLUSH(hw);
 935
 936         /* configure PF module if SRIOV enabled */
 937         igb_pf_host_configure(dev);
 938
 939 #ifdef RTE_NEXT_ABI
 940         /* check and configure queue intr-vector mapping */
 941         if (dev->data->dev_conf.intr_conf.rxq != 0)
 942                 intr_vector = dev->data->nb_rx_queues;
 943
 944         if (rte_intr_efd_enable(intr_handle, intr_vector))
 945                 return -1;
 946
 947         if (rte_intr_dp_is_en(intr_handle)) {
 948                 intr_handle->intr_vec =
 949                         rte_zmalloc("intr_vec",
 950                                     dev->data->nb_rx_queues * sizeof(int), 0);
 951                 if (intr_handle->intr_vec == NULL) {
 952                         PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
 953                                      " intr_vec\n", dev->data->nb_rx_queues);
 954                         return -ENOMEM;
 955                 }
 956         }
 957 #endif
 958
 959         /* confiugre msix for rx interrupt */
 960         eth_igb_configure_msix_intr(dev);
 961
 962         /* Configure for OS presence */
 963         igb_init_manageability(hw);
 964
 965         eth_igb_tx_init(dev);
 966
 967         /* This can fail when allocating mbufs for descriptor rings */
 968         ret = eth_igb_rx_init(dev);
 969         if (ret) {
 970                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
 971                 igb_dev_clear_queues(dev);
 972                 return ret;
 973         }
 974
 975         e1000_clear_hw_cntrs_base_generic(hw);
 976
 977         /*
 978          * VLAN Offload Settings
 979          */
 980         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
 981                         ETH_VLAN_EXTEND_MASK;
 982         eth_igb_vlan_offload_set(dev, mask);
 983
 984         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
 985                 /* Enable VLAN filter since VMDq always use VLAN filter */
 986                 igb_vmdq_vlan_hw_filter_enable(dev);
 987         }
 988
 989         if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) ||
 990                 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) ||
 991                 (hw->mac.type == e1000_i211)) {
 992                 /* Configure EITR with the maximum possible value (0xFFFF) */
 993                 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF);
 994         }
 995
 996         /* Setup link speed and duplex */
 997         switch (dev->data->dev_conf.link_speed) {
 998         case ETH_LINK_SPEED_AUTONEG:
 999                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
1000                         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
1001                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
1002                         hw->phy.autoneg_advertised = E1000_ALL_HALF_DUPLEX;
1003                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
1004                         hw->phy.autoneg_advertised = E1000_ALL_FULL_DUPLEX;
1005                 else
1006                         goto error_invalid_config;
1007                 break;
1008         case ETH_LINK_SPEED_10:
1009                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
1010                         hw->phy.autoneg_advertised = E1000_ALL_10_SPEED;
1011                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
1012                         hw->phy.autoneg_advertised = ADVERTISE_10_HALF;
1013                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
1014                         hw->phy.autoneg_advertised = ADVERTISE_10_FULL;
1015                 else
1016                         goto error_invalid_config;
1017                 break;
1018         case ETH_LINK_SPEED_100:
1019                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
1020                         hw->phy.autoneg_advertised = E1000_ALL_100_SPEED;
1021                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
1022                         hw->phy.autoneg_advertised = ADVERTISE_100_HALF;
1023                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
1024                         hw->phy.autoneg_advertised = ADVERTISE_100_FULL;
1025                 else
1026                         goto error_invalid_config;
1027                 break;
1028         case ETH_LINK_SPEED_1000:
1029                 if ((dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX) ||
1030                                 (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX))
1031                         hw->phy.autoneg_advertised = ADVERTISE_1000_FULL;
1032                 else
1033                         goto error_invalid_config;
1034                 break;
1035         case ETH_LINK_SPEED_10000:
1036         default:
1037                 goto error_invalid_config;
1038         }
1039         e1000_setup_link(hw);
1040
1041         /* check if lsc interrupt feature is enabled */
1042         if (dev->data->dev_conf.intr_conf.lsc != 0) {
1043                 if (rte_intr_allow_others(intr_handle)) {
1044                         rte_intr_callback_register(intr_handle,
1045                                                    eth_igb_interrupt_handler,
1046                                                    (void *)dev);
1047                         eth_igb_lsc_interrupt_setup(dev);
1048                 } else
1049                         PMD_INIT_LOG(INFO, "lsc won't enable because of"
1050                                      " no intr multiplex\n");
1051         }
1052
1053 #ifdef RTE_NEXT_ABI
1054         /* check if rxq interrupt is enabled */
1055         if (dev->data->dev_conf.intr_conf.rxq != 0)
1056                 eth_igb_rxq_interrupt_setup(dev);
1057 #endif
1058
1059         /* enable uio/vfio intr/eventfd mapping */
1060         rte_intr_enable(intr_handle);
1061
1062         /* resume enabled intr since hw reset */
1063         igb_intr_enable(dev);
1064
1065         PMD_INIT_LOG(DEBUG, "<<");
1066
1067         return (0);
1068
1069 error_invalid_config:
1070         PMD_INIT_LOG(ERR, "Invalid link_speed/link_duplex (%u/%u) for port %u",
1071                      dev->data->dev_conf.link_speed,
1072                      dev->data->dev_conf.link_duplex, dev->data->port_id);
1073         igb_dev_clear_queues(dev);
1074         return (-EINVAL);
1075 }
1076
1077 /*********************************************************************
1078  *
1079  *  This routine disables all traffic on the adapter by issuing a
1080  *  global reset on the MAC.
1081  *
1082  **********************************************************************/
1083 static void
1084 eth_igb_stop(struct rte_eth_dev *dev)
1085 {
1086         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1087         struct e1000_filter_info *filter_info =
1088                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
1089         struct rte_eth_link link;
1090         struct e1000_flex_filter *p_flex;
1091         struct e1000_5tuple_filter *p_5tuple, *p_5tuple_next;
1092         struct e1000_2tuple_filter *p_2tuple, *p_2tuple_next;
1093         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1094
1095         igb_intr_disable(hw);
1096
1097         /* disable intr eventfd mapping */
1098         rte_intr_disable(intr_handle);
1099
1100         igb_pf_reset_hw(hw);
1101         E1000_WRITE_REG(hw, E1000_WUC, 0);
1102
1103         /* Set bit for Go Link disconnect */
1104         if (hw->mac.type >= e1000_82580) {
1105                 uint32_t phpm_reg;
1106
1107                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1108                 phpm_reg |= E1000_82580_PM_GO_LINKD;
1109                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1110         }
1111
1112         /* Power down the phy. Needed to make the link go Down */
1113         if (hw->phy.media_type == e1000_media_type_copper)
1114                 e1000_power_down_phy(hw);
1115         else
1116                 e1000_shutdown_fiber_serdes_link(hw);
1117
1118         igb_dev_clear_queues(dev);
1119
1120         /* clear the recorded link status */
1121         memset(&link, 0, sizeof(link));
1122         rte_igb_dev_atomic_write_link_status(dev, &link);
1123
1124         /* Remove all flex filters of the device */
1125         while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) {
1126                 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries);
1127                 rte_free(p_flex);
1128         }
1129         filter_info->flex_mask = 0;
1130
1131         /* Remove all ntuple filters of the device */
1132         for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list);
1133              p_5tuple != NULL; p_5tuple = p_5tuple_next) {
1134                 p_5tuple_next = TAILQ_NEXT(p_5tuple, entries);
1135                 TAILQ_REMOVE(&filter_info->fivetuple_list,
1136                              p_5tuple, entries);
1137                 rte_free(p_5tuple);
1138         }
1139         filter_info->fivetuple_mask = 0;
1140         for (p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list);
1141              p_2tuple != NULL; p_2tuple = p_2tuple_next) {
1142                 p_2tuple_next = TAILQ_NEXT(p_2tuple, entries);
1143                 TAILQ_REMOVE(&filter_info->twotuple_list,
1144                              p_2tuple, entries);
1145                 rte_free(p_2tuple);
1146         }
1147         filter_info->twotuple_mask = 0;
1148
1149 #ifdef RTE_NEXT_ABI
1150         /* Clean datapath event and queue/vec mapping */
1151         rte_intr_efd_disable(intr_handle);
1152         if (intr_handle->intr_vec != NULL) {
1153                 rte_free(intr_handle->intr_vec);
1154                 intr_handle->intr_vec = NULL;
1155         }
1156 #endif
1157 }
1158
1159 static void
1160 eth_igb_close(struct rte_eth_dev *dev)
1161 {
1162         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1163         struct e1000_adapter *adapter =
1164                 E1000_DEV_PRIVATE(dev->data->dev_private);
1165         struct rte_eth_link link;
1166 #ifdef RTE_NEXT_ABI
1167         struct rte_pci_device *pci_dev;
1168 #endif
1169
1170         eth_igb_stop(dev);
1171         adapter->stopped = 1;
1172
1173         e1000_phy_hw_reset(hw);
1174         igb_release_manageability(hw);
1175         igb_hw_control_release(hw);
1176
1177         /* Clear bit for Go Link disconnect */
1178         if (hw->mac.type >= e1000_82580) {
1179                 uint32_t phpm_reg;
1180
1181                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1182                 phpm_reg &= ~E1000_82580_PM_GO_LINKD;
1183                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1184         }
1185
1186         igb_dev_free_queues(dev);
1187
1188 #ifdef RTE_NEXT_ABI
1189         pci_dev = dev->pci_dev;
1190         if (pci_dev->intr_handle.intr_vec) {
1191                 rte_free(pci_dev->intr_handle.intr_vec);
1192                 pci_dev->intr_handle.intr_vec = NULL;
1193         }
1194 #endif
1195
1196         memset(&link, 0, sizeof(link));
1197         rte_igb_dev_atomic_write_link_status(dev, &link);
1198 }
1199
1200 static int
1201 igb_get_rx_buffer_size(struct e1000_hw *hw)
1202 {
1203         uint32_t rx_buf_size;
1204         if (hw->mac.type == e1000_82576) {
1205                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10;
1206         } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) {
1207                 /* PBS needs to be translated according to a lookup table */
1208                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf);
1209                 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size);
1210                 rx_buf_size = (rx_buf_size << 10);
1211         } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) {
1212                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10;
1213         } else {
1214                 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10;
1215         }
1216
1217         return rx_buf_size;
1218 }
1219
1220 /*********************************************************************
1221  *
1222  *  Initialize the hardware
1223  *
1224  **********************************************************************/
1225 static int
1226 igb_hardware_init(struct e1000_hw *hw)
1227 {
1228         uint32_t rx_buf_size;
1229         int diag;
1230
1231         /* Let the firmware know the OS is in control */
1232         igb_hw_control_acquire(hw);
1233
1234         /*
1235          * These parameters control the automatic generation (Tx) and
1236          * response (Rx) to Ethernet PAUSE frames.
1237          * - High water mark should allow for at least two standard size (1518)
1238          *   frames to be received after sending an XOFF.
1239          * - Low water mark works best when it is very near the high water mark.
1240          *   This allows the receiver to restart by sending XON when it has
1241          *   drained a bit. Here we use an arbitrary value of 1500 which will
1242          *   restart after one full frame is pulled from the buffer. There
1243          *   could be several smaller frames in the buffer and if so they will
1244          *   not trigger the XON until their total number reduces the buffer
1245          *   by 1500.
1246          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
1247          */
1248         rx_buf_size = igb_get_rx_buffer_size(hw);
1249
1250         hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2);
1251         hw->fc.low_water = hw->fc.high_water - 1500;
1252         hw->fc.pause_time = IGB_FC_PAUSE_TIME;
1253         hw->fc.send_xon = 1;
1254
1255         /* Set Flow control, use the tunable location if sane */
1256         if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4))
1257                 hw->fc.requested_mode = igb_fc_setting;
1258         else
1259                 hw->fc.requested_mode = e1000_fc_none;
1260
1261         /* Issue a global reset */
1262         igb_pf_reset_hw(hw);
1263         E1000_WRITE_REG(hw, E1000_WUC, 0);
1264
1265         diag = e1000_init_hw(hw);
1266         if (diag < 0)
1267                 return (diag);
1268
1269         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1270         e1000_get_phy_info(hw);
1271         e1000_check_for_link(hw);
1272
1273         return (0);
1274 }
1275
1276 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */
1277 static void
1278 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1279 {
1280         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1281         struct e1000_hw_stats *stats =
1282                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1283         int pause_frames;
1284
1285         if(hw->phy.media_type == e1000_media_type_copper ||
1286             (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
1287                 stats->symerrs +=
1288                     E1000_READ_REG(hw,E1000_SYMERRS);
1289                 stats->sec += E1000_READ_REG(hw, E1000_SEC);
1290         }
1291
1292         stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
1293         stats->mpc += E1000_READ_REG(hw, E1000_MPC);
1294         stats->scc += E1000_READ_REG(hw, E1000_SCC);
1295         stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
1296
1297         stats->mcc += E1000_READ_REG(hw, E1000_MCC);
1298         stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
1299         stats->colc += E1000_READ_REG(hw, E1000_COLC);
1300         stats->dc += E1000_READ_REG(hw, E1000_DC);
1301         stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
1302         stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
1303         stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
1304         /*
1305         ** For watchdog management we need to know if we have been
1306         ** paused during the last interval, so capture that here.
1307         */
1308         pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC);
1309         stats->xoffrxc += pause_frames;
1310         stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
1311         stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
1312         stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
1313         stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
1314         stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
1315         stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
1316         stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
1317         stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
1318         stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
1319         stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
1320         stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
1321         stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
1322
1323         /* For the 64-bit byte counters the low dword must be read first. */
1324         /* Both registers clear on the read of the high dword */
1325
1326         stats->gorc += E1000_READ_REG(hw, E1000_GORCL);
1327         stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32);
1328         stats->gotc += E1000_READ_REG(hw, E1000_GOTCL);
1329         stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32);
1330
1331         stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
1332         stats->ruc += E1000_READ_REG(hw, E1000_RUC);
1333         stats->rfc += E1000_READ_REG(hw, E1000_RFC);
1334         stats->roc += E1000_READ_REG(hw, E1000_ROC);
1335         stats->rjc += E1000_READ_REG(hw, E1000_RJC);
1336
1337         stats->tor += E1000_READ_REG(hw, E1000_TORH);
1338         stats->tot += E1000_READ_REG(hw, E1000_TOTH);
1339
1340         stats->tpr += E1000_READ_REG(hw, E1000_TPR);
1341         stats->tpt += E1000_READ_REG(hw, E1000_TPT);
1342         stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
1343         stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
1344         stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
1345         stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
1346         stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
1347         stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
1348         stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
1349         stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
1350
1351         /* Interrupt Counts */
1352
1353         stats->iac += E1000_READ_REG(hw, E1000_IAC);
1354         stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
1355         stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
1356         stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
1357         stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
1358         stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
1359         stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
1360         stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
1361         stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
1362
1363         /* Host to Card Statistics */
1364
1365         stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
1366         stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
1367         stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
1368         stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
1369         stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
1370         stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
1371         stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
1372         stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL);
1373         stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32);
1374         stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL);
1375         stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32);
1376         stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
1377         stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
1378         stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
1379
1380         stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
1381         stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
1382         stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
1383         stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
1384         stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
1385         stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
1386
1387         if (rte_stats == NULL)
1388                 return;
1389
1390         /* Rx Errors */
1391         rte_stats->ibadcrc = stats->crcerrs;
1392         rte_stats->ibadlen = stats->rlec + stats->ruc + stats->roc;
1393         rte_stats->imissed = stats->mpc;
1394         rte_stats->ierrors = rte_stats->ibadcrc +
1395                              rte_stats->ibadlen +
1396                              rte_stats->imissed +
1397                              stats->rxerrc + stats->algnerrc + stats->cexterr;
1398
1399         /* Tx Errors */
1400         rte_stats->oerrors = stats->ecol + stats->latecol;
1401
1402         /* XON/XOFF pause frames */
1403         rte_stats->tx_pause_xon  = stats->xontxc;
1404         rte_stats->rx_pause_xon  = stats->xonrxc;
1405         rte_stats->tx_pause_xoff = stats->xofftxc;
1406         rte_stats->rx_pause_xoff = stats->xoffrxc;
1407
1408         rte_stats->ipackets = stats->gprc;
1409         rte_stats->opackets = stats->gptc;
1410         rte_stats->ibytes   = stats->gorc;
1411         rte_stats->obytes   = stats->gotc;
1412 }
1413
1414 static void
1415 eth_igb_stats_reset(struct rte_eth_dev *dev)
1416 {
1417         struct e1000_hw_stats *hw_stats =
1418                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1419
1420         /* HW registers are cleared on read */
1421         eth_igb_stats_get(dev, NULL);
1422
1423         /* Reset software totals */
1424         memset(hw_stats, 0, sizeof(*hw_stats));
1425 }
1426
1427 static void
1428 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1429 {
1430         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1431         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1432                           E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1433
1434         /* Good Rx packets, include VF loopback */
1435         UPDATE_VF_STAT(E1000_VFGPRC,
1436             hw_stats->last_gprc, hw_stats->gprc);
1437
1438         /* Good Rx octets, include VF loopback */
1439         UPDATE_VF_STAT(E1000_VFGORC,
1440             hw_stats->last_gorc, hw_stats->gorc);
1441
1442         /* Good Tx packets, include VF loopback */
1443         UPDATE_VF_STAT(E1000_VFGPTC,
1444             hw_stats->last_gptc, hw_stats->gptc);
1445
1446         /* Good Tx octets, include VF loopback */
1447         UPDATE_VF_STAT(E1000_VFGOTC,
1448             hw_stats->last_gotc, hw_stats->gotc);
1449
1450         /* Rx Multicst packets */
1451         UPDATE_VF_STAT(E1000_VFMPRC,
1452             hw_stats->last_mprc, hw_stats->mprc);
1453
1454         /* Good Rx loopback packets */
1455         UPDATE_VF_STAT(E1000_VFGPRLBC,
1456             hw_stats->last_gprlbc, hw_stats->gprlbc);
1457
1458         /* Good Rx loopback octets */
1459         UPDATE_VF_STAT(E1000_VFGORLBC,
1460             hw_stats->last_gorlbc, hw_stats->gorlbc);
1461
1462         /* Good Tx loopback packets */
1463         UPDATE_VF_STAT(E1000_VFGPTLBC,
1464             hw_stats->last_gptlbc, hw_stats->gptlbc);
1465
1466         /* Good Tx loopback octets */
1467         UPDATE_VF_STAT(E1000_VFGOTLBC,
1468             hw_stats->last_gotlbc, hw_stats->gotlbc);
1469
1470         if (rte_stats == NULL)
1471                 return;
1472
1473         rte_stats->ipackets = hw_stats->gprc;
1474         rte_stats->ibytes = hw_stats->gorc;
1475         rte_stats->opackets = hw_stats->gptc;
1476         rte_stats->obytes = hw_stats->gotc;
1477         rte_stats->imcasts = hw_stats->mprc;
1478         rte_stats->ilbpackets = hw_stats->gprlbc;
1479         rte_stats->ilbbytes = hw_stats->gorlbc;
1480         rte_stats->olbpackets = hw_stats->gptlbc;
1481         rte_stats->olbbytes = hw_stats->gotlbc;
1482
1483 }
1484
1485 static void
1486 eth_igbvf_stats_reset(struct rte_eth_dev *dev)
1487 {
1488         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1489                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1490
1491         /* Sync HW register to the last stats */
1492         eth_igbvf_stats_get(dev, NULL);
1493
1494         /* reset HW current stats*/
1495         memset(&hw_stats->gprc, 0, sizeof(*hw_stats) -
1496                offsetof(struct e1000_vf_stats, gprc));
1497
1498 }
1499
1500 static void
1501 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1502 {
1503         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1504
1505         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1506         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1507         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1508         dev_info->rx_offload_capa =
1509                 DEV_RX_OFFLOAD_VLAN_STRIP |
1510                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1511                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1512                 DEV_RX_OFFLOAD_TCP_CKSUM;
1513         dev_info->tx_offload_capa =
1514                 DEV_TX_OFFLOAD_VLAN_INSERT |
1515                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1516                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1517                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1518                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1519
1520         switch (hw->mac.type) {
1521         case e1000_82575:
1522                 dev_info->max_rx_queues = 4;
1523                 dev_info->max_tx_queues = 4;
1524                 dev_info->max_vmdq_pools = 0;
1525                 break;
1526
1527         case e1000_82576:
1528                 dev_info->max_rx_queues = 16;
1529                 dev_info->max_tx_queues = 16;
1530                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1531                 dev_info->vmdq_queue_num = 16;
1532                 break;
1533
1534         case e1000_82580:
1535                 dev_info->max_rx_queues = 8;
1536                 dev_info->max_tx_queues = 8;
1537                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1538                 dev_info->vmdq_queue_num = 8;
1539                 break;
1540
1541         case e1000_i350:
1542                 dev_info->max_rx_queues = 8;
1543                 dev_info->max_tx_queues = 8;
1544                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1545                 dev_info->vmdq_queue_num = 8;
1546                 break;
1547
1548         case e1000_i354:
1549                 dev_info->max_rx_queues = 8;
1550                 dev_info->max_tx_queues = 8;
1551                 break;
1552
1553         case e1000_i210:
1554                 dev_info->max_rx_queues = 4;
1555                 dev_info->max_tx_queues = 4;
1556                 dev_info->max_vmdq_pools = 0;
1557                 break;
1558
1559         case e1000_i211:
1560                 dev_info->max_rx_queues = 2;
1561                 dev_info->max_tx_queues = 2;
1562                 dev_info->max_vmdq_pools = 0;
1563                 break;
1564
1565         default:
1566                 /* Should not happen */
1567                 break;
1568         }
1569         dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t);
1570         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
1571         dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL;
1572
1573         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1574                 .rx_thresh = {
1575                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1576                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1577                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
1578                 },
1579                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
1580                 .rx_drop_en = 0,
1581         };
1582
1583         dev_info->default_txconf = (struct rte_eth_txconf) {
1584                 .tx_thresh = {
1585                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
1586                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
1587                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
1588                 },
1589                 .txq_flags = 0,
1590         };
1591 }
1592
1593 static void
1594 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1595 {
1596         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1597
1598         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1599         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1600         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1601         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
1602                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1603                                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1604                                 DEV_RX_OFFLOAD_TCP_CKSUM;
1605         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
1606                                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1607                                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1608                                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1609                                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1610         switch (hw->mac.type) {
1611         case e1000_vfadapt:
1612                 dev_info->max_rx_queues = 2;
1613                 dev_info->max_tx_queues = 2;
1614                 break;
1615         case e1000_vfadapt_i350:
1616                 dev_info->max_rx_queues = 1;
1617                 dev_info->max_tx_queues = 1;
1618                 break;
1619         default:
1620                 /* Should not happen */
1621                 break;
1622         }
1623
1624         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1625                 .rx_thresh = {
1626                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1627                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1628                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
1629                 },
1630                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
1631                 .rx_drop_en = 0,
1632         };
1633
1634         dev_info->default_txconf = (struct rte_eth_txconf) {
1635                 .tx_thresh = {
1636                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
1637                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
1638                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
1639                 },
1640                 .txq_flags = 0,
1641         };
1642 }
1643
1644 /* return 0 means link status changed, -1 means not changed */
1645 static int
1646 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1647 {
1648         struct e1000_hw *hw =
1649                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1650         struct rte_eth_link link, old;
1651         int link_check, count;
1652
1653         link_check = 0;
1654         hw->mac.get_link_status = 1;
1655
1656         /* possible wait-to-complete in up to 9 seconds */
1657         for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) {
1658                 /* Read the real link status */
1659                 switch (hw->phy.media_type) {
1660                 case e1000_media_type_copper:
1661                         /* Do the work to read phy */
1662                         e1000_check_for_link(hw);
1663                         link_check = !hw->mac.get_link_status;
1664                         break;
1665
1666                 case e1000_media_type_fiber:
1667                         e1000_check_for_link(hw);
1668                         link_check = (E1000_READ_REG(hw, E1000_STATUS) &
1669                                       E1000_STATUS_LU);
1670                         break;
1671
1672                 case e1000_media_type_internal_serdes:
1673                         e1000_check_for_link(hw);
1674                         link_check = hw->mac.serdes_has_link;
1675                         break;
1676
1677                 /* VF device is type_unknown */
1678                 case e1000_media_type_unknown:
1679                         eth_igbvf_link_update(hw);
1680                         link_check = !hw->mac.get_link_status;
1681                         break;
1682
1683                 default:
1684                         break;
1685                 }
1686                 if (link_check || wait_to_complete == 0)
1687                         break;
1688                 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL);
1689         }
1690         memset(&link, 0, sizeof(link));
1691         rte_igb_dev_atomic_read_link_status(dev, &link);
1692         old = link;
1693
1694         /* Now we check if a transition has happened */
1695         if (link_check) {
1696                 hw->mac.ops.get_link_up_info(hw, &link.link_speed,
1697                                           &link.link_duplex);
1698                 link.link_status = 1;
1699         } else if (!link_check) {
1700                 link.link_speed = 0;
1701                 link.link_duplex = 0;
1702                 link.link_status = 0;
1703         }
1704         rte_igb_dev_atomic_write_link_status(dev, &link);
1705
1706         /* not changed */
1707         if (old.link_status == link.link_status)
1708                 return -1;
1709
1710         /* changed */
1711         return 0;
1712 }
1713
1714 /*
1715  * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
1716  * For ASF and Pass Through versions of f/w this means
1717  * that the driver is loaded.
1718  */
1719 static void
1720 igb_hw_control_acquire(struct e1000_hw *hw)
1721 {
1722         uint32_t ctrl_ext;
1723
1724         /* Let firmware know the driver has taken over */
1725         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1726         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1727 }
1728
1729 /*
1730  * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
1731  * For ASF and Pass Through versions of f/w this means that the
1732  * driver is no longer loaded.
1733  */
1734 static void
1735 igb_hw_control_release(struct e1000_hw *hw)
1736 {
1737         uint32_t ctrl_ext;
1738
1739         /* Let firmware taken over control of h/w */
1740         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1741         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1742                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1743 }
1744
1745 /*
1746  * Bit of a misnomer, what this really means is
1747  * to enable OS management of the system... aka
1748  * to disable special hardware management features.
1749  */
1750 static void
1751 igb_init_manageability(struct e1000_hw *hw)
1752 {
1753         if (e1000_enable_mng_pass_thru(hw)) {
1754                 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H);
1755                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
1756
1757                 /* disable hardware interception of ARP */
1758                 manc &= ~(E1000_MANC_ARP_EN);
1759
1760                 /* enable receiving management packets to the host */
1761                 manc |= E1000_MANC_EN_MNG2HOST;
1762                 manc2h |= 1 << 5;  /* Mng Port 623 */
1763                 manc2h |= 1 << 6;  /* Mng Port 664 */
1764                 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h);
1765                 E1000_WRITE_REG(hw, E1000_MANC, manc);
1766         }
1767 }
1768
1769 static void
1770 igb_release_manageability(struct e1000_hw *hw)
1771 {
1772         if (e1000_enable_mng_pass_thru(hw)) {
1773                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
1774
1775                 manc |= E1000_MANC_ARP_EN;
1776                 manc &= ~E1000_MANC_EN_MNG2HOST;
1777
1778                 E1000_WRITE_REG(hw, E1000_MANC, manc);
1779         }
1780 }
1781
1782 static void
1783 eth_igb_promiscuous_enable(struct rte_eth_dev *dev)
1784 {
1785         struct e1000_hw *hw =
1786                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1787         uint32_t rctl;
1788
1789         rctl = E1000_READ_REG(hw, E1000_RCTL);
1790         rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1791         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1792 }
1793
1794 static void
1795 eth_igb_promiscuous_disable(struct rte_eth_dev *dev)
1796 {
1797         struct e1000_hw *hw =
1798                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1799         uint32_t rctl;
1800
1801         rctl = E1000_READ_REG(hw, E1000_RCTL);
1802         rctl &= (~E1000_RCTL_UPE);
1803         if (dev->data->all_multicast == 1)
1804                 rctl |= E1000_RCTL_MPE;
1805         else
1806                 rctl &= (~E1000_RCTL_MPE);
1807         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1808 }
1809
1810 static void
1811 eth_igb_allmulticast_enable(struct rte_eth_dev *dev)
1812 {
1813         struct e1000_hw *hw =
1814                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1815         uint32_t rctl;
1816
1817         rctl = E1000_READ_REG(hw, E1000_RCTL);
1818         rctl |= E1000_RCTL_MPE;
1819         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1820 }
1821
1822 static void
1823 eth_igb_allmulticast_disable(struct rte_eth_dev *dev)
1824 {
1825         struct e1000_hw *hw =
1826                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1827         uint32_t rctl;
1828
1829         if (dev->data->promiscuous == 1)
1830                 return; /* must remain in all_multicast mode */
1831         rctl = E1000_READ_REG(hw, E1000_RCTL);
1832         rctl &= (~E1000_RCTL_MPE);
1833         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1834 }
1835
1836 static int
1837 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1838 {
1839         struct e1000_hw *hw =
1840                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1841         struct e1000_vfta * shadow_vfta =
1842                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1843         uint32_t vfta;
1844         uint32_t vid_idx;
1845         uint32_t vid_bit;
1846
1847         vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) &
1848                               E1000_VFTA_ENTRY_MASK);
1849         vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK));
1850         vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx);
1851         if (on)
1852                 vfta |= vid_bit;
1853         else
1854                 vfta &= ~vid_bit;
1855         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta);
1856
1857         /* update local VFTA copy */
1858         shadow_vfta->vfta[vid_idx] = vfta;
1859
1860         return 0;
1861 }
1862
1863 static void
1864 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1865 {
1866         struct e1000_hw *hw =
1867                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1868         uint32_t reg = ETHER_TYPE_VLAN ;
1869
1870         reg |= (tpid << 16);
1871         E1000_WRITE_REG(hw, E1000_VET, reg);
1872 }
1873
1874 static void
1875 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1876 {
1877         struct e1000_hw *hw =
1878                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1879         uint32_t reg;
1880
1881         /* Filter Table Disable */
1882         reg = E1000_READ_REG(hw, E1000_RCTL);
1883         reg &= ~E1000_RCTL_CFIEN;
1884         reg &= ~E1000_RCTL_VFE;
1885         E1000_WRITE_REG(hw, E1000_RCTL, reg);
1886 }
1887
1888 static void
1889 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1890 {
1891         struct e1000_hw *hw =
1892                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1893         struct e1000_vfta * shadow_vfta =
1894                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1895         uint32_t reg;
1896         int i;
1897
1898         /* Filter Table Enable, CFI not used for packet acceptance */
1899         reg = E1000_READ_REG(hw, E1000_RCTL);
1900         reg &= ~E1000_RCTL_CFIEN;
1901         reg |= E1000_RCTL_VFE;
1902         E1000_WRITE_REG(hw, E1000_RCTL, reg);
1903
1904         /* restore VFTA table */
1905         for (i = 0; i < IGB_VFTA_SIZE; i++)
1906                 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]);
1907 }
1908
1909 static void
1910 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev)
1911 {
1912         struct e1000_hw *hw =
1913                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1914         uint32_t reg;
1915
1916         /* VLAN Mode Disable */
1917         reg = E1000_READ_REG(hw, E1000_CTRL);
1918         reg &= ~E1000_CTRL_VME;
1919         E1000_WRITE_REG(hw, E1000_CTRL, reg);
1920 }
1921
1922 static void
1923 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev)
1924 {
1925         struct e1000_hw *hw =
1926                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1927         uint32_t reg;
1928
1929         /* VLAN Mode Enable */
1930         reg = E1000_READ_REG(hw, E1000_CTRL);
1931         reg |= E1000_CTRL_VME;
1932         E1000_WRITE_REG(hw, E1000_CTRL, reg);
1933 }
1934
1935 static void
1936 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1937 {
1938         struct e1000_hw *hw =
1939                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1940         uint32_t reg;
1941
1942         /* CTRL_EXT: Extended VLAN */
1943         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
1944         reg &= ~E1000_CTRL_EXT_EXTEND_VLAN;
1945         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
1946
1947         /* Update maximum packet length */
1948         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
1949                 E1000_WRITE_REG(hw, E1000_RLPML,
1950                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
1951                                                 VLAN_TAG_SIZE);
1952 }
1953
1954 static void
1955 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1956 {
1957         struct e1000_hw *hw =
1958                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1959         uint32_t reg;
1960
1961         /* CTRL_EXT: Extended VLAN */
1962         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
1963         reg |= E1000_CTRL_EXT_EXTEND_VLAN;
1964         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
1965
1966         /* Update maximum packet length */
1967         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
1968                 E1000_WRITE_REG(hw, E1000_RLPML,
1969                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
1970                                                 2 * VLAN_TAG_SIZE);
1971 }
1972
1973 static void
1974 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1975 {
1976         if(mask & ETH_VLAN_STRIP_MASK){
1977                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1978                         igb_vlan_hw_strip_enable(dev);
1979                 else
1980                         igb_vlan_hw_strip_disable(dev);
1981         }
1982
1983         if(mask & ETH_VLAN_FILTER_MASK){
1984                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1985                         igb_vlan_hw_filter_enable(dev);
1986                 else
1987                         igb_vlan_hw_filter_disable(dev);
1988         }
1989
1990         if(mask & ETH_VLAN_EXTEND_MASK){
1991                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1992                         igb_vlan_hw_extend_enable(dev);
1993                 else
1994                         igb_vlan_hw_extend_disable(dev);
1995         }
1996 }
1997
1998
1999 /**
2000  * It enables the interrupt mask and then enable the interrupt.
2001  *
2002  * @param dev
2003  *  Pointer to struct rte_eth_dev.
2004  *
2005  * @return
2006  *  - On success, zero.
2007  *  - On failure, a negative value.
2008  */
2009 static int
2010 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev)
2011 {
2012         struct e1000_interrupt *intr =
2013                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2014
2015         intr->mask |= E1000_ICR_LSC;
2016
2017         return 0;
2018 }
2019
2020 #ifdef RTE_NEXT_ABI
2021 /* It clears the interrupt causes and enables the interrupt.
2022  * It will be called once only during nic initialized.
2023  *
2024  * @param dev
2025  *  Pointer to struct rte_eth_dev.
2026  *
2027  * @return
2028  *  - On success, zero.
2029  *  - On failure, a negative value.
2030  */
2031 static int eth_igb_rxq_interrupt_setup(struct rte_eth_dev *dev)
2032 {
2033         uint32_t mask, regval;
2034         struct e1000_hw *hw =
2035                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2036         struct rte_eth_dev_info dev_info;
2037
2038         memset(&dev_info, 0, sizeof(dev_info));
2039         eth_igb_infos_get(dev, &dev_info);
2040
2041         mask = 0xFFFFFFFF >> (32 - dev_info.max_rx_queues);
2042         regval = E1000_READ_REG(hw, E1000_EIMS);
2043         E1000_WRITE_REG(hw, E1000_EIMS, regval | mask);
2044
2045         return 0;
2046 }
2047 #endif
2048
2049 /*
2050  * It reads ICR and gets interrupt causes, check it and set a bit flag
2051  * to update link status.
2052  *
2053  * @param dev
2054  *  Pointer to struct rte_eth_dev.
2055  *
2056  * @return
2057  *  - On success, zero.
2058  *  - On failure, a negative value.
2059  */
2060 static int
2061 eth_igb_interrupt_get_status(struct rte_eth_dev *dev)
2062 {
2063         uint32_t icr;
2064         struct e1000_hw *hw =
2065                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2066         struct e1000_interrupt *intr =
2067                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2068
2069         igb_intr_disable(hw);
2070
2071         /* read-on-clear nic registers here */
2072         icr = E1000_READ_REG(hw, E1000_ICR);
2073
2074         intr->flags = 0;
2075         if (icr & E1000_ICR_LSC) {
2076                 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
2077         }
2078
2079         if (icr & E1000_ICR_VMMB)
2080                 intr->flags |= E1000_FLAG_MAILBOX;
2081
2082         return 0;
2083 }
2084
2085 /*
2086  * It executes link_update after knowing an interrupt is prsent.
2087  *
2088  * @param dev
2089  *  Pointer to struct rte_eth_dev.
2090  *
2091  * @return
2092  *  - On success, zero.
2093  *  - On failure, a negative value.
2094  */
2095 static int
2096 eth_igb_interrupt_action(struct rte_eth_dev *dev)
2097 {
2098         struct e1000_hw *hw =
2099                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2100         struct e1000_interrupt *intr =
2101                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2102         uint32_t tctl, rctl;
2103         struct rte_eth_link link;
2104         int ret;
2105
2106         if (intr->flags & E1000_FLAG_MAILBOX) {
2107                 igb_pf_mbx_process(dev);
2108                 intr->flags &= ~E1000_FLAG_MAILBOX;
2109         }
2110
2111         igb_intr_enable(dev);
2112         rte_intr_enable(&(dev->pci_dev->intr_handle));
2113
2114         if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) {
2115                 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE;
2116
2117                 /* set get_link_status to check register later */
2118                 hw->mac.get_link_status = 1;
2119                 ret = eth_igb_link_update(dev, 0);
2120
2121                 /* check if link has changed */
2122                 if (ret < 0)
2123                         return 0;
2124
2125                 memset(&link, 0, sizeof(link));
2126                 rte_igb_dev_atomic_read_link_status(dev, &link);
2127                 if (link.link_status) {
2128                         PMD_INIT_LOG(INFO,
2129                                      " Port %d: Link Up - speed %u Mbps - %s",
2130                                      dev->data->port_id,
2131                                      (unsigned)link.link_speed,
2132                                      link.link_duplex == ETH_LINK_FULL_DUPLEX ?
2133                                      "full-duplex" : "half-duplex");
2134                 } else {
2135                         PMD_INIT_LOG(INFO, " Port %d: Link Down",
2136                                      dev->data->port_id);
2137                 }
2138
2139                 PMD_INIT_LOG(DEBUG, "PCI Address: %04d:%02d:%02d:%d",
2140                              dev->pci_dev->addr.domain,
2141                              dev->pci_dev->addr.bus,
2142                              dev->pci_dev->addr.devid,
2143                              dev->pci_dev->addr.function);
2144                 tctl = E1000_READ_REG(hw, E1000_TCTL);
2145                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2146                 if (link.link_status) {
2147                         /* enable Tx/Rx */
2148                         tctl |= E1000_TCTL_EN;
2149                         rctl |= E1000_RCTL_EN;
2150                 } else {
2151                         /* disable Tx/Rx */
2152                         tctl &= ~E1000_TCTL_EN;
2153                         rctl &= ~E1000_RCTL_EN;
2154                 }
2155                 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
2156                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2157                 E1000_WRITE_FLUSH(hw);
2158                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
2159         }
2160
2161         return 0;
2162 }
2163
2164 /**
2165  * Interrupt handler which shall be registered at first.
2166  *
2167  * @param handle
2168  *  Pointer to interrupt handle.
2169  * @param param
2170  *  The address of parameter (struct rte_eth_dev *) regsitered before.
2171  *
2172  * @return
2173  *  void
2174  */
2175 static void
2176 eth_igb_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
2177                                                         void *param)
2178 {
2179         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2180
2181         eth_igb_interrupt_get_status(dev);
2182         eth_igb_interrupt_action(dev);
2183 }
2184
2185 static int
2186 eth_igb_led_on(struct rte_eth_dev *dev)
2187 {
2188         struct e1000_hw *hw;
2189
2190         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2191         return (e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
2192 }
2193
2194 static int
2195 eth_igb_led_off(struct rte_eth_dev *dev)
2196 {
2197         struct e1000_hw *hw;
2198
2199         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2200         return (e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
2201 }
2202
2203 static int
2204 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2205 {
2206         struct e1000_hw *hw;
2207         uint32_t ctrl;
2208         int tx_pause;
2209         int rx_pause;
2210
2211         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2212         fc_conf->pause_time = hw->fc.pause_time;
2213         fc_conf->high_water = hw->fc.high_water;
2214         fc_conf->low_water = hw->fc.low_water;
2215         fc_conf->send_xon = hw->fc.send_xon;
2216         fc_conf->autoneg = hw->mac.autoneg;
2217
2218         /*
2219          * Return rx_pause and tx_pause status according to actual setting of
2220          * the TFCE and RFCE bits in the CTRL register.
2221          */
2222         ctrl = E1000_READ_REG(hw, E1000_CTRL);
2223         if (ctrl & E1000_CTRL_TFCE)
2224                 tx_pause = 1;
2225         else
2226                 tx_pause = 0;
2227
2228         if (ctrl & E1000_CTRL_RFCE)
2229                 rx_pause = 1;
2230         else
2231                 rx_pause = 0;
2232
2233         if (rx_pause && tx_pause)
2234                 fc_conf->mode = RTE_FC_FULL;
2235         else if (rx_pause)
2236                 fc_conf->mode = RTE_FC_RX_PAUSE;
2237         else if (tx_pause)
2238                 fc_conf->mode = RTE_FC_TX_PAUSE;
2239         else
2240                 fc_conf->mode = RTE_FC_NONE;
2241
2242         return 0;
2243 }
2244
2245 static int
2246 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2247 {
2248         struct e1000_hw *hw;
2249         int err;
2250         enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = {
2251                 e1000_fc_none,
2252                 e1000_fc_rx_pause,
2253                 e1000_fc_tx_pause,
2254                 e1000_fc_full
2255         };
2256         uint32_t rx_buf_size;
2257         uint32_t max_high_water;
2258         uint32_t rctl;
2259
2260         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2261         if (fc_conf->autoneg != hw->mac.autoneg)
2262                 return -ENOTSUP;
2263         rx_buf_size = igb_get_rx_buffer_size(hw);
2264         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2265
2266         /* At least reserve one Ethernet frame for watermark */
2267         max_high_water = rx_buf_size - ETHER_MAX_LEN;
2268         if ((fc_conf->high_water > max_high_water) ||
2269             (fc_conf->high_water < fc_conf->low_water)) {
2270                 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value");
2271                 PMD_INIT_LOG(ERR, "high water must <=  0x%x", max_high_water);
2272                 return (-EINVAL);
2273         }
2274
2275         hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode];
2276         hw->fc.pause_time     = fc_conf->pause_time;
2277         hw->fc.high_water     = fc_conf->high_water;
2278         hw->fc.low_water      = fc_conf->low_water;
2279         hw->fc.send_xon       = fc_conf->send_xon;
2280
2281         err = e1000_setup_link_generic(hw);
2282         if (err == E1000_SUCCESS) {
2283
2284                 /* check if we want to forward MAC frames - driver doesn't have native
2285                  * capability to do that, so we'll write the registers ourselves */
2286
2287                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2288
2289                 /* set or clear MFLCN.PMCF bit depending on configuration */
2290                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2291                         rctl |= E1000_RCTL_PMCF;
2292                 else
2293                         rctl &= ~E1000_RCTL_PMCF;
2294
2295                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2296                 E1000_WRITE_FLUSH(hw);
2297
2298                 return 0;
2299         }
2300
2301         PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err);
2302         return (-EIO);
2303 }
2304
2305 #define E1000_RAH_POOLSEL_SHIFT      (18)
2306 static void
2307 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2308                 uint32_t index, __rte_unused uint32_t pool)
2309 {
2310         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2311         uint32_t rah;
2312
2313         e1000_rar_set(hw, mac_addr->addr_bytes, index);
2314         rah = E1000_READ_REG(hw, E1000_RAH(index));
2315         rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool));
2316         E1000_WRITE_REG(hw, E1000_RAH(index), rah);
2317 }
2318
2319 static void
2320 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index)
2321 {
2322         uint8_t addr[ETHER_ADDR_LEN];
2323         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2324
2325         memset(addr, 0, sizeof(addr));
2326
2327         e1000_rar_set(hw, addr, index);
2328 }
2329
2330 static void
2331 eth_igb_default_mac_addr_set(struct rte_eth_dev *dev,
2332                                 struct ether_addr *addr)
2333 {
2334         eth_igb_rar_clear(dev, 0);
2335
2336         eth_igb_rar_set(dev, (void *)addr, 0, 0);
2337 }
2338 /*
2339  * Virtual Function operations
2340  */
2341 static void
2342 igbvf_intr_disable(struct e1000_hw *hw)
2343 {
2344         PMD_INIT_FUNC_TRACE();
2345
2346         /* Clear interrupt mask to stop from interrupts being generated */
2347         E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF);
2348
2349         E1000_WRITE_FLUSH(hw);
2350 }
2351
2352 static void
2353 igbvf_stop_adapter(struct rte_eth_dev *dev)
2354 {
2355         u32 reg_val;
2356         u16 i;
2357         struct rte_eth_dev_info dev_info;
2358         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2359
2360         memset(&dev_info, 0, sizeof(dev_info));
2361         eth_igbvf_infos_get(dev, &dev_info);
2362
2363         /* Clear interrupt mask to stop from interrupts being generated */
2364         igbvf_intr_disable(hw);
2365
2366         /* Clear any pending interrupts, flush previous writes */
2367         E1000_READ_REG(hw, E1000_EICR);
2368
2369         /* Disable the transmit unit.  Each queue must be disabled. */
2370         for (i = 0; i < dev_info.max_tx_queues; i++)
2371                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH);
2372
2373         /* Disable the receive unit by stopping each queue */
2374         for (i = 0; i < dev_info.max_rx_queues; i++) {
2375                 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i));
2376                 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE;
2377                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val);
2378                 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE)
2379                         ;
2380         }
2381
2382         /* flush all queues disables */
2383         E1000_WRITE_FLUSH(hw);
2384         msec_delay(2);
2385 }
2386
2387 static int eth_igbvf_link_update(struct e1000_hw *hw)
2388 {
2389         struct e1000_mbx_info *mbx = &hw->mbx;
2390         struct e1000_mac_info *mac = &hw->mac;
2391         int ret_val = E1000_SUCCESS;
2392
2393         PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf");
2394
2395         /*
2396          * We only want to run this if there has been a rst asserted.
2397          * in this case that could mean a link change, device reset,
2398          * or a virtual function reset
2399          */
2400
2401         /* If we were hit with a reset or timeout drop the link */
2402         if (!e1000_check_for_rst(hw, 0) || !mbx->timeout)
2403                 mac->get_link_status = TRUE;
2404
2405         if (!mac->get_link_status)
2406                 goto out;
2407
2408         /* if link status is down no point in checking to see if pf is up */
2409         if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU))
2410                 goto out;
2411
2412         /* if we passed all the tests above then the link is up and we no
2413          * longer need to check for link */
2414         mac->get_link_status = FALSE;
2415
2416 out:
2417         return ret_val;
2418 }
2419
2420
2421 static int
2422 igbvf_dev_configure(struct rte_eth_dev *dev)
2423 {
2424         struct rte_eth_conf* conf = &dev->data->dev_conf;
2425
2426         PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
2427                      dev->data->port_id);
2428
2429         /*
2430          * VF has no ability to enable/disable HW CRC
2431          * Keep the persistent behavior the same as Host PF
2432          */
2433 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC
2434         if (!conf->rxmode.hw_strip_crc) {
2435                 PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip");
2436                 conf->rxmode.hw_strip_crc = 1;
2437         }
2438 #else
2439         if (conf->rxmode.hw_strip_crc) {
2440                 PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip");
2441                 conf->rxmode.hw_strip_crc = 0;
2442         }
2443 #endif
2444
2445         return 0;
2446 }
2447
2448 static int
2449 igbvf_dev_start(struct rte_eth_dev *dev)
2450 {
2451         struct e1000_hw *hw =
2452                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2453         struct e1000_adapter *adapter =
2454                 E1000_DEV_PRIVATE(dev->data->dev_private);
2455         int ret;
2456
2457         PMD_INIT_FUNC_TRACE();
2458
2459         hw->mac.ops.reset_hw(hw);
2460         adapter->stopped = 0;
2461
2462         /* Set all vfta */
2463         igbvf_set_vfta_all(dev,1);
2464
2465         eth_igbvf_tx_init(dev);
2466
2467         /* This can fail when allocating mbufs for descriptor rings */
2468         ret = eth_igbvf_rx_init(dev);
2469         if (ret) {
2470                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
2471                 igb_dev_clear_queues(dev);
2472                 return ret;
2473         }
2474
2475         return 0;
2476 }
2477
2478 static void
2479 igbvf_dev_stop(struct rte_eth_dev *dev)
2480 {
2481         PMD_INIT_FUNC_TRACE();
2482
2483         igbvf_stop_adapter(dev);
2484
2485         /*
2486           * Clear what we set, but we still keep shadow_vfta to
2487           * restore after device starts
2488           */
2489         igbvf_set_vfta_all(dev,0);
2490
2491         igb_dev_clear_queues(dev);
2492 }
2493
2494 static void
2495 igbvf_dev_close(struct rte_eth_dev *dev)
2496 {
2497         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2498         struct e1000_adapter *adapter =
2499                 E1000_DEV_PRIVATE(dev->data->dev_private);
2500
2501         PMD_INIT_FUNC_TRACE();
2502
2503         e1000_reset_hw(hw);
2504
2505         igbvf_dev_stop(dev);
2506         adapter->stopped = 1;
2507         igb_dev_free_queues(dev);
2508 }
2509
2510 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on)
2511 {
2512         struct e1000_mbx_info *mbx = &hw->mbx;
2513         uint32_t msgbuf[2];
2514
2515         /* After set vlan, vlan strip will also be enabled in igb driver*/
2516         msgbuf[0] = E1000_VF_SET_VLAN;
2517         msgbuf[1] = vid;
2518         /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */
2519         if (on)
2520                 msgbuf[0] |= E1000_VF_SET_VLAN_ADD;
2521
2522         return (mbx->ops.write_posted(hw, msgbuf, 2, 0));
2523 }
2524
2525 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2526 {
2527         struct e1000_hw *hw =
2528                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2529         struct e1000_vfta * shadow_vfta =
2530                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2531         int i = 0, j = 0, vfta = 0, mask = 1;
2532
2533         for (i = 0; i < IGB_VFTA_SIZE; i++){
2534                 vfta = shadow_vfta->vfta[i];
2535                 if(vfta){
2536                         mask = 1;
2537                         for (j = 0; j < 32; j++){
2538                                 if(vfta & mask)
2539                                         igbvf_set_vfta(hw,
2540                                                 (uint16_t)((i<<5)+j), on);
2541                                 mask<<=1;
2542                         }
2543                 }
2544         }
2545
2546 }
2547
2548 static int
2549 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2550 {
2551         struct e1000_hw *hw =
2552                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2553         struct e1000_vfta * shadow_vfta =
2554                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2555         uint32_t vid_idx = 0;
2556         uint32_t vid_bit = 0;
2557         int ret = 0;
2558
2559         PMD_INIT_FUNC_TRACE();
2560
2561         /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/
2562         ret = igbvf_set_vfta(hw, vlan_id, !!on);
2563         if(ret){
2564                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2565                 return ret;
2566         }
2567         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2568         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2569
2570         /*Save what we set and retore it after device reset*/
2571         if (on)
2572                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2573         else
2574                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2575
2576         return 0;
2577 }
2578
2579 static void
2580 igbvf_default_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *addr)
2581 {
2582         struct e1000_hw *hw =
2583                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2584
2585         /* index is not used by rar_set() */
2586         hw->mac.ops.rar_set(hw, (void *)addr, 0);
2587 }
2588
2589
2590 static int
2591 eth_igb_rss_reta_update(struct rte_eth_dev *dev,
2592                         struct rte_eth_rss_reta_entry64 *reta_conf,
2593                         uint16_t reta_size)
2594 {
2595         uint8_t i, j, mask;
2596         uint32_t reta, r;
2597         uint16_t idx, shift;
2598         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2599
2600         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2601                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2602                         "(%d) doesn't match the number hardware can supported "
2603                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2604                 return -EINVAL;
2605         }
2606
2607         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
2608                 idx = i / RTE_RETA_GROUP_SIZE;
2609                 shift = i % RTE_RETA_GROUP_SIZE;
2610                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2611                                                 IGB_4_BIT_MASK);
2612                 if (!mask)
2613                         continue;
2614                 if (mask == IGB_4_BIT_MASK)
2615                         r = 0;
2616                 else
2617                         r = E1000_READ_REG(hw, E1000_RETA(i >> 2));
2618                 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) {
2619                         if (mask & (0x1 << j))
2620                                 reta |= reta_conf[idx].reta[shift + j] <<
2621                                                         (CHAR_BIT * j);
2622                         else
2623                                 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j));
2624                 }
2625                 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta);
2626         }
2627
2628         return 0;
2629 }
2630
2631 static int
2632 eth_igb_rss_reta_query(struct rte_eth_dev *dev,
2633                        struct rte_eth_rss_reta_entry64 *reta_conf,
2634                        uint16_t reta_size)
2635 {
2636         uint8_t i, j, mask;
2637         uint32_t reta;
2638         uint16_t idx, shift;
2639         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2640
2641         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2642                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2643                         "(%d) doesn't match the number hardware can supported "
2644                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2645                 return -EINVAL;
2646         }
2647
2648         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
2649                 idx = i / RTE_RETA_GROUP_SIZE;
2650                 shift = i % RTE_RETA_GROUP_SIZE;
2651                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2652                                                 IGB_4_BIT_MASK);
2653                 if (!mask)
2654                         continue;
2655                 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2));
2656                 for (j = 0; j < IGB_4_BIT_WIDTH; j++) {
2657                         if (mask & (0x1 << j))
2658                                 reta_conf[idx].reta[shift + j] =
2659                                         ((reta >> (CHAR_BIT * j)) &
2660                                                 IGB_8_BIT_MASK);
2661                 }
2662         }
2663
2664         return 0;
2665 }
2666
2667 #define MAC_TYPE_FILTER_SUP(type)    do {\
2668         if ((type) != e1000_82580 && (type) != e1000_i350 &&\
2669                 (type) != e1000_82576)\
2670                 return -ENOTSUP;\
2671 } while (0)
2672
2673 static int
2674 eth_igb_syn_filter_set(struct rte_eth_dev *dev,
2675                         struct rte_eth_syn_filter *filter,
2676                         bool add)
2677 {
2678         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2679         uint32_t synqf, rfctl;
2680
2681         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
2682                 return -EINVAL;
2683
2684         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
2685
2686         if (add) {
2687                 if (synqf & E1000_SYN_FILTER_ENABLE)
2688                         return -EINVAL;
2689
2690                 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) &
2691                         E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE);
2692
2693                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
2694                 if (filter->hig_pri)
2695                         rfctl |= E1000_RFCTL_SYNQFP;
2696                 else
2697                         rfctl &= ~E1000_RFCTL_SYNQFP;
2698
2699                 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl);
2700         } else {
2701                 if (!(synqf & E1000_SYN_FILTER_ENABLE))
2702                         return -ENOENT;
2703                 synqf = 0;
2704         }
2705
2706         E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf);
2707         E1000_WRITE_FLUSH(hw);
2708         return 0;
2709 }
2710
2711 static int
2712 eth_igb_syn_filter_get(struct rte_eth_dev *dev,
2713                         struct rte_eth_syn_filter *filter)
2714 {
2715         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2716         uint32_t synqf, rfctl;
2717
2718         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
2719         if (synqf & E1000_SYN_FILTER_ENABLE) {
2720                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
2721                 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0;
2722                 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >>
2723                                 E1000_SYN_FILTER_QUEUE_SHIFT);
2724                 return 0;
2725         }
2726
2727         return -ENOENT;
2728 }
2729
2730 static int
2731 eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
2732                         enum rte_filter_op filter_op,
2733                         void *arg)
2734 {
2735         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2736         int ret;
2737
2738         MAC_TYPE_FILTER_SUP(hw->mac.type);
2739
2740         if (filter_op == RTE_ETH_FILTER_NOP)
2741                 return 0;
2742
2743         if (arg == NULL) {
2744                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
2745                             filter_op);
2746                 return -EINVAL;
2747         }
2748
2749         switch (filter_op) {
2750         case RTE_ETH_FILTER_ADD:
2751                 ret = eth_igb_syn_filter_set(dev,
2752                                 (struct rte_eth_syn_filter *)arg,
2753                                 TRUE);
2754                 break;
2755         case RTE_ETH_FILTER_DELETE:
2756                 ret = eth_igb_syn_filter_set(dev,
2757                                 (struct rte_eth_syn_filter *)arg,
2758                                 FALSE);
2759                 break;
2760         case RTE_ETH_FILTER_GET:
2761                 ret = eth_igb_syn_filter_get(dev,
2762                                 (struct rte_eth_syn_filter *)arg);
2763                 break;
2764         default:
2765                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
2766                 ret = -EINVAL;
2767                 break;
2768         }
2769
2770         return ret;
2771 }
2772
2773 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
2774         if ((type) != e1000_82580 && (type) != e1000_i350)\
2775                 return -ENOSYS; \
2776 } while (0)
2777
2778 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/
2779 static inline int
2780 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter,
2781                         struct e1000_2tuple_filter_info *filter_info)
2782 {
2783         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
2784                 return -EINVAL;
2785         if (filter->priority > E1000_2TUPLE_MAX_PRI)
2786                 return -EINVAL;  /* filter index is out of range. */
2787         if (filter->tcp_flags > TCP_FLAG_ALL)
2788                 return -EINVAL;  /* flags is invalid. */
2789
2790         switch (filter->dst_port_mask) {
2791         case UINT16_MAX:
2792                 filter_info->dst_port_mask = 0;
2793                 filter_info->dst_port = filter->dst_port;
2794                 break;
2795         case 0:
2796                 filter_info->dst_port_mask = 1;
2797                 break;
2798         default:
2799                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
2800                 return -EINVAL;
2801         }
2802
2803         switch (filter->proto_mask) {
2804         case UINT8_MAX:
2805                 filter_info->proto_mask = 0;
2806                 filter_info->proto = filter->proto;
2807                 break;
2808         case 0:
2809                 filter_info->proto_mask = 1;
2810                 break;
2811         default:
2812                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
2813                 return -EINVAL;
2814         }
2815
2816         filter_info->priority = (uint8_t)filter->priority;
2817         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
2818                 filter_info->tcp_flags = filter->tcp_flags;
2819         else
2820                 filter_info->tcp_flags = 0;
2821
2822         return 0;
2823 }
2824
2825 static inline struct e1000_2tuple_filter *
2826 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list,
2827                         struct e1000_2tuple_filter_info *key)
2828 {
2829         struct e1000_2tuple_filter *it;
2830
2831         TAILQ_FOREACH(it, filter_list, entries) {
2832                 if (memcmp(key, &it->filter_info,
2833                         sizeof(struct e1000_2tuple_filter_info)) == 0) {
2834                         return it;
2835                 }
2836         }
2837         return NULL;
2838 }
2839
2840 /*
2841  * igb_add_2tuple_filter - add a 2tuple filter
2842  *
2843  * @param
2844  * dev: Pointer to struct rte_eth_dev.
2845  * ntuple_filter: ponter to the filter that will be added.
2846  *
2847  * @return
2848  *    - On success, zero.
2849  *    - On failure, a negative value.
2850  */
2851 static int
2852 igb_add_2tuple_filter(struct rte_eth_dev *dev,
2853                         struct rte_eth_ntuple_filter *ntuple_filter)
2854 {
2855         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2856         struct e1000_filter_info *filter_info =
2857                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
2858         struct e1000_2tuple_filter *filter;
2859         uint32_t ttqf = E1000_TTQF_DISABLE_MASK;
2860         uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP;
2861         int i, ret;
2862
2863         filter = rte_zmalloc("e1000_2tuple_filter",
2864                         sizeof(struct e1000_2tuple_filter), 0);
2865         if (filter == NULL)
2866                 return -ENOMEM;
2867
2868         ret = ntuple_filter_to_2tuple(ntuple_filter,
2869                                       &filter->filter_info);
2870         if (ret < 0) {
2871                 rte_free(filter);
2872                 return ret;
2873         }
2874         if (igb_2tuple_filter_lookup(&filter_info->twotuple_list,
2875                                          &filter->filter_info) != NULL) {
2876                 PMD_DRV_LOG(ERR, "filter exists.");
2877                 rte_free(filter);
2878                 return -EEXIST;
2879         }
2880         filter->queue = ntuple_filter->queue;
2881
2882         /*
2883          * look for an unused 2tuple filter index,
2884          * and insert the filter to list.
2885          */
2886         for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) {
2887                 if (!(filter_info->twotuple_mask & (1 << i))) {
2888                         filter_info->twotuple_mask |= 1 << i;
2889                         filter->index = i;
2890                         TAILQ_INSERT_TAIL(&filter_info->twotuple_list,
2891                                           filter,
2892                                           entries);
2893                         break;
2894                 }
2895         }
2896         if (i >= E1000_MAX_TTQF_FILTERS) {
2897                 PMD_DRV_LOG(ERR, "2tuple filters are full.");
2898                 rte_free(filter);
2899                 return -ENOSYS;
2900         }
2901
2902         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
2903         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
2904                 imir |= E1000_IMIR_PORT_BP;
2905         else
2906                 imir &= ~E1000_IMIR_PORT_BP;
2907
2908         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
2909
2910         ttqf |= E1000_TTQF_QUEUE_ENABLE;
2911         ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT);
2912         ttqf |= (uint32_t)(filter->filter_info.proto & E1000_TTQF_PROTOCOL_MASK);
2913         if (filter->filter_info.proto_mask == 0)
2914                 ttqf &= ~E1000_TTQF_MASK_ENABLE;
2915
2916         /* tcp flags bits setting. */
2917         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
2918                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
2919                         imir_ext |= E1000_IMIREXT_CTRL_URG;
2920                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
2921                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
2922                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
2923                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
2924                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
2925                         imir_ext |= E1000_IMIREXT_CTRL_RST;
2926                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
2927                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
2928                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
2929                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
2930         } else
2931                 imir_ext |= E1000_IMIREXT_CTRL_BP;
2932         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
2933         E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf);
2934         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
2935         return 0;
2936 }
2937
2938 /*
2939  * igb_remove_2tuple_filter - remove a 2tuple filter
2940  *
2941  * @param
2942  * dev: Pointer to struct rte_eth_dev.
2943  * ntuple_filter: ponter to the filter that will be removed.
2944  *
2945  * @return
2946  *    - On success, zero.
2947  *    - On failure, a negative value.
2948  */
2949 static int
2950 igb_remove_2tuple_filter(struct rte_eth_dev *dev,
2951                         struct rte_eth_ntuple_filter *ntuple_filter)
2952 {
2953         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2954         struct e1000_filter_info *filter_info =
2955                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
2956         struct e1000_2tuple_filter_info filter_2tuple;
2957         struct e1000_2tuple_filter *filter;
2958         int ret;
2959
2960         memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info));
2961         ret = ntuple_filter_to_2tuple(ntuple_filter,
2962                                       &filter_2tuple);
2963         if (ret < 0)
2964                 return ret;
2965
2966         filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list,
2967                                          &filter_2tuple);
2968         if (filter == NULL) {
2969                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
2970                 return -ENOENT;
2971         }
2972
2973         filter_info->twotuple_mask &= ~(1 << filter->index);
2974         TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries);
2975         rte_free(filter);
2976
2977         E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK);
2978         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
2979         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
2980         return 0;
2981 }
2982
2983 static inline struct e1000_flex_filter *
2984 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list,
2985                         struct e1000_flex_filter_info *key)
2986 {
2987         struct e1000_flex_filter *it;
2988
2989         TAILQ_FOREACH(it, filter_list, entries) {
2990                 if (memcmp(key, &it->filter_info,
2991                         sizeof(struct e1000_flex_filter_info)) == 0)
2992                         return it;
2993         }
2994
2995         return NULL;
2996 }
2997
2998 static int
2999 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
3000                         struct rte_eth_flex_filter *filter,
3001                         bool add)
3002 {
3003         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3004         struct e1000_filter_info *filter_info =
3005                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3006         struct e1000_flex_filter *flex_filter, *it;
3007         uint32_t wufc, queueing, mask;
3008         uint32_t reg_off;
3009         uint8_t shift, i, j = 0;
3010
3011         flex_filter = rte_zmalloc("e1000_flex_filter",
3012                         sizeof(struct e1000_flex_filter), 0);
3013         if (flex_filter == NULL)
3014                 return -ENOMEM;
3015
3016         flex_filter->filter_info.len = filter->len;
3017         flex_filter->filter_info.priority = filter->priority;
3018         memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len);
3019         for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) {
3020                 mask = 0;
3021                 /* reverse bits in flex filter's mask*/
3022                 for (shift = 0; shift < CHAR_BIT; shift++) {
3023                         if (filter->mask[i] & (0x01 << shift))
3024                                 mask |= (0x80 >> shift);
3025                 }
3026                 flex_filter->filter_info.mask[i] = mask;
3027         }
3028
3029         wufc = E1000_READ_REG(hw, E1000_WUFC);
3030         if (flex_filter->index < E1000_MAX_FHFT)
3031                 reg_off = E1000_FHFT(flex_filter->index);
3032         else
3033                 reg_off = E1000_FHFT_EXT(flex_filter->index - E1000_MAX_FHFT);
3034
3035         if (add) {
3036                 if (eth_igb_flex_filter_lookup(&filter_info->flex_list,
3037                                 &flex_filter->filter_info) != NULL) {
3038                         PMD_DRV_LOG(ERR, "filter exists.");
3039                         rte_free(flex_filter);
3040                         return -EEXIST;
3041                 }
3042                 flex_filter->queue = filter->queue;
3043                 /*
3044                  * look for an unused flex filter index
3045                  * and insert the filter into the list.
3046                  */
3047                 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) {
3048                         if (!(filter_info->flex_mask & (1 << i))) {
3049                                 filter_info->flex_mask |= 1 << i;
3050                                 flex_filter->index = i;
3051                                 TAILQ_INSERT_TAIL(&filter_info->flex_list,
3052                                         flex_filter,
3053                                         entries);
3054                                 break;
3055                         }
3056                 }
3057                 if (i >= E1000_MAX_FLEX_FILTERS) {
3058                         PMD_DRV_LOG(ERR, "flex filters are full.");
3059                         rte_free(flex_filter);
3060                         return -ENOSYS;
3061                 }
3062
3063                 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ |
3064                                 (E1000_WUFC_FLX0 << flex_filter->index));
3065                 queueing = filter->len |
3066                         (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) |
3067                         (filter->priority << E1000_FHFT_QUEUEING_PRIO_SHIFT);
3068                 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET,
3069                                 queueing);
3070                 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) {
3071                         E1000_WRITE_REG(hw, reg_off,
3072                                         flex_filter->filter_info.dwords[j]);
3073                         reg_off += sizeof(uint32_t);
3074                         E1000_WRITE_REG(hw, reg_off,
3075                                         flex_filter->filter_info.dwords[++j]);
3076                         reg_off += sizeof(uint32_t);
3077                         E1000_WRITE_REG(hw, reg_off,
3078                                 (uint32_t)flex_filter->filter_info.mask[i]);
3079                         reg_off += sizeof(uint32_t) * 2;
3080                         ++j;
3081                 }
3082         } else {
3083                 it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
3084                                 &flex_filter->filter_info);
3085                 if (it == NULL) {
3086                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3087                         rte_free(flex_filter);
3088                         return -ENOENT;
3089                 }
3090
3091                 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++)
3092                         E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0);
3093                 E1000_WRITE_REG(hw, E1000_WUFC, wufc &
3094                         (~(E1000_WUFC_FLX0 << it->index)));
3095
3096                 filter_info->flex_mask &= ~(1 << it->index);
3097                 TAILQ_REMOVE(&filter_info->flex_list, it, entries);
3098                 rte_free(it);
3099                 rte_free(flex_filter);
3100         }
3101
3102         return 0;
3103 }
3104
3105 static int
3106 eth_igb_get_flex_filter(struct rte_eth_dev *dev,
3107                         struct rte_eth_flex_filter *filter)
3108 {
3109         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3110         struct e1000_filter_info *filter_info =
3111                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3112         struct e1000_flex_filter flex_filter, *it;
3113         uint32_t wufc, queueing, wufc_en = 0;
3114
3115         memset(&flex_filter, 0, sizeof(struct e1000_flex_filter));
3116         flex_filter.filter_info.len = filter->len;
3117         flex_filter.filter_info.priority = filter->priority;
3118         memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len);
3119         memcpy(flex_filter.filter_info.mask, filter->mask,
3120                         RTE_ALIGN(filter->len, sizeof(char)) / sizeof(char));
3121
3122         it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
3123                                 &flex_filter.filter_info);
3124         if (it == NULL) {
3125                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3126                 return -ENOENT;
3127         }
3128
3129         wufc = E1000_READ_REG(hw, E1000_WUFC);
3130         wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index);
3131
3132         if ((wufc & wufc_en) == wufc_en) {
3133                 uint32_t reg_off = 0;
3134                 if (it->index < E1000_MAX_FHFT)
3135                         reg_off = E1000_FHFT(it->index);
3136                 else
3137                         reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT);
3138
3139                 queueing = E1000_READ_REG(hw,
3140                                 reg_off + E1000_FHFT_QUEUEING_OFFSET);
3141                 filter->len = queueing & E1000_FHFT_QUEUEING_LEN;
3142                 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >>
3143                         E1000_FHFT_QUEUEING_PRIO_SHIFT;
3144                 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >>
3145                         E1000_FHFT_QUEUEING_QUEUE_SHIFT;
3146                 return 0;
3147         }
3148         return -ENOENT;
3149 }
3150
3151 static int
3152 eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
3153                         enum rte_filter_op filter_op,
3154                         void *arg)
3155 {
3156         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3157         struct rte_eth_flex_filter *filter;
3158         int ret = 0;
3159
3160         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
3161
3162         if (filter_op == RTE_ETH_FILTER_NOP)
3163                 return ret;
3164
3165         if (arg == NULL) {
3166                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
3167                             filter_op);
3168                 return -EINVAL;
3169         }
3170
3171         filter = (struct rte_eth_flex_filter *)arg;
3172         if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN
3173             || filter->len % sizeof(uint64_t) != 0) {
3174                 PMD_DRV_LOG(ERR, "filter's length is out of range");
3175                 return -EINVAL;
3176         }
3177         if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) {
3178                 PMD_DRV_LOG(ERR, "filter's priority is out of range");
3179                 return -EINVAL;
3180         }
3181
3182         switch (filter_op) {
3183         case RTE_ETH_FILTER_ADD:
3184                 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE);
3185                 break;
3186         case RTE_ETH_FILTER_DELETE:
3187                 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE);
3188                 break;
3189         case RTE_ETH_FILTER_GET:
3190                 ret = eth_igb_get_flex_filter(dev, filter);
3191                 break;
3192         default:
3193                 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op);
3194                 ret = -EINVAL;
3195                 break;
3196         }
3197
3198         return ret;
3199 }
3200
3201 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/
3202 static inline int
3203 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter,
3204                         struct e1000_5tuple_filter_info *filter_info)
3205 {
3206         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576)
3207                 return -EINVAL;
3208         if (filter->priority > E1000_2TUPLE_MAX_PRI)
3209                 return -EINVAL;  /* filter index is out of range. */
3210         if (filter->tcp_flags > TCP_FLAG_ALL)
3211                 return -EINVAL;  /* flags is invalid. */
3212
3213         switch (filter->dst_ip_mask) {
3214         case UINT32_MAX:
3215                 filter_info->dst_ip_mask = 0;
3216                 filter_info->dst_ip = filter->dst_ip;
3217                 break;
3218         case 0:
3219                 filter_info->dst_ip_mask = 1;
3220                 break;
3221         default:
3222                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
3223                 return -EINVAL;
3224         }
3225
3226         switch (filter->src_ip_mask) {
3227         case UINT32_MAX:
3228                 filter_info->src_ip_mask = 0;
3229                 filter_info->src_ip = filter->src_ip;
3230                 break;
3231         case 0:
3232                 filter_info->src_ip_mask = 1;
3233                 break;
3234         default:
3235                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
3236                 return -EINVAL;
3237         }
3238
3239         switch (filter->dst_port_mask) {
3240         case UINT16_MAX:
3241                 filter_info->dst_port_mask = 0;
3242                 filter_info->dst_port = filter->dst_port;
3243                 break;
3244         case 0:
3245                 filter_info->dst_port_mask = 1;
3246                 break;
3247         default:
3248                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
3249                 return -EINVAL;
3250         }
3251
3252         switch (filter->src_port_mask) {
3253         case UINT16_MAX:
3254                 filter_info->src_port_mask = 0;
3255                 filter_info->src_port = filter->src_port;
3256                 break;
3257         case 0:
3258                 filter_info->src_port_mask = 1;
3259                 break;
3260         default:
3261                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
3262                 return -EINVAL;
3263         }
3264
3265         switch (filter->proto_mask) {
3266         case UINT8_MAX:
3267                 filter_info->proto_mask = 0;
3268                 filter_info->proto = filter->proto;
3269                 break;
3270         case 0:
3271                 filter_info->proto_mask = 1;
3272                 break;
3273         default:
3274                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
3275                 return -EINVAL;
3276         }
3277
3278         filter_info->priority = (uint8_t)filter->priority;
3279         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
3280                 filter_info->tcp_flags = filter->tcp_flags;
3281         else
3282                 filter_info->tcp_flags = 0;
3283
3284         return 0;
3285 }
3286
3287 static inline struct e1000_5tuple_filter *
3288 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list,
3289                         struct e1000_5tuple_filter_info *key)
3290 {
3291         struct e1000_5tuple_filter *it;
3292
3293         TAILQ_FOREACH(it, filter_list, entries) {
3294                 if (memcmp(key, &it->filter_info,
3295                         sizeof(struct e1000_5tuple_filter_info)) == 0) {
3296                         return it;
3297                 }
3298         }
3299         return NULL;
3300 }
3301
3302 /*
3303  * igb_add_5tuple_filter_82576 - add a 5tuple filter
3304  *
3305  * @param
3306  * dev: Pointer to struct rte_eth_dev.
3307  * ntuple_filter: ponter to the filter that will be added.
3308  *
3309  * @return
3310  *    - On success, zero.
3311  *    - On failure, a negative value.
3312  */
3313 static int
3314 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
3315                         struct rte_eth_ntuple_filter *ntuple_filter)
3316 {
3317         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3318         struct e1000_filter_info *filter_info =
3319                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3320         struct e1000_5tuple_filter *filter;
3321         uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK;
3322         uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP;
3323         uint8_t i;
3324         int ret;
3325
3326         filter = rte_zmalloc("e1000_5tuple_filter",
3327                         sizeof(struct e1000_5tuple_filter), 0);
3328         if (filter == NULL)
3329                 return -ENOMEM;
3330
3331         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3332                                             &filter->filter_info);
3333         if (ret < 0) {
3334                 rte_free(filter);
3335                 return ret;
3336         }
3337
3338         if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
3339                                          &filter->filter_info) != NULL) {
3340                 PMD_DRV_LOG(ERR, "filter exists.");
3341                 rte_free(filter);
3342                 return -EEXIST;
3343         }
3344         filter->queue = ntuple_filter->queue;
3345
3346         /*
3347          * look for an unused 5tuple filter index,
3348          * and insert the filter to list.
3349          */
3350         for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) {
3351                 if (!(filter_info->fivetuple_mask & (1 << i))) {
3352                         filter_info->fivetuple_mask |= 1 << i;
3353                         filter->index = i;
3354                         TAILQ_INSERT_TAIL(&filter_info->fivetuple_list,
3355                                           filter,
3356                                           entries);
3357                         break;
3358                 }
3359         }
3360         if (i >= E1000_MAX_FTQF_FILTERS) {
3361                 PMD_DRV_LOG(ERR, "5tuple filters are full.");
3362                 rte_free(filter);
3363                 return -ENOSYS;
3364         }
3365
3366         ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK;
3367         if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */
3368                 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP;
3369         if (filter->filter_info.dst_ip_mask == 0)
3370                 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP;
3371         if (filter->filter_info.src_port_mask == 0)
3372                 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
3373         if (filter->filter_info.proto_mask == 0)
3374                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP;
3375         ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) &
3376                 E1000_FTQF_QUEUE_MASK;
3377         ftqf |= E1000_FTQF_QUEUE_ENABLE;
3378         E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf);
3379         E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip);
3380         E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip);
3381
3382         spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT;
3383         E1000_WRITE_REG(hw, E1000_SPQF(i), spqf);
3384
3385         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
3386         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
3387                 imir |= E1000_IMIR_PORT_BP;
3388         else
3389                 imir &= ~E1000_IMIR_PORT_BP;
3390         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
3391
3392         /* tcp flags bits setting. */
3393         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
3394                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
3395                         imir_ext |= E1000_IMIREXT_CTRL_URG;
3396                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
3397                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
3398                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
3399                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
3400                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
3401                         imir_ext |= E1000_IMIREXT_CTRL_RST;
3402                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
3403                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
3404                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
3405                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
3406         } else
3407                 imir_ext |= E1000_IMIREXT_CTRL_BP;
3408         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
3409         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
3410         return 0;
3411 }
3412
3413 /*
3414  * igb_remove_5tuple_filter_82576 - remove a 5tuple filter
3415  *
3416  * @param
3417  * dev: Pointer to struct rte_eth_dev.
3418  * ntuple_filter: ponter to the filter that will be removed.
3419  *
3420  * @return
3421  *    - On success, zero.
3422  *    - On failure, a negative value.
3423  */
3424 static int
3425 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
3426                                 struct rte_eth_ntuple_filter *ntuple_filter)
3427 {
3428         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3429         struct e1000_filter_info *filter_info =
3430                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3431         struct e1000_5tuple_filter_info filter_5tuple;
3432         struct e1000_5tuple_filter *filter;
3433         int ret;
3434
3435         memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info));
3436         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3437                                             &filter_5tuple);
3438         if (ret < 0)
3439                 return ret;
3440
3441         filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
3442                                          &filter_5tuple);
3443         if (filter == NULL) {
3444                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3445                 return -ENOENT;
3446         }
3447
3448         filter_info->fivetuple_mask &= ~(1 << filter->index);
3449         TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries);
3450         rte_free(filter);
3451
3452         E1000_WRITE_REG(hw, E1000_FTQF(filter->index),
3453                         E1000_FTQF_VF_BP | E1000_FTQF_MASK);
3454         E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0);
3455         E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0);
3456         E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0);
3457         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
3458         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
3459         return 0;
3460 }
3461
3462 static int
3463 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
3464 {
3465         uint32_t rctl;
3466         struct e1000_hw *hw;
3467         struct rte_eth_dev_info dev_info;
3468         uint32_t frame_size = mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN +
3469                                      VLAN_TAG_SIZE);
3470
3471         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3472
3473 #ifdef RTE_LIBRTE_82571_SUPPORT
3474         /* XXX: not bigger than max_rx_pktlen */
3475         if (hw->mac.type == e1000_82571)
3476                 return -ENOTSUP;
3477 #endif
3478         eth_igb_infos_get(dev, &dev_info);
3479
3480         /* check that mtu is within the allowed range */
3481         if ((mtu < ETHER_MIN_MTU) ||
3482             (frame_size > dev_info.max_rx_pktlen))
3483                 return -EINVAL;
3484
3485         /* refuse mtu that requires the support of scattered packets when this
3486          * feature has not been enabled before. */
3487         if (!dev->data->scattered_rx &&
3488             frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)
3489                 return -EINVAL;
3490
3491         rctl = E1000_READ_REG(hw, E1000_RCTL);
3492
3493         /* switch to jumbo mode if needed */
3494         if (frame_size > ETHER_MAX_LEN) {
3495                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
3496                 rctl |= E1000_RCTL_LPE;
3497         } else {
3498                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
3499                 rctl &= ~E1000_RCTL_LPE;
3500         }
3501         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3502
3503         /* update max frame size */
3504         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
3505
3506         E1000_WRITE_REG(hw, E1000_RLPML,
3507                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
3508
3509         return 0;
3510 }
3511
3512 /*
3513  * igb_add_del_ntuple_filter - add or delete a ntuple filter
3514  *
3515  * @param
3516  * dev: Pointer to struct rte_eth_dev.
3517  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
3518  * add: if true, add filter, if false, remove filter
3519  *
3520  * @return
3521  *    - On success, zero.
3522  *    - On failure, a negative value.
3523  */
3524 static int
3525 igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
3526                         struct rte_eth_ntuple_filter *ntuple_filter,
3527                         bool add)
3528 {
3529         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3530         int ret;
3531
3532         switch (ntuple_filter->flags) {
3533         case RTE_5TUPLE_FLAGS:
3534         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3535                 if (hw->mac.type != e1000_82576)
3536                         return -ENOTSUP;
3537                 if (add)
3538                         ret = igb_add_5tuple_filter_82576(dev,
3539                                                           ntuple_filter);
3540                 else
3541                         ret = igb_remove_5tuple_filter_82576(dev,
3542                                                              ntuple_filter);
3543                 break;
3544         case RTE_2TUPLE_FLAGS:
3545         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3546                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
3547                         return -ENOTSUP;
3548                 if (add)
3549                         ret = igb_add_2tuple_filter(dev, ntuple_filter);
3550                 else
3551                         ret = igb_remove_2tuple_filter(dev, ntuple_filter);
3552                 break;
3553         default:
3554                 ret = -EINVAL;
3555                 break;
3556         }
3557
3558         return ret;
3559 }
3560
3561 /*
3562  * igb_get_ntuple_filter - get a ntuple filter
3563  *
3564  * @param
3565  * dev: Pointer to struct rte_eth_dev.
3566  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
3567  *
3568  * @return
3569  *    - On success, zero.
3570  *    - On failure, a negative value.
3571  */
3572 static int
3573 igb_get_ntuple_filter(struct rte_eth_dev *dev,
3574                         struct rte_eth_ntuple_filter *ntuple_filter)
3575 {
3576         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3577         struct e1000_filter_info *filter_info =
3578                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3579         struct e1000_5tuple_filter_info filter_5tuple;
3580         struct e1000_2tuple_filter_info filter_2tuple;
3581         struct e1000_5tuple_filter *p_5tuple_filter;
3582         struct e1000_2tuple_filter *p_2tuple_filter;
3583         int ret;
3584
3585         switch (ntuple_filter->flags) {
3586         case RTE_5TUPLE_FLAGS:
3587         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3588                 if (hw->mac.type != e1000_82576)
3589                         return -ENOTSUP;
3590                 memset(&filter_5tuple,
3591                         0,
3592                         sizeof(struct e1000_5tuple_filter_info));
3593                 ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3594                                                     &filter_5tuple);
3595                 if (ret < 0)
3596                         return ret;
3597                 p_5tuple_filter = igb_5tuple_filter_lookup_82576(
3598                                         &filter_info->fivetuple_list,
3599                                         &filter_5tuple);
3600                 if (p_5tuple_filter == NULL) {
3601                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3602                         return -ENOENT;
3603                 }
3604                 ntuple_filter->queue = p_5tuple_filter->queue;
3605                 break;
3606         case RTE_2TUPLE_FLAGS:
3607         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3608                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
3609                         return -ENOTSUP;
3610                 memset(&filter_2tuple,
3611                         0,
3612                         sizeof(struct e1000_2tuple_filter_info));
3613                 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple);
3614                 if (ret < 0)
3615                         return ret;
3616                 p_2tuple_filter = igb_2tuple_filter_lookup(
3617                                         &filter_info->twotuple_list,
3618                                         &filter_2tuple);
3619                 if (p_2tuple_filter == NULL) {
3620                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3621                         return -ENOENT;
3622                 }
3623                 ntuple_filter->queue = p_2tuple_filter->queue;
3624                 break;
3625         default:
3626                 ret = -EINVAL;
3627                 break;
3628         }
3629
3630         return 0;
3631 }
3632
3633 /*
3634  * igb_ntuple_filter_handle - Handle operations for ntuple filter.
3635  * @dev: pointer to rte_eth_dev structure
3636  * @filter_op:operation will be taken.
3637  * @arg: a pointer to specific structure corresponding to the filter_op
3638  */
3639 static int
3640 igb_ntuple_filter_handle(struct rte_eth_dev *dev,
3641                                 enum rte_filter_op filter_op,
3642                                 void *arg)
3643 {
3644         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3645         int ret;
3646
3647         MAC_TYPE_FILTER_SUP(hw->mac.type);
3648
3649         if (filter_op == RTE_ETH_FILTER_NOP)
3650                 return 0;
3651
3652         if (arg == NULL) {
3653                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
3654                             filter_op);
3655                 return -EINVAL;
3656         }
3657
3658         switch (filter_op) {
3659         case RTE_ETH_FILTER_ADD:
3660                 ret = igb_add_del_ntuple_filter(dev,
3661                         (struct rte_eth_ntuple_filter *)arg,
3662                         TRUE);
3663                 break;
3664         case RTE_ETH_FILTER_DELETE:
3665                 ret = igb_add_del_ntuple_filter(dev,
3666                         (struct rte_eth_ntuple_filter *)arg,
3667                         FALSE);
3668                 break;
3669         case RTE_ETH_FILTER_GET:
3670                 ret = igb_get_ntuple_filter(dev,
3671                         (struct rte_eth_ntuple_filter *)arg);
3672                 break;
3673         default:
3674                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
3675                 ret = -EINVAL;
3676                 break;
3677         }
3678         return ret;
3679 }
3680
3681 static inline int
3682 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info,
3683                         uint16_t ethertype)
3684 {
3685         int i;
3686
3687         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
3688                 if (filter_info->ethertype_filters[i] == ethertype &&
3689                     (filter_info->ethertype_mask & (1 << i)))
3690                         return i;
3691         }
3692         return -1;
3693 }
3694
3695 static inline int
3696 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info,
3697                         uint16_t ethertype)
3698 {
3699         int i;
3700
3701         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
3702                 if (!(filter_info->ethertype_mask & (1 << i))) {
3703                         filter_info->ethertype_mask |= 1 << i;
3704                         filter_info->ethertype_filters[i] = ethertype;
3705                         return i;
3706                 }
3707         }
3708         return -1;
3709 }
3710
3711 static inline int
3712 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info,
3713                         uint8_t idx)
3714 {
3715         if (idx >= E1000_MAX_ETQF_FILTERS)
3716                 return -1;
3717         filter_info->ethertype_mask &= ~(1 << idx);
3718         filter_info->ethertype_filters[idx] = 0;
3719         return idx;
3720 }
3721
3722
3723 static int
3724 igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
3725                         struct rte_eth_ethertype_filter *filter,
3726                         bool add)
3727 {
3728         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3729         struct e1000_filter_info *filter_info =
3730                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3731         uint32_t etqf = 0;
3732         int ret;
3733
3734         if (filter->ether_type == ETHER_TYPE_IPv4 ||
3735                 filter->ether_type == ETHER_TYPE_IPv6) {
3736                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
3737                         " ethertype filter.", filter->ether_type);
3738                 return -EINVAL;
3739         }
3740
3741         if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) {
3742                 PMD_DRV_LOG(ERR, "mac compare is unsupported.");
3743                 return -EINVAL;
3744         }
3745         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) {
3746                 PMD_DRV_LOG(ERR, "drop option is unsupported.");
3747                 return -EINVAL;
3748         }
3749
3750         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
3751         if (ret >= 0 && add) {
3752                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.",
3753                             filter->ether_type);
3754                 return -EEXIST;
3755         }
3756         if (ret < 0 && !add) {
3757                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
3758                             filter->ether_type);
3759                 return -ENOENT;
3760         }
3761
3762         if (add) {
3763                 ret = igb_ethertype_filter_insert(filter_info,
3764                         filter->ether_type);
3765                 if (ret < 0) {
3766                         PMD_DRV_LOG(ERR, "ethertype filters are full.");
3767                         return -ENOSYS;
3768                 }
3769
3770                 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE;
3771                 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE);
3772                 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT;
3773         } else {
3774                 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret);
3775                 if (ret < 0)
3776                         return -ENOSYS;
3777         }
3778         E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf);
3779         E1000_WRITE_FLUSH(hw);
3780
3781         return 0;
3782 }
3783
3784 static int
3785 igb_get_ethertype_filter(struct rte_eth_dev *dev,
3786                         struct rte_eth_ethertype_filter *filter)
3787 {
3788         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3789         struct e1000_filter_info *filter_info =
3790                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3791         uint32_t etqf;
3792         int ret;
3793
3794         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
3795         if (ret < 0) {
3796                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
3797                             filter->ether_type);
3798                 return -ENOENT;
3799         }
3800
3801         etqf = E1000_READ_REG(hw, E1000_ETQF(ret));
3802         if (etqf & E1000_ETQF_FILTER_ENABLE) {
3803                 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE;
3804                 filter->flags = 0;
3805                 filter->queue = (etqf & E1000_ETQF_QUEUE) >>
3806                                 E1000_ETQF_QUEUE_SHIFT;
3807                 return 0;
3808         }
3809
3810         return -ENOENT;
3811 }
3812
3813 /*
3814  * igb_ethertype_filter_handle - Handle operations for ethertype filter.
3815  * @dev: pointer to rte_eth_dev structure
3816  * @filter_op:operation will be taken.
3817  * @arg: a pointer to specific structure corresponding to the filter_op
3818  */
3819 static int
3820 igb_ethertype_filter_handle(struct rte_eth_dev *dev,
3821                                 enum rte_filter_op filter_op,
3822                                 void *arg)
3823 {
3824         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3825         int ret;
3826
3827         MAC_TYPE_FILTER_SUP(hw->mac.type);
3828
3829         if (filter_op == RTE_ETH_FILTER_NOP)
3830                 return 0;
3831
3832         if (arg == NULL) {
3833                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
3834                             filter_op);
3835                 return -EINVAL;
3836         }
3837
3838         switch (filter_op) {
3839         case RTE_ETH_FILTER_ADD:
3840                 ret = igb_add_del_ethertype_filter(dev,
3841                         (struct rte_eth_ethertype_filter *)arg,
3842                         TRUE);
3843                 break;
3844         case RTE_ETH_FILTER_DELETE:
3845                 ret = igb_add_del_ethertype_filter(dev,
3846                         (struct rte_eth_ethertype_filter *)arg,
3847                         FALSE);
3848                 break;
3849         case RTE_ETH_FILTER_GET:
3850                 ret = igb_get_ethertype_filter(dev,
3851                         (struct rte_eth_ethertype_filter *)arg);
3852                 break;
3853         default:
3854                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
3855                 ret = -EINVAL;
3856                 break;
3857         }
3858         return ret;
3859 }
3860
3861 static int
3862 eth_igb_filter_ctrl(struct rte_eth_dev *dev,
3863                      enum rte_filter_type filter_type,
3864                      enum rte_filter_op filter_op,
3865                      void *arg)
3866 {
3867         int ret = -EINVAL;
3868
3869         switch (filter_type) {
3870         case RTE_ETH_FILTER_NTUPLE:
3871                 ret = igb_ntuple_filter_handle(dev, filter_op, arg);
3872                 break;
3873         case RTE_ETH_FILTER_ETHERTYPE:
3874                 ret = igb_ethertype_filter_handle(dev, filter_op, arg);
3875                 break;
3876         case RTE_ETH_FILTER_SYN:
3877                 ret = eth_igb_syn_filter_handle(dev, filter_op, arg);
3878                 break;
3879         case RTE_ETH_FILTER_FLEXIBLE:
3880                 ret = eth_igb_flex_filter_handle(dev, filter_op, arg);
3881                 break;
3882         default:
3883                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
3884                                                         filter_type);
3885                 break;
3886         }
3887
3888         return ret;
3889 }
3890
3891 static int
3892 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
3893                          struct ether_addr *mc_addr_set,
3894                          uint32_t nb_mc_addr)
3895 {
3896         struct e1000_hw *hw;
3897
3898         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3899         e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr);
3900         return 0;
3901 }
3902
3903 static int
3904 igb_timesync_enable(struct rte_eth_dev *dev)
3905 {
3906         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3907         uint32_t tsync_ctl;
3908         uint32_t tsauxc;
3909
3910         /* Enable system time for it isn't on by default. */
3911         tsauxc = E1000_READ_REG(hw, E1000_TSAUXC);
3912         tsauxc &= ~E1000_TSAUXC_DISABLE_SYSTIME;
3913         E1000_WRITE_REG(hw, E1000_TSAUXC, tsauxc);
3914
3915         /* Start incrementing the register used to timestamp PTP packets. */
3916         E1000_WRITE_REG(hw, E1000_TIMINCA, E1000_TIMINCA_INIT);
3917
3918         /* Enable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
3919         E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588),
3920                         (ETHER_TYPE_1588 |
3921                          E1000_ETQF_FILTER_ENABLE |
3922                          E1000_ETQF_1588));
3923
3924         /* Enable timestamping of received PTP packets. */
3925         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
3926         tsync_ctl |= E1000_TSYNCRXCTL_ENABLED;
3927         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl);
3928
3929         /* Enable Timestamping of transmitted PTP packets. */
3930         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
3931         tsync_ctl |= E1000_TSYNCTXCTL_ENABLED;
3932         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl);
3933
3934         return 0;
3935 }
3936
3937 static int
3938 igb_timesync_disable(struct rte_eth_dev *dev)
3939 {
3940         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3941         uint32_t tsync_ctl;
3942
3943         /* Disable timestamping of transmitted PTP packets. */
3944         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
3945         tsync_ctl &= ~E1000_TSYNCTXCTL_ENABLED;
3946         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, tsync_ctl);
3947
3948         /* Disable timestamping of received PTP packets. */
3949         tsync_ctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
3950         tsync_ctl &= ~E1000_TSYNCRXCTL_ENABLED;
3951         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, tsync_ctl);
3952
3953         /* Disable L2 filtering of IEEE1588/802.1AS Ethernet frame types. */
3954         E1000_WRITE_REG(hw, E1000_ETQF(E1000_ETQF_FILTER_1588), 0);
3955
3956         /* Stop incrementating the System Time registers. */
3957         E1000_WRITE_REG(hw, E1000_TIMINCA, 0);
3958
3959         return 0;
3960 }
3961
3962 static int
3963 igb_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
3964                                struct timespec *timestamp,
3965                                uint32_t flags __rte_unused)
3966 {
3967         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3968         uint32_t tsync_rxctl;
3969         uint32_t rx_stmpl;
3970         uint32_t rx_stmph;
3971
3972         tsync_rxctl = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
3973         if ((tsync_rxctl & E1000_TSYNCRXCTL_VALID) == 0)
3974                 return -EINVAL;
3975
3976         rx_stmpl = E1000_READ_REG(hw, E1000_RXSTMPL);
3977         rx_stmph = E1000_READ_REG(hw, E1000_RXSTMPH);
3978
3979         timestamp->tv_sec = (uint64_t)(((uint64_t)rx_stmph << 32) | rx_stmpl);
3980         timestamp->tv_nsec = 0;
3981
3982         return  0;
3983 }
3984
3985 static int
3986 igb_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
3987                                struct timespec *timestamp)
3988 {
3989         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3990         uint32_t tsync_txctl;
3991         uint32_t tx_stmpl;
3992         uint32_t tx_stmph;
3993
3994         tsync_txctl = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
3995         if ((tsync_txctl & E1000_TSYNCTXCTL_VALID) == 0)
3996                 return -EINVAL;
3997
3998         tx_stmpl = E1000_READ_REG(hw, E1000_TXSTMPL);
3999         tx_stmph = E1000_READ_REG(hw, E1000_TXSTMPH);
4000
4001         timestamp->tv_sec = (uint64_t)(((uint64_t)tx_stmph << 32) | tx_stmpl);
4002         timestamp->tv_nsec = 0;
4003
4004         return  0;
4005 }
4006
4007 static int
4008 eth_igb_get_reg_length(struct rte_eth_dev *dev __rte_unused)
4009 {
4010         int count = 0;
4011         int g_ind = 0;
4012         const struct reg_info *reg_group;
4013
4014         while ((reg_group = igb_regs[g_ind++]))
4015                 count += igb_reg_group_count(reg_group);
4016
4017         return count;
4018 }
4019
4020 static int
4021 igbvf_get_reg_length(struct rte_eth_dev *dev __rte_unused)
4022 {
4023         int count = 0;
4024         int g_ind = 0;
4025         const struct reg_info *reg_group;
4026
4027         while ((reg_group = igbvf_regs[g_ind++]))
4028                 count += igb_reg_group_count(reg_group);
4029
4030         return count;
4031 }
4032
4033 static int
4034 eth_igb_get_regs(struct rte_eth_dev *dev,
4035         struct rte_dev_reg_info *regs)
4036 {
4037         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4038         uint32_t *data = regs->data;
4039         int g_ind = 0;
4040         int count = 0;
4041         const struct reg_info *reg_group;
4042
4043         /* Support only full register dump */
4044         if ((regs->length == 0) ||
4045             (regs->length == (uint32_t)eth_igb_get_reg_length(dev))) {
4046                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4047                         hw->device_id;
4048                 while ((reg_group = igb_regs[g_ind++]))
4049                         count += igb_read_regs_group(dev, &data[count],
4050                                                         reg_group);
4051                 return 0;
4052         }
4053
4054         return -ENOTSUP;
4055 }
4056
4057 static int
4058 igbvf_get_regs(struct rte_eth_dev *dev,
4059         struct rte_dev_reg_info *regs)
4060 {
4061         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4062         uint32_t *data = regs->data;
4063         int g_ind = 0;
4064         int count = 0;
4065         const struct reg_info *reg_group;
4066
4067         /* Support only full register dump */
4068         if ((regs->length == 0) ||
4069             (regs->length == (uint32_t)igbvf_get_reg_length(dev))) {
4070                 regs->version = hw->mac.type << 24 | hw->revision_id << 16 |
4071                         hw->device_id;
4072                 while ((reg_group = igbvf_regs[g_ind++]))
4073                         count += igb_read_regs_group(dev, &data[count],
4074                                                         reg_group);
4075                 return 0;
4076         }
4077
4078         return -ENOTSUP;
4079 }
4080
4081 static int
4082 eth_igb_get_eeprom_length(struct rte_eth_dev *dev)
4083 {
4084         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4085
4086         /* Return unit is byte count */
4087         return hw->nvm.word_size * 2;
4088 }
4089
4090 static int
4091 eth_igb_get_eeprom(struct rte_eth_dev *dev,
4092         struct rte_dev_eeprom_info *in_eeprom)
4093 {
4094         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4095         struct e1000_nvm_info *nvm = &hw->nvm;
4096         uint16_t *data = in_eeprom->data;
4097         int first, length;
4098
4099         first = in_eeprom->offset >> 1;
4100         length = in_eeprom->length >> 1;
4101         if ((first >= hw->nvm.word_size) ||
4102             ((first + length) >= hw->nvm.word_size))
4103                 return -EINVAL;
4104
4105         in_eeprom->magic = hw->vendor_id |
4106                 ((uint32_t)hw->device_id << 16);
4107
4108         if ((nvm->ops.read) == NULL)
4109                 return -ENOTSUP;
4110
4111         return nvm->ops.read(hw, first, length, data);
4112 }
4113
4114 static int
4115 eth_igb_set_eeprom(struct rte_eth_dev *dev,
4116         struct rte_dev_eeprom_info *in_eeprom)
4117 {
4118         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4119         struct e1000_nvm_info *nvm = &hw->nvm;
4120         uint16_t *data = in_eeprom->data;
4121         int first, length;
4122
4123         first = in_eeprom->offset >> 1;
4124         length = in_eeprom->length >> 1;
4125         if ((first >= hw->nvm.word_size) ||
4126             ((first + length) >= hw->nvm.word_size))
4127                 return -EINVAL;
4128
4129         in_eeprom->magic = (uint32_t)hw->vendor_id |
4130                 ((uint32_t)hw->device_id << 16);
4131
4132         if ((nvm->ops.write) == NULL)
4133                 return -ENOTSUP;
4134         return nvm->ops.write(hw,  first, length, data);
4135 }
4136
4137 static struct rte_driver pmd_igb_drv = {
4138         .type = PMD_PDEV,
4139         .init = rte_igb_pmd_init,
4140 };
4141
4142 static struct rte_driver pmd_igbvf_drv = {
4143         .type = PMD_PDEV,
4144         .init = rte_igbvf_pmd_init,
4145 };
4146
4147 #ifdef RTE_NEXT_ABI
4148 static int
4149 eth_igb_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
4150 {
4151         struct e1000_hw *hw =
4152                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4153         uint32_t mask = 1 << queue_id;
4154
4155         E1000_WRITE_REG(hw, E1000_EIMC, mask);
4156         E1000_WRITE_FLUSH(hw);
4157
4158         return 0;
4159 }
4160
4161 static int
4162 eth_igb_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
4163 {
4164         struct e1000_hw *hw =
4165                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4166         uint32_t mask = 1 << queue_id;
4167         uint32_t regval;
4168
4169         regval = E1000_READ_REG(hw, E1000_EIMS);
4170         E1000_WRITE_REG(hw, E1000_EIMS, regval | mask);
4171         E1000_WRITE_FLUSH(hw);
4172
4173         rte_intr_enable(&dev->pci_dev->intr_handle);
4174
4175         return 0;
4176 }
4177
4178 static void
4179 eth_igb_write_ivar(struct e1000_hw *hw, uint8_t  msix_vector,
4180                    uint8_t index, uint8_t offset)
4181 {
4182         uint32_t val = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
4183
4184         /* clear bits */
4185         val &= ~((uint32_t)0xFF << offset);
4186
4187         /* write vector and valid bit */
4188         val |= (msix_vector | E1000_IVAR_VALID) << offset;
4189
4190         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, val);
4191 }
4192
4193 static void
4194 eth_igb_assign_msix_vector(struct e1000_hw *hw, int8_t direction,
4195                            uint8_t queue, uint8_t msix_vector)
4196 {
4197         uint32_t tmp = 0;
4198
4199         if (hw->mac.type == e1000_82575) {
4200                 if (direction == 0)
4201                         tmp = E1000_EICR_RX_QUEUE0 << queue;
4202                 else if (direction == 1)
4203                         tmp = E1000_EICR_TX_QUEUE0 << queue;
4204                 E1000_WRITE_REG(hw, E1000_MSIXBM(msix_vector), tmp);
4205         } else if (hw->mac.type == e1000_82576) {
4206                 if ((direction == 0) || (direction == 1))
4207                         eth_igb_write_ivar(hw, msix_vector, queue & 0x7,
4208                                            ((queue & 0x8) << 1) +
4209                                            8 * direction);
4210         } else if ((hw->mac.type == e1000_82580) ||
4211                         (hw->mac.type == e1000_i350) ||
4212                         (hw->mac.type == e1000_i354) ||
4213                         (hw->mac.type == e1000_i210) ||
4214                         (hw->mac.type == e1000_i211)) {
4215                 if ((direction == 0) || (direction == 1))
4216                         eth_igb_write_ivar(hw, msix_vector,
4217                                            queue >> 1,
4218                                            ((queue & 0x1) << 4) +
4219                                            8 * direction);
4220         }
4221 }
4222 #endif
4223
4224 /* Sets up the hardware to generate MSI-X interrupts properly
4225  * @hw
4226  *  board private structure
4227  */
4228 static void
4229 eth_igb_configure_msix_intr(struct rte_eth_dev *dev)
4230 {
4231 #ifdef RTE_NEXT_ABI
4232         int queue_id;
4233         uint32_t tmpval, regval, intr_mask;
4234         struct e1000_hw *hw =
4235                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4236         uint32_t vec = 0;
4237 #endif
4238         struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
4239
4240         /* won't configure msix register if no mapping is done
4241          * between intr vector and event fd
4242          */
4243         if (!rte_intr_dp_is_en(intr_handle))
4244                 return;
4245
4246 #ifdef RTE_NEXT_ABI
4247         /* set interrupt vector for other causes */
4248         if (hw->mac.type == e1000_82575) {
4249                 tmpval = E1000_READ_REG(hw, E1000_CTRL_EXT);
4250                 /* enable MSI-X PBA support */
4251                 tmpval |= E1000_CTRL_EXT_PBA_CLR;
4252
4253                 /* Auto-Mask interrupts upon ICR read */
4254                 tmpval |= E1000_CTRL_EXT_EIAME;
4255                 tmpval |= E1000_CTRL_EXT_IRCA;
4256
4257                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmpval);
4258
4259                 /* enable msix_other interrupt */
4260                 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 0, E1000_EIMS_OTHER);
4261                 regval = E1000_READ_REG(hw, E1000_EIAC);
4262                 E1000_WRITE_REG(hw, E1000_EIAC, regval | E1000_EIMS_OTHER);
4263                 regval = E1000_READ_REG(hw, E1000_EIAM);
4264                 E1000_WRITE_REG(hw, E1000_EIMS, regval | E1000_EIMS_OTHER);
4265         } else if ((hw->mac.type == e1000_82576) ||
4266                         (hw->mac.type == e1000_82580) ||
4267                         (hw->mac.type == e1000_i350) ||
4268                         (hw->mac.type == e1000_i354) ||
4269                         (hw->mac.type == e1000_i210) ||
4270                         (hw->mac.type == e1000_i211)) {
4271                 /* turn on MSI-X capability first */
4272                 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE |
4273                                         E1000_GPIE_PBA | E1000_GPIE_EIAME |
4274                                         E1000_GPIE_NSICR);
4275
4276                 intr_mask = (1 << intr_handle->max_intr) - 1;
4277                 regval = E1000_READ_REG(hw, E1000_EIAC);
4278                 E1000_WRITE_REG(hw, E1000_EIAC, regval | intr_mask);
4279
4280                 /* enable msix_other interrupt */
4281                 regval = E1000_READ_REG(hw, E1000_EIMS);
4282                 E1000_WRITE_REG(hw, E1000_EIMS, regval | intr_mask);
4283                 tmpval = (dev->data->nb_rx_queues | E1000_IVAR_VALID) << 8;
4284                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmpval);
4285         }
4286
4287         /* use EIAM to auto-mask when MSI-X interrupt
4288          * is asserted, this saves a register write for every interrupt
4289          */
4290         intr_mask = (1 << intr_handle->nb_efd) - 1;
4291         regval = E1000_READ_REG(hw, E1000_EIAM);
4292         E1000_WRITE_REG(hw, E1000_EIAM, regval | intr_mask);
4293
4294         for (queue_id = 0; queue_id < dev->data->nb_rx_queues; queue_id++) {
4295                 eth_igb_assign_msix_vector(hw, 0, queue_id, vec);
4296                 intr_handle->intr_vec[queue_id] = vec;
4297                 if (vec < intr_handle->nb_efd - 1)
4298                         vec++;
4299         }
4300
4301         E1000_WRITE_FLUSH(hw);
4302 #endif
4303 }
4304
4305 PMD_REGISTER_DRIVER(pmd_igb_drv);
4306 PMD_REGISTER_DRIVER(pmd_igbvf_drv);