drivers/net/e1000/igb_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <stdarg.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_interrupts.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_log.h>
  44 #include <rte_debug.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_memory.h>
  49 #include <rte_memzone.h>
  50 #include <rte_eal.h>
  51 #include <rte_atomic.h>
  52 #include <rte_malloc.h>
  53 #include <rte_dev.h>
  54
  55 #include "e1000_logs.h"
  56 #include "base/e1000_api.h"
  57 #include "e1000_ethdev.h"
  58
  59 /*
  60  * Default values for port configuration
  61  */
  62 #define IGB_DEFAULT_RX_FREE_THRESH  32
  63 #define IGB_DEFAULT_RX_PTHRESH      8
  64 #define IGB_DEFAULT_RX_HTHRESH      8
  65 #define IGB_DEFAULT_RX_WTHRESH      0
  66
  67 #define IGB_DEFAULT_TX_PTHRESH      32
  68 #define IGB_DEFAULT_TX_HTHRESH      0
  69 #define IGB_DEFAULT_TX_WTHRESH      0
  70
  71 #define IGB_HKEY_MAX_INDEX 10
  72
  73 /* Bit shift and mask */
  74 #define IGB_4_BIT_WIDTH  (CHAR_BIT / 2)
  75 #define IGB_4_BIT_MASK   RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t)
  76 #define IGB_8_BIT_WIDTH  CHAR_BIT
  77 #define IGB_8_BIT_MASK   UINT8_MAX
  78
  79 static int  eth_igb_configure(struct rte_eth_dev *dev);
  80 static int  eth_igb_start(struct rte_eth_dev *dev);
  81 static void eth_igb_stop(struct rte_eth_dev *dev);
  82 static void eth_igb_close(struct rte_eth_dev *dev);
  83 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev);
  84 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev);
  85 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev);
  86 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev);
  87 static int  eth_igb_link_update(struct rte_eth_dev *dev,
  88                                 int wait_to_complete);
  89 static void eth_igb_stats_get(struct rte_eth_dev *dev,
  90                                 struct rte_eth_stats *rte_stats);
  91 static void eth_igb_stats_reset(struct rte_eth_dev *dev);
  92 static void eth_igb_infos_get(struct rte_eth_dev *dev,
  93                               struct rte_eth_dev_info *dev_info);
  94 static void eth_igbvf_infos_get(struct rte_eth_dev *dev,
  95                                 struct rte_eth_dev_info *dev_info);
  96 static int  eth_igb_flow_ctrl_get(struct rte_eth_dev *dev,
  97                                 struct rte_eth_fc_conf *fc_conf);
  98 static int  eth_igb_flow_ctrl_set(struct rte_eth_dev *dev,
  99                                 struct rte_eth_fc_conf *fc_conf);
 100 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev);
 101 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev);
 102 static int eth_igb_interrupt_action(struct rte_eth_dev *dev);
 103 static void eth_igb_interrupt_handler(struct rte_intr_handle *handle,
 104                                                         void *param);
 105 static int  igb_hardware_init(struct e1000_hw *hw);
 106 static void igb_hw_control_acquire(struct e1000_hw *hw);
 107 static void igb_hw_control_release(struct e1000_hw *hw);
 108 static void igb_init_manageability(struct e1000_hw *hw);
 109 static void igb_release_manageability(struct e1000_hw *hw);
 110
 111 static int  eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 112
 113 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev,
 114                 uint16_t vlan_id, int on);
 115 static void eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 116 static void eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 117
 118 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 119 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 120 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev);
 121 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev);
 122 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 123 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 124
 125 static int eth_igb_led_on(struct rte_eth_dev *dev);
 126 static int eth_igb_led_off(struct rte_eth_dev *dev);
 127
 128 static void igb_intr_disable(struct e1000_hw *hw);
 129 static int  igb_get_rx_buffer_size(struct e1000_hw *hw);
 130 static void eth_igb_rar_set(struct rte_eth_dev *dev,
 131                 struct ether_addr *mac_addr,
 132                 uint32_t index, uint32_t pool);
 133 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 134
 135 static void igbvf_intr_disable(struct e1000_hw *hw);
 136 static int igbvf_dev_configure(struct rte_eth_dev *dev);
 137 static int igbvf_dev_start(struct rte_eth_dev *dev);
 138 static void igbvf_dev_stop(struct rte_eth_dev *dev);
 139 static void igbvf_dev_close(struct rte_eth_dev *dev);
 140 static int eth_igbvf_link_update(struct e1000_hw *hw);
 141 static void eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats);
 142 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev);
 143 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev,
 144                 uint16_t vlan_id, int on);
 145 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on);
 146 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 147 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev,
 148                                    struct rte_eth_rss_reta_entry64 *reta_conf,
 149                                    uint16_t reta_size);
 150 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev,
 151                                   struct rte_eth_rss_reta_entry64 *reta_conf,
 152                                   uint16_t reta_size);
 153
 154 static int eth_igb_syn_filter_set(struct rte_eth_dev *dev,
 155                         struct rte_eth_syn_filter *filter,
 156                         bool add);
 157 static int eth_igb_syn_filter_get(struct rte_eth_dev *dev,
 158                         struct rte_eth_syn_filter *filter);
 159 static int eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
 160                         enum rte_filter_op filter_op,
 161                         void *arg);
 162 static int igb_add_2tuple_filter(struct rte_eth_dev *dev,
 163                         struct rte_eth_ntuple_filter *ntuple_filter);
 164 static int igb_remove_2tuple_filter(struct rte_eth_dev *dev,
 165                         struct rte_eth_ntuple_filter *ntuple_filter);
 166 static int eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
 167                         struct rte_eth_flex_filter *filter,
 168                         bool add);
 169 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev,
 170                         struct rte_eth_flex_filter *filter);
 171 static int eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
 172                         enum rte_filter_op filter_op,
 173                         void *arg);
 174 static int igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
 175                         struct rte_eth_ntuple_filter *ntuple_filter);
 176 static int igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
 177                         struct rte_eth_ntuple_filter *ntuple_filter);
 178 static int igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
 179                         struct rte_eth_ntuple_filter *filter,
 180                         bool add);
 181 static int igb_get_ntuple_filter(struct rte_eth_dev *dev,
 182                         struct rte_eth_ntuple_filter *filter);
 183 static int igb_ntuple_filter_handle(struct rte_eth_dev *dev,
 184                                 enum rte_filter_op filter_op,
 185                                 void *arg);
 186 static int igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
 187                         struct rte_eth_ethertype_filter *filter,
 188                         bool add);
 189 static int igb_ethertype_filter_handle(struct rte_eth_dev *dev,
 190                                 enum rte_filter_op filter_op,
 191                                 void *arg);
 192 static int igb_get_ethertype_filter(struct rte_eth_dev *dev,
 193                         struct rte_eth_ethertype_filter *filter);
 194 static int eth_igb_filter_ctrl(struct rte_eth_dev *dev,
 195                      enum rte_filter_type filter_type,
 196                      enum rte_filter_op filter_op,
 197                      void *arg);
 198
 199 static int eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
 200                                     struct ether_addr *mc_addr_set,
 201                                     uint32_t nb_mc_addr);
 202
 203 /*
 204  * Define VF Stats MACRO for Non "cleared on read" register
 205  */
 206 #define UPDATE_VF_STAT(reg, last, cur)            \
 207 {                                                 \
 208         u32 latest = E1000_READ_REG(hw, reg);     \
 209         cur += latest - last;                     \
 210         last = latest;                            \
 211 }
 212
 213
 214 #define IGB_FC_PAUSE_TIME 0x0680
 215 #define IGB_LINK_UPDATE_CHECK_TIMEOUT  90  /* 9s */
 216 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */
 217
 218 #define IGBVF_PMD_NAME "rte_igbvf_pmd"     /* PMD name */
 219
 220 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full;
 221
 222 /*
 223  * The set of PCI devices this driver supports
 224  */
 225 static const struct rte_pci_id pci_id_igb_map[] = {
 226
 227 #define RTE_PCI_DEV_ID_DECL_IGB(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 228 #include "rte_pci_dev_ids.h"
 229
 230 {0},
 231 };
 232
 233 /*
 234  * The set of PCI devices this driver supports (for 82576&I350 VF)
 235  */
 236 static const struct rte_pci_id pci_id_igbvf_map[] = {
 237
 238 #define RTE_PCI_DEV_ID_DECL_IGBVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 239 #include "rte_pci_dev_ids.h"
 240
 241 {0},
 242 };
 243
 244 static const struct eth_dev_ops eth_igb_ops = {
 245         .dev_configure        = eth_igb_configure,
 246         .dev_start            = eth_igb_start,
 247         .dev_stop             = eth_igb_stop,
 248         .dev_close            = eth_igb_close,
 249         .promiscuous_enable   = eth_igb_promiscuous_enable,
 250         .promiscuous_disable  = eth_igb_promiscuous_disable,
 251         .allmulticast_enable  = eth_igb_allmulticast_enable,
 252         .allmulticast_disable = eth_igb_allmulticast_disable,
 253         .link_update          = eth_igb_link_update,
 254         .stats_get            = eth_igb_stats_get,
 255         .stats_reset          = eth_igb_stats_reset,
 256         .dev_infos_get        = eth_igb_infos_get,
 257         .mtu_set              = eth_igb_mtu_set,
 258         .vlan_filter_set      = eth_igb_vlan_filter_set,
 259         .vlan_tpid_set        = eth_igb_vlan_tpid_set,
 260         .vlan_offload_set     = eth_igb_vlan_offload_set,
 261         .rx_queue_setup       = eth_igb_rx_queue_setup,
 262         .rx_queue_release     = eth_igb_rx_queue_release,
 263         .rx_queue_count       = eth_igb_rx_queue_count,
 264         .rx_descriptor_done   = eth_igb_rx_descriptor_done,
 265         .tx_queue_setup       = eth_igb_tx_queue_setup,
 266         .tx_queue_release     = eth_igb_tx_queue_release,
 267         .dev_led_on           = eth_igb_led_on,
 268         .dev_led_off          = eth_igb_led_off,
 269         .flow_ctrl_get        = eth_igb_flow_ctrl_get,
 270         .flow_ctrl_set        = eth_igb_flow_ctrl_set,
 271         .mac_addr_add         = eth_igb_rar_set,
 272         .mac_addr_remove      = eth_igb_rar_clear,
 273         .reta_update          = eth_igb_rss_reta_update,
 274         .reta_query           = eth_igb_rss_reta_query,
 275         .rss_hash_update      = eth_igb_rss_hash_update,
 276         .rss_hash_conf_get    = eth_igb_rss_hash_conf_get,
 277         .filter_ctrl          = eth_igb_filter_ctrl,
 278         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 279 };
 280
 281 /*
 282  * dev_ops for virtual function, bare necessities for basic vf
 283  * operation have been implemented
 284  */
 285 static const struct eth_dev_ops igbvf_eth_dev_ops = {
 286         .dev_configure        = igbvf_dev_configure,
 287         .dev_start            = igbvf_dev_start,
 288         .dev_stop             = igbvf_dev_stop,
 289         .dev_close            = igbvf_dev_close,
 290         .link_update          = eth_igb_link_update,
 291         .stats_get            = eth_igbvf_stats_get,
 292         .stats_reset          = eth_igbvf_stats_reset,
 293         .vlan_filter_set      = igbvf_vlan_filter_set,
 294         .dev_infos_get        = eth_igbvf_infos_get,
 295         .rx_queue_setup       = eth_igb_rx_queue_setup,
 296         .rx_queue_release     = eth_igb_rx_queue_release,
 297         .tx_queue_setup       = eth_igb_tx_queue_setup,
 298         .tx_queue_release     = eth_igb_tx_queue_release,
 299         .set_mc_addr_list     = eth_igb_set_mc_addr_list,
 300 };
 301
 302 /**
 303  * Atomically reads the link status information from global
 304  * structure rte_eth_dev.
 305  *
 306  * @param dev
 307  *   - Pointer to the structure rte_eth_dev to read from.
 308  *   - Pointer to the buffer to be saved with the link status.
 309  *
 310  * @return
 311  *   - On success, zero.
 312  *   - On failure, negative value.
 313  */
 314 static inline int
 315 rte_igb_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 316                                 struct rte_eth_link *link)
 317 {
 318         struct rte_eth_link *dst = link;
 319         struct rte_eth_link *src = &(dev->data->dev_link);
 320
 321         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 322                                         *(uint64_t *)src) == 0)
 323                 return -1;
 324
 325         return 0;
 326 }
 327
 328 /**
 329  * Atomically writes the link status information into global
 330  * structure rte_eth_dev.
 331  *
 332  * @param dev
 333  *   - Pointer to the structure rte_eth_dev to read from.
 334  *   - Pointer to the buffer to be saved with the link status.
 335  *
 336  * @return
 337  *   - On success, zero.
 338  *   - On failure, negative value.
 339  */
 340 static inline int
 341 rte_igb_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 342                                 struct rte_eth_link *link)
 343 {
 344         struct rte_eth_link *dst = &(dev->data->dev_link);
 345         struct rte_eth_link *src = link;
 346
 347         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 348                                         *(uint64_t *)src) == 0)
 349                 return -1;
 350
 351         return 0;
 352 }
 353
 354 static inline void
 355 igb_intr_enable(struct rte_eth_dev *dev)
 356 {
 357         struct e1000_interrupt *intr =
 358                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 359         struct e1000_hw *hw =
 360                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 361
 362         E1000_WRITE_REG(hw, E1000_IMS, intr->mask);
 363         E1000_WRITE_FLUSH(hw);
 364 }
 365
 366 static void
 367 igb_intr_disable(struct e1000_hw *hw)
 368 {
 369         E1000_WRITE_REG(hw, E1000_IMC, ~0);
 370         E1000_WRITE_FLUSH(hw);
 371 }
 372
 373 static inline int32_t
 374 igb_pf_reset_hw(struct e1000_hw *hw)
 375 {
 376         uint32_t ctrl_ext;
 377         int32_t status;
 378
 379         status = e1000_reset_hw(hw);
 380
 381         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 382         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 383         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 384         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 385         E1000_WRITE_FLUSH(hw);
 386
 387         return status;
 388 }
 389
 390 static void
 391 igb_identify_hardware(struct rte_eth_dev *dev)
 392 {
 393         struct e1000_hw *hw =
 394                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 395
 396         hw->vendor_id = dev->pci_dev->id.vendor_id;
 397         hw->device_id = dev->pci_dev->id.device_id;
 398         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
 399         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
 400
 401         e1000_set_mac_type(hw);
 402
 403         /* need to check if it is a vf device below */
 404 }
 405
 406 static int
 407 igb_reset_swfw_lock(struct e1000_hw *hw)
 408 {
 409         int ret_val;
 410
 411         /*
 412          * Do mac ops initialization manually here, since we will need
 413          * some function pointers set by this call.
 414          */
 415         ret_val = e1000_init_mac_params(hw);
 416         if (ret_val)
 417                 return ret_val;
 418
 419         /*
 420          * SMBI lock should not fail in this early stage. If this is the case,
 421          * it is due to an improper exit of the application.
 422          * So force the release of the faulty lock.
 423          */
 424         if (e1000_get_hw_semaphore_generic(hw) < 0) {
 425                 PMD_DRV_LOG(DEBUG, "SMBI lock released");
 426         }
 427         e1000_put_hw_semaphore_generic(hw);
 428
 429         if (hw->mac.ops.acquire_swfw_sync != NULL) {
 430                 uint16_t mask;
 431
 432                 /*
 433                  * Phy lock should not fail in this early stage. If this is the case,
 434                  * it is due to an improper exit of the application.
 435                  * So force the release of the faulty lock.
 436                  */
 437                 mask = E1000_SWFW_PHY0_SM << hw->bus.func;
 438                 if (hw->bus.func > E1000_FUNC_1)
 439                         mask <<= 2;
 440                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 441                         PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released",
 442                                     hw->bus.func);
 443                 }
 444                 hw->mac.ops.release_swfw_sync(hw, mask);
 445
 446                 /*
 447                  * This one is more tricky since it is common to all ports; but
 448                  * swfw_sync retries last long enough (1s) to be almost sure that if
 449                  * lock can not be taken it is due to an improper lock of the
 450                  * semaphore.
 451                  */
 452                 mask = E1000_SWFW_EEP_SM;
 453                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 454                         PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 455                 }
 456                 hw->mac.ops.release_swfw_sync(hw, mask);
 457         }
 458
 459         return E1000_SUCCESS;
 460 }
 461
 462 static int
 463 eth_igb_dev_init(struct rte_eth_dev *eth_dev)
 464 {
 465         int error = 0;
 466         struct rte_pci_device *pci_dev;
 467         struct e1000_hw *hw =
 468                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 469         struct e1000_vfta * shadow_vfta =
 470                         E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 471         struct e1000_filter_info *filter_info =
 472                 E1000_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
 473         uint32_t ctrl_ext;
 474
 475         pci_dev = eth_dev->pci_dev;
 476         eth_dev->dev_ops = &eth_igb_ops;
 477         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 478         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 479
 480         /* for secondary processes, we don't initialise any further as primary
 481          * has already done this work. Only check we don't need a different
 482          * RX function */
 483         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 484                 if (eth_dev->data->scattered_rx)
 485                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 486                 return 0;
 487         }
 488
 489         hw->hw_addr= (void *)pci_dev->mem_resource[0].addr;
 490
 491         igb_identify_hardware(eth_dev);
 492         if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) {
 493                 error = -EIO;
 494                 goto err_late;
 495         }
 496
 497         e1000_get_bus_info(hw);
 498
 499         /* Reset any pending lock */
 500         if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) {
 501                 error = -EIO;
 502                 goto err_late;
 503         }
 504
 505         /* Finish initialization */
 506         if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) {
 507                 error = -EIO;
 508                 goto err_late;
 509         }
 510
 511         hw->mac.autoneg = 1;
 512         hw->phy.autoneg_wait_to_complete = 0;
 513         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 514
 515         /* Copper options */
 516         if (hw->phy.media_type == e1000_media_type_copper) {
 517                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
 518                 hw->phy.disable_polarity_correction = 0;
 519                 hw->phy.ms_type = e1000_ms_hw_default;
 520         }
 521
 522         /*
 523          * Start from a known state, this is important in reading the nvm
 524          * and mac from that.
 525          */
 526         igb_pf_reset_hw(hw);
 527
 528         /* Make sure we have a good EEPROM before we read from it */
 529         if (e1000_validate_nvm_checksum(hw) < 0) {
 530                 /*
 531                  * Some PCI-E parts fail the first check due to
 532                  * the link being in sleep state, call it again,
 533                  * if it fails a second time its a real issue.
 534                  */
 535                 if (e1000_validate_nvm_checksum(hw) < 0) {
 536                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
 537                         error = -EIO;
 538                         goto err_late;
 539                 }
 540         }
 541
 542         /* Read the permanent MAC address out of the EEPROM */
 543         if (e1000_read_mac_addr(hw) != 0) {
 544                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
 545                 error = -EIO;
 546                 goto err_late;
 547         }
 548
 549         /* Allocate memory for storing MAC addresses */
 550         eth_dev->data->mac_addrs = rte_zmalloc("e1000",
 551                 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
 552         if (eth_dev->data->mac_addrs == NULL) {
 553                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to "
 554                                                 "store MAC addresses",
 555                                 ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 556                 error = -ENOMEM;
 557                 goto err_late;
 558         }
 559
 560         /* Copy the permanent MAC address */
 561         ether_addr_copy((struct ether_addr *)hw->mac.addr, &eth_dev->data->mac_addrs[0]);
 562
 563         /* initialize the vfta */
 564         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 565
 566         /* Now initialize the hardware */
 567         if (igb_hardware_init(hw) != 0) {
 568                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
 569                 rte_free(eth_dev->data->mac_addrs);
 570                 eth_dev->data->mac_addrs = NULL;
 571                 error = -ENODEV;
 572                 goto err_late;
 573         }
 574         hw->mac.get_link_status = 1;
 575
 576         /* Indicate SOL/IDER usage */
 577         if (e1000_check_reset_block(hw) < 0) {
 578                 PMD_INIT_LOG(ERR, "PHY reset is blocked due to"
 579                                         "SOL/IDER session");
 580         }
 581
 582         /* initialize PF if max_vfs not zero */
 583         igb_pf_host_init(eth_dev);
 584
 585         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 586         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 587         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 588         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 589         E1000_WRITE_FLUSH(hw);
 590
 591         PMD_INIT_LOG(INFO, "port_id %d vendorID=0x%x deviceID=0x%x",
 592                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 593                      pci_dev->id.device_id);
 594
 595         rte_intr_callback_register(&(pci_dev->intr_handle),
 596                 eth_igb_interrupt_handler, (void *)eth_dev);
 597
 598         /* enable uio intr after callback register */
 599         rte_intr_enable(&(pci_dev->intr_handle));
 600
 601         /* enable support intr */
 602         igb_intr_enable(eth_dev);
 603
 604         TAILQ_INIT(&filter_info->flex_list);
 605         filter_info->flex_mask = 0;
 606         TAILQ_INIT(&filter_info->twotuple_list);
 607         filter_info->twotuple_mask = 0;
 608         TAILQ_INIT(&filter_info->fivetuple_list);
 609         filter_info->fivetuple_mask = 0;
 610
 611         return 0;
 612
 613 err_late:
 614         igb_hw_control_release(hw);
 615
 616         return (error);
 617 }
 618
 619 /*
 620  * Virtual Function device init
 621  */
 622 static int
 623 eth_igbvf_dev_init(struct rte_eth_dev *eth_dev)
 624 {
 625         struct rte_pci_device *pci_dev;
 626         struct e1000_hw *hw =
 627                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 628         int diag;
 629
 630         PMD_INIT_FUNC_TRACE();
 631
 632         eth_dev->dev_ops = &igbvf_eth_dev_ops;
 633         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 634         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 635
 636         /* for secondary processes, we don't initialise any further as primary
 637          * has already done this work. Only check we don't need a different
 638          * RX function */
 639         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 640                 if (eth_dev->data->scattered_rx)
 641                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 642                 return 0;
 643         }
 644
 645         pci_dev = eth_dev->pci_dev;
 646
 647         hw->device_id = pci_dev->id.device_id;
 648         hw->vendor_id = pci_dev->id.vendor_id;
 649         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 650
 651         /* Initialize the shared code (base driver) */
 652         diag = e1000_setup_init_funcs(hw, TRUE);
 653         if (diag != 0) {
 654                 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d",
 655                         diag);
 656                 return -EIO;
 657         }
 658
 659         /* init_mailbox_params */
 660         hw->mbx.ops.init_params(hw);
 661
 662         /* Disable the interrupts for VF */
 663         igbvf_intr_disable(hw);
 664
 665         diag = hw->mac.ops.reset_hw(hw);
 666
 667         /* Allocate memory for storing MAC addresses */
 668         eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN *
 669                 hw->mac.rar_entry_count, 0);
 670         if (eth_dev->data->mac_addrs == NULL) {
 671                 PMD_INIT_LOG(ERR,
 672                         "Failed to allocate %d bytes needed to store MAC "
 673                         "addresses",
 674                         ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 675                 return -ENOMEM;
 676         }
 677
 678         /* Copy the permanent MAC address */
 679         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 680                         &eth_dev->data->mac_addrs[0]);
 681
 682         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x "
 683                      "mac.type=%s",
 684                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 685                      pci_dev->id.device_id, "igb_mac_82576_vf");
 686
 687         return 0;
 688 }
 689
 690 static struct eth_driver rte_igb_pmd = {
 691         .pci_drv = {
 692                 .name = "rte_igb_pmd",
 693                 .id_table = pci_id_igb_map,
 694                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
 695         },
 696         .eth_dev_init = eth_igb_dev_init,
 697         .dev_private_size = sizeof(struct e1000_adapter),
 698 };
 699
 700 /*
 701  * virtual function driver struct
 702  */
 703 static struct eth_driver rte_igbvf_pmd = {
 704         .pci_drv = {
 705                 .name = "rte_igbvf_pmd",
 706                 .id_table = pci_id_igbvf_map,
 707                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 708         },
 709         .eth_dev_init = eth_igbvf_dev_init,
 710         .dev_private_size = sizeof(struct e1000_adapter),
 711 };
 712
 713 static int
 714 rte_igb_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 715 {
 716         rte_eth_driver_register(&rte_igb_pmd);
 717         return 0;
 718 }
 719
 720 static void
 721 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 722 {
 723         struct e1000_hw *hw =
 724                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 725         /* RCTL: enable VLAN filter since VMDq always use VLAN filter */
 726         uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL);
 727         rctl |= E1000_RCTL_VFE;
 728         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
 729 }
 730
 731 /*
 732  * VF Driver initialization routine.
 733  * Invoked one at EAL init time.
 734  * Register itself as the [Virtual Poll Mode] Driver of PCI IGB devices.
 735  */
 736 static int
 737 rte_igbvf_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 738 {
 739         PMD_INIT_FUNC_TRACE();
 740
 741         rte_eth_driver_register(&rte_igbvf_pmd);
 742         return (0);
 743 }
 744
 745 static int
 746 eth_igb_configure(struct rte_eth_dev *dev)
 747 {
 748         struct e1000_interrupt *intr =
 749                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 750
 751         PMD_INIT_FUNC_TRACE();
 752         intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
 753         PMD_INIT_FUNC_TRACE();
 754
 755         return (0);
 756 }
 757
 758 static int
 759 eth_igb_start(struct rte_eth_dev *dev)
 760 {
 761         struct e1000_hw *hw =
 762                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 763         int ret, i, mask;
 764         uint32_t ctrl_ext;
 765
 766         PMD_INIT_FUNC_TRACE();
 767
 768         /* Power up the phy. Needed to make the link go Up */
 769         e1000_power_up_phy(hw);
 770
 771         /*
 772          * Packet Buffer Allocation (PBA)
 773          * Writing PBA sets the receive portion of the buffer
 774          * the remainder is used for the transmit buffer.
 775          */
 776         if (hw->mac.type == e1000_82575) {
 777                 uint32_t pba;
 778
 779                 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
 780                 E1000_WRITE_REG(hw, E1000_PBA, pba);
 781         }
 782
 783         /* Put the address into the Receive Address Array */
 784         e1000_rar_set(hw, hw->mac.addr, 0);
 785
 786         /* Initialize the hardware */
 787         if (igb_hardware_init(hw)) {
 788                 PMD_INIT_LOG(ERR, "Unable to initialize the hardware");
 789                 return (-EIO);
 790         }
 791
 792         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
 793
 794         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 795         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 796         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 797         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 798         E1000_WRITE_FLUSH(hw);
 799
 800         /* configure PF module if SRIOV enabled */
 801         igb_pf_host_configure(dev);
 802
 803         /* Configure for OS presence */
 804         igb_init_manageability(hw);
 805
 806         eth_igb_tx_init(dev);
 807
 808         /* This can fail when allocating mbufs for descriptor rings */
 809         ret = eth_igb_rx_init(dev);
 810         if (ret) {
 811                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
 812                 igb_dev_clear_queues(dev);
 813                 return ret;
 814         }
 815
 816         e1000_clear_hw_cntrs_base_generic(hw);
 817
 818         /*
 819          * VLAN Offload Settings
 820          */
 821         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
 822                         ETH_VLAN_EXTEND_MASK;
 823         eth_igb_vlan_offload_set(dev, mask);
 824
 825         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
 826                 /* Enable VLAN filter since VMDq always use VLAN filter */
 827                 igb_vmdq_vlan_hw_filter_enable(dev);
 828         }
 829
 830         /*
 831          * Configure the Interrupt Moderation register (EITR) with the maximum
 832          * possible value (0xFFFF) to minimize "System Partial Write" issued by
 833          * spurious [DMA] memory updates of RX and TX ring descriptors.
 834          *
 835          * With a EITR granularity of 2 microseconds in the 82576, only 7/8
 836          * spurious memory updates per second should be expected.
 837          * ((65535 * 2) / 1000.1000 ~= 0.131 second).
 838          *
 839          * Because interrupts are not used at all, the MSI-X is not activated
 840          * and interrupt moderation is controlled by EITR[0].
 841          *
 842          * Note that having [almost] disabled memory updates of RX and TX ring
 843          * descriptors through the Interrupt Moderation mechanism, memory
 844          * updates of ring descriptors are now moderated by the configurable
 845          * value of Write-Back Threshold registers.
 846          */
 847         if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) ||
 848                 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) ||
 849                 (hw->mac.type == e1000_i211)) {
 850                 uint32_t ivar;
 851
 852                 /* Enable all RX & TX queues in the IVAR registers */
 853                 ivar = (uint32_t) ((E1000_IVAR_VALID << 16) | E1000_IVAR_VALID);
 854                 for (i = 0; i < 8; i++)
 855                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, i, ivar);
 856
 857                 /* Configure EITR with the maximum possible value (0xFFFF) */
 858                 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF);
 859         }
 860
 861         /* Setup link speed and duplex */
 862         switch (dev->data->dev_conf.link_speed) {
 863         case ETH_LINK_SPEED_AUTONEG:
 864                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
 865                         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 866                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
 867                         hw->phy.autoneg_advertised = E1000_ALL_HALF_DUPLEX;
 868                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
 869                         hw->phy.autoneg_advertised = E1000_ALL_FULL_DUPLEX;
 870                 else
 871                         goto error_invalid_config;
 872                 break;
 873         case ETH_LINK_SPEED_10:
 874                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
 875                         hw->phy.autoneg_advertised = E1000_ALL_10_SPEED;
 876                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
 877                         hw->phy.autoneg_advertised = ADVERTISE_10_HALF;
 878                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
 879                         hw->phy.autoneg_advertised = ADVERTISE_10_FULL;
 880                 else
 881                         goto error_invalid_config;
 882                 break;
 883         case ETH_LINK_SPEED_100:
 884                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
 885                         hw->phy.autoneg_advertised = E1000_ALL_100_SPEED;
 886                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
 887                         hw->phy.autoneg_advertised = ADVERTISE_100_HALF;
 888                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
 889                         hw->phy.autoneg_advertised = ADVERTISE_100_FULL;
 890                 else
 891                         goto error_invalid_config;
 892                 break;
 893         case ETH_LINK_SPEED_1000:
 894                 if ((dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX) ||
 895                                 (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX))
 896                         hw->phy.autoneg_advertised = ADVERTISE_1000_FULL;
 897                 else
 898                         goto error_invalid_config;
 899                 break;
 900         case ETH_LINK_SPEED_10000:
 901         default:
 902                 goto error_invalid_config;
 903         }
 904         e1000_setup_link(hw);
 905
 906         /* check if lsc interrupt feature is enabled */
 907         if (dev->data->dev_conf.intr_conf.lsc != 0)
 908                 ret = eth_igb_lsc_interrupt_setup(dev);
 909
 910         /* resume enabled intr since hw reset */
 911         igb_intr_enable(dev);
 912
 913         PMD_INIT_LOG(DEBUG, "<<");
 914
 915         return (0);
 916
 917 error_invalid_config:
 918         PMD_INIT_LOG(ERR, "Invalid link_speed/link_duplex (%u/%u) for port %u",
 919                      dev->data->dev_conf.link_speed,
 920                      dev->data->dev_conf.link_duplex, dev->data->port_id);
 921         igb_dev_clear_queues(dev);
 922         return (-EINVAL);
 923 }
 924
 925 /*********************************************************************
 926  *
 927  *  This routine disables all traffic on the adapter by issuing a
 928  *  global reset on the MAC.
 929  *
 930  **********************************************************************/
 931 static void
 932 eth_igb_stop(struct rte_eth_dev *dev)
 933 {
 934         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 935         struct e1000_filter_info *filter_info =
 936                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
 937         struct rte_eth_link link;
 938         struct e1000_flex_filter *p_flex;
 939         struct e1000_5tuple_filter *p_5tuple, *p_5tuple_next;
 940         struct e1000_2tuple_filter *p_2tuple, *p_2tuple_next;
 941
 942         igb_intr_disable(hw);
 943         igb_pf_reset_hw(hw);
 944         E1000_WRITE_REG(hw, E1000_WUC, 0);
 945
 946         /* Set bit for Go Link disconnect */
 947         if (hw->mac.type >= e1000_82580) {
 948                 uint32_t phpm_reg;
 949
 950                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
 951                 phpm_reg |= E1000_82580_PM_GO_LINKD;
 952                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
 953         }
 954
 955         /* Power down the phy. Needed to make the link go Down */
 956         if (hw->phy.media_type == e1000_media_type_copper)
 957                 e1000_power_down_phy(hw);
 958         else
 959                 e1000_shutdown_fiber_serdes_link(hw);
 960
 961         igb_dev_clear_queues(dev);
 962
 963         /* clear the recorded link status */
 964         memset(&link, 0, sizeof(link));
 965         rte_igb_dev_atomic_write_link_status(dev, &link);
 966
 967         /* Remove all flex filters of the device */
 968         while ((p_flex = TAILQ_FIRST(&filter_info->flex_list))) {
 969                 TAILQ_REMOVE(&filter_info->flex_list, p_flex, entries);
 970                 rte_free(p_flex);
 971         }
 972         filter_info->flex_mask = 0;
 973
 974         /* Remove all ntuple filters of the device */
 975         for (p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list);
 976              p_5tuple != NULL; p_5tuple = p_5tuple_next) {
 977                 p_5tuple_next = TAILQ_NEXT(p_5tuple, entries);
 978                 TAILQ_REMOVE(&filter_info->fivetuple_list,
 979                              p_5tuple, entries);
 980                 rte_free(p_5tuple);
 981         }
 982         filter_info->fivetuple_mask = 0;
 983         for (p_2tuple = TAILQ_FIRST(&filter_info->twotuple_list);
 984              p_2tuple != NULL; p_2tuple = p_2tuple_next) {
 985                 p_2tuple_next = TAILQ_NEXT(p_2tuple, entries);
 986                 TAILQ_REMOVE(&filter_info->twotuple_list,
 987                              p_2tuple, entries);
 988                 rte_free(p_2tuple);
 989         }
 990         filter_info->twotuple_mask = 0;
 991 }
 992
 993 static void
 994 eth_igb_close(struct rte_eth_dev *dev)
 995 {
 996         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 997         struct rte_eth_link link;
 998
 999         eth_igb_stop(dev);
1000         e1000_phy_hw_reset(hw);
1001         igb_release_manageability(hw);
1002         igb_hw_control_release(hw);
1003
1004         /* Clear bit for Go Link disconnect */
1005         if (hw->mac.type >= e1000_82580) {
1006                 uint32_t phpm_reg;
1007
1008                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
1009                 phpm_reg &= ~E1000_82580_PM_GO_LINKD;
1010                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
1011         }
1012
1013         igb_dev_clear_queues(dev);
1014
1015         memset(&link, 0, sizeof(link));
1016         rte_igb_dev_atomic_write_link_status(dev, &link);
1017 }
1018
1019 static int
1020 igb_get_rx_buffer_size(struct e1000_hw *hw)
1021 {
1022         uint32_t rx_buf_size;
1023         if (hw->mac.type == e1000_82576) {
1024                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10;
1025         } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) {
1026                 /* PBS needs to be translated according to a lookup table */
1027                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf);
1028                 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size);
1029                 rx_buf_size = (rx_buf_size << 10);
1030         } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) {
1031                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10;
1032         } else {
1033                 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10;
1034         }
1035
1036         return rx_buf_size;
1037 }
1038
1039 /*********************************************************************
1040  *
1041  *  Initialize the hardware
1042  *
1043  **********************************************************************/
1044 static int
1045 igb_hardware_init(struct e1000_hw *hw)
1046 {
1047         uint32_t rx_buf_size;
1048         int diag;
1049
1050         /* Let the firmware know the OS is in control */
1051         igb_hw_control_acquire(hw);
1052
1053         /*
1054          * These parameters control the automatic generation (Tx) and
1055          * response (Rx) to Ethernet PAUSE frames.
1056          * - High water mark should allow for at least two standard size (1518)
1057          *   frames to be received after sending an XOFF.
1058          * - Low water mark works best when it is very near the high water mark.
1059          *   This allows the receiver to restart by sending XON when it has
1060          *   drained a bit. Here we use an arbitrary value of 1500 which will
1061          *   restart after one full frame is pulled from the buffer. There
1062          *   could be several smaller frames in the buffer and if so they will
1063          *   not trigger the XON until their total number reduces the buffer
1064          *   by 1500.
1065          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
1066          */
1067         rx_buf_size = igb_get_rx_buffer_size(hw);
1068
1069         hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2);
1070         hw->fc.low_water = hw->fc.high_water - 1500;
1071         hw->fc.pause_time = IGB_FC_PAUSE_TIME;
1072         hw->fc.send_xon = 1;
1073
1074         /* Set Flow control, use the tunable location if sane */
1075         if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4))
1076                 hw->fc.requested_mode = igb_fc_setting;
1077         else
1078                 hw->fc.requested_mode = e1000_fc_none;
1079
1080         /* Issue a global reset */
1081         igb_pf_reset_hw(hw);
1082         E1000_WRITE_REG(hw, E1000_WUC, 0);
1083
1084         diag = e1000_init_hw(hw);
1085         if (diag < 0)
1086                 return (diag);
1087
1088         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1089         e1000_get_phy_info(hw);
1090         e1000_check_for_link(hw);
1091
1092         return (0);
1093 }
1094
1095 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */
1096 static void
1097 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1098 {
1099         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1100         struct e1000_hw_stats *stats =
1101                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1102         int pause_frames;
1103
1104         if(hw->phy.media_type == e1000_media_type_copper ||
1105             (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
1106                 stats->symerrs +=
1107                     E1000_READ_REG(hw,E1000_SYMERRS);
1108                 stats->sec += E1000_READ_REG(hw, E1000_SEC);
1109         }
1110
1111         stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
1112         stats->mpc += E1000_READ_REG(hw, E1000_MPC);
1113         stats->scc += E1000_READ_REG(hw, E1000_SCC);
1114         stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
1115
1116         stats->mcc += E1000_READ_REG(hw, E1000_MCC);
1117         stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
1118         stats->colc += E1000_READ_REG(hw, E1000_COLC);
1119         stats->dc += E1000_READ_REG(hw, E1000_DC);
1120         stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
1121         stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
1122         stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
1123         /*
1124         ** For watchdog management we need to know if we have been
1125         ** paused during the last interval, so capture that here.
1126         */
1127         pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC);
1128         stats->xoffrxc += pause_frames;
1129         stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
1130         stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
1131         stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
1132         stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
1133         stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
1134         stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
1135         stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
1136         stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
1137         stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
1138         stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
1139         stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
1140         stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
1141
1142         /* For the 64-bit byte counters the low dword must be read first. */
1143         /* Both registers clear on the read of the high dword */
1144
1145         stats->gorc += E1000_READ_REG(hw, E1000_GORCL);
1146         stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32);
1147         stats->gotc += E1000_READ_REG(hw, E1000_GOTCL);
1148         stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32);
1149
1150         stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
1151         stats->ruc += E1000_READ_REG(hw, E1000_RUC);
1152         stats->rfc += E1000_READ_REG(hw, E1000_RFC);
1153         stats->roc += E1000_READ_REG(hw, E1000_ROC);
1154         stats->rjc += E1000_READ_REG(hw, E1000_RJC);
1155
1156         stats->tor += E1000_READ_REG(hw, E1000_TORH);
1157         stats->tot += E1000_READ_REG(hw, E1000_TOTH);
1158
1159         stats->tpr += E1000_READ_REG(hw, E1000_TPR);
1160         stats->tpt += E1000_READ_REG(hw, E1000_TPT);
1161         stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
1162         stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
1163         stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
1164         stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
1165         stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
1166         stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
1167         stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
1168         stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
1169
1170         /* Interrupt Counts */
1171
1172         stats->iac += E1000_READ_REG(hw, E1000_IAC);
1173         stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
1174         stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
1175         stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
1176         stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
1177         stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
1178         stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
1179         stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
1180         stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
1181
1182         /* Host to Card Statistics */
1183
1184         stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
1185         stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
1186         stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
1187         stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
1188         stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
1189         stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
1190         stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
1191         stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL);
1192         stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32);
1193         stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL);
1194         stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32);
1195         stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
1196         stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
1197         stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
1198
1199         stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
1200         stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
1201         stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
1202         stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
1203         stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
1204         stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
1205
1206         if (rte_stats == NULL)
1207                 return;
1208
1209         /* Rx Errors */
1210         rte_stats->ibadcrc = stats->crcerrs;
1211         rte_stats->ibadlen = stats->rlec + stats->ruc + stats->roc;
1212         rte_stats->imissed = stats->mpc;
1213         rte_stats->ierrors = rte_stats->ibadcrc +
1214                              rte_stats->ibadlen +
1215                              rte_stats->imissed +
1216                              stats->rxerrc + stats->algnerrc + stats->cexterr;
1217
1218         /* Tx Errors */
1219         rte_stats->oerrors = stats->ecol + stats->latecol;
1220
1221         /* XON/XOFF pause frames */
1222         rte_stats->tx_pause_xon  = stats->xontxc;
1223         rte_stats->rx_pause_xon  = stats->xonrxc;
1224         rte_stats->tx_pause_xoff = stats->xofftxc;
1225         rte_stats->rx_pause_xoff = stats->xoffrxc;
1226
1227         rte_stats->ipackets = stats->gprc;
1228         rte_stats->opackets = stats->gptc;
1229         rte_stats->ibytes   = stats->gorc;
1230         rte_stats->obytes   = stats->gotc;
1231 }
1232
1233 static void
1234 eth_igb_stats_reset(struct rte_eth_dev *dev)
1235 {
1236         struct e1000_hw_stats *hw_stats =
1237                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1238
1239         /* HW registers are cleared on read */
1240         eth_igb_stats_get(dev, NULL);
1241
1242         /* Reset software totals */
1243         memset(hw_stats, 0, sizeof(*hw_stats));
1244 }
1245
1246 static void
1247 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1248 {
1249         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1250         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1251                           E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1252
1253         /* Good Rx packets, include VF loopback */
1254         UPDATE_VF_STAT(E1000_VFGPRC,
1255             hw_stats->last_gprc, hw_stats->gprc);
1256
1257         /* Good Rx octets, include VF loopback */
1258         UPDATE_VF_STAT(E1000_VFGORC,
1259             hw_stats->last_gorc, hw_stats->gorc);
1260
1261         /* Good Tx packets, include VF loopback */
1262         UPDATE_VF_STAT(E1000_VFGPTC,
1263             hw_stats->last_gptc, hw_stats->gptc);
1264
1265         /* Good Tx octets, include VF loopback */
1266         UPDATE_VF_STAT(E1000_VFGOTC,
1267             hw_stats->last_gotc, hw_stats->gotc);
1268
1269         /* Rx Multicst packets */
1270         UPDATE_VF_STAT(E1000_VFMPRC,
1271             hw_stats->last_mprc, hw_stats->mprc);
1272
1273         /* Good Rx loopback packets */
1274         UPDATE_VF_STAT(E1000_VFGPRLBC,
1275             hw_stats->last_gprlbc, hw_stats->gprlbc);
1276
1277         /* Good Rx loopback octets */
1278         UPDATE_VF_STAT(E1000_VFGORLBC,
1279             hw_stats->last_gorlbc, hw_stats->gorlbc);
1280
1281         /* Good Tx loopback packets */
1282         UPDATE_VF_STAT(E1000_VFGPTLBC,
1283             hw_stats->last_gptlbc, hw_stats->gptlbc);
1284
1285         /* Good Tx loopback octets */
1286         UPDATE_VF_STAT(E1000_VFGOTLBC,
1287             hw_stats->last_gotlbc, hw_stats->gotlbc);
1288
1289         if (rte_stats == NULL)
1290                 return;
1291
1292         rte_stats->ipackets = hw_stats->gprc;
1293         rte_stats->ibytes = hw_stats->gorc;
1294         rte_stats->opackets = hw_stats->gptc;
1295         rte_stats->obytes = hw_stats->gotc;
1296         rte_stats->imcasts = hw_stats->mprc;
1297         rte_stats->ilbpackets = hw_stats->gprlbc;
1298         rte_stats->ilbbytes = hw_stats->gorlbc;
1299         rte_stats->olbpackets = hw_stats->gptlbc;
1300         rte_stats->olbbytes = hw_stats->gotlbc;
1301
1302 }
1303
1304 static void
1305 eth_igbvf_stats_reset(struct rte_eth_dev *dev)
1306 {
1307         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1308                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1309
1310         /* Sync HW register to the last stats */
1311         eth_igbvf_stats_get(dev, NULL);
1312
1313         /* reset HW current stats*/
1314         memset(&hw_stats->gprc, 0, sizeof(*hw_stats) -
1315                offsetof(struct e1000_vf_stats, gprc));
1316
1317 }
1318
1319 static void
1320 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1321 {
1322         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1323
1324         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1325         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1326         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1327         dev_info->rx_offload_capa =
1328                 DEV_RX_OFFLOAD_VLAN_STRIP |
1329                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1330                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1331                 DEV_RX_OFFLOAD_TCP_CKSUM;
1332         dev_info->tx_offload_capa =
1333                 DEV_TX_OFFLOAD_VLAN_INSERT |
1334                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1335                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1336                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1337                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1338
1339         switch (hw->mac.type) {
1340         case e1000_82575:
1341                 dev_info->max_rx_queues = 4;
1342                 dev_info->max_tx_queues = 4;
1343                 dev_info->max_vmdq_pools = 0;
1344                 break;
1345
1346         case e1000_82576:
1347                 dev_info->max_rx_queues = 16;
1348                 dev_info->max_tx_queues = 16;
1349                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1350                 dev_info->vmdq_queue_num = 16;
1351                 break;
1352
1353         case e1000_82580:
1354                 dev_info->max_rx_queues = 8;
1355                 dev_info->max_tx_queues = 8;
1356                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1357                 dev_info->vmdq_queue_num = 8;
1358                 break;
1359
1360         case e1000_i350:
1361                 dev_info->max_rx_queues = 8;
1362                 dev_info->max_tx_queues = 8;
1363                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1364                 dev_info->vmdq_queue_num = 8;
1365                 break;
1366
1367         case e1000_i354:
1368                 dev_info->max_rx_queues = 8;
1369                 dev_info->max_tx_queues = 8;
1370                 break;
1371
1372         case e1000_i210:
1373                 dev_info->max_rx_queues = 4;
1374                 dev_info->max_tx_queues = 4;
1375                 dev_info->max_vmdq_pools = 0;
1376                 break;
1377
1378         case e1000_i211:
1379                 dev_info->max_rx_queues = 2;
1380                 dev_info->max_tx_queues = 2;
1381                 dev_info->max_vmdq_pools = 0;
1382                 break;
1383
1384         default:
1385                 /* Should not happen */
1386                 break;
1387         }
1388         dev_info->hash_key_size = IGB_HKEY_MAX_INDEX * sizeof(uint32_t);
1389         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
1390         dev_info->flow_type_rss_offloads = IGB_RSS_OFFLOAD_ALL;
1391
1392         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1393                 .rx_thresh = {
1394                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1395                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1396                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
1397                 },
1398                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
1399                 .rx_drop_en = 0,
1400         };
1401
1402         dev_info->default_txconf = (struct rte_eth_txconf) {
1403                 .tx_thresh = {
1404                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
1405                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
1406                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
1407                 },
1408                 .txq_flags = 0,
1409         };
1410 }
1411
1412 static void
1413 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1414 {
1415         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1416
1417         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1418         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1419         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1420         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
1421                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1422                                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1423                                 DEV_RX_OFFLOAD_TCP_CKSUM;
1424         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
1425                                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1426                                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1427                                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1428                                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1429         switch (hw->mac.type) {
1430         case e1000_vfadapt:
1431                 dev_info->max_rx_queues = 2;
1432                 dev_info->max_tx_queues = 2;
1433                 break;
1434         case e1000_vfadapt_i350:
1435                 dev_info->max_rx_queues = 1;
1436                 dev_info->max_tx_queues = 1;
1437                 break;
1438         default:
1439                 /* Should not happen */
1440                 break;
1441         }
1442
1443         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1444                 .rx_thresh = {
1445                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1446                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1447                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
1448                 },
1449                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
1450                 .rx_drop_en = 0,
1451         };
1452
1453         dev_info->default_txconf = (struct rte_eth_txconf) {
1454                 .tx_thresh = {
1455                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
1456                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
1457                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
1458                 },
1459                 .txq_flags = 0,
1460         };
1461 }
1462
1463 /* return 0 means link status changed, -1 means not changed */
1464 static int
1465 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1466 {
1467         struct e1000_hw *hw =
1468                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1469         struct rte_eth_link link, old;
1470         int link_check, count;
1471
1472         link_check = 0;
1473         hw->mac.get_link_status = 1;
1474
1475         /* possible wait-to-complete in up to 9 seconds */
1476         for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) {
1477                 /* Read the real link status */
1478                 switch (hw->phy.media_type) {
1479                 case e1000_media_type_copper:
1480                         /* Do the work to read phy */
1481                         e1000_check_for_link(hw);
1482                         link_check = !hw->mac.get_link_status;
1483                         break;
1484
1485                 case e1000_media_type_fiber:
1486                         e1000_check_for_link(hw);
1487                         link_check = (E1000_READ_REG(hw, E1000_STATUS) &
1488                                       E1000_STATUS_LU);
1489                         break;
1490
1491                 case e1000_media_type_internal_serdes:
1492                         e1000_check_for_link(hw);
1493                         link_check = hw->mac.serdes_has_link;
1494                         break;
1495
1496                 /* VF device is type_unknown */
1497                 case e1000_media_type_unknown:
1498                         eth_igbvf_link_update(hw);
1499                         link_check = !hw->mac.get_link_status;
1500                         break;
1501
1502                 default:
1503                         break;
1504                 }
1505                 if (link_check || wait_to_complete == 0)
1506                         break;
1507                 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL);
1508         }
1509         memset(&link, 0, sizeof(link));
1510         rte_igb_dev_atomic_read_link_status(dev, &link);
1511         old = link;
1512
1513         /* Now we check if a transition has happened */
1514         if (link_check) {
1515                 hw->mac.ops.get_link_up_info(hw, &link.link_speed,
1516                                           &link.link_duplex);
1517                 link.link_status = 1;
1518         } else if (!link_check) {
1519                 link.link_speed = 0;
1520                 link.link_duplex = 0;
1521                 link.link_status = 0;
1522         }
1523         rte_igb_dev_atomic_write_link_status(dev, &link);
1524
1525         /* not changed */
1526         if (old.link_status == link.link_status)
1527                 return -1;
1528
1529         /* changed */
1530         return 0;
1531 }
1532
1533 /*
1534  * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
1535  * For ASF and Pass Through versions of f/w this means
1536  * that the driver is loaded.
1537  */
1538 static void
1539 igb_hw_control_acquire(struct e1000_hw *hw)
1540 {
1541         uint32_t ctrl_ext;
1542
1543         /* Let firmware know the driver has taken over */
1544         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1545         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1546 }
1547
1548 /*
1549  * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
1550  * For ASF and Pass Through versions of f/w this means that the
1551  * driver is no longer loaded.
1552  */
1553 static void
1554 igb_hw_control_release(struct e1000_hw *hw)
1555 {
1556         uint32_t ctrl_ext;
1557
1558         /* Let firmware taken over control of h/w */
1559         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1560         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1561                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1562 }
1563
1564 /*
1565  * Bit of a misnomer, what this really means is
1566  * to enable OS management of the system... aka
1567  * to disable special hardware management features.
1568  */
1569 static void
1570 igb_init_manageability(struct e1000_hw *hw)
1571 {
1572         if (e1000_enable_mng_pass_thru(hw)) {
1573                 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H);
1574                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
1575
1576                 /* disable hardware interception of ARP */
1577                 manc &= ~(E1000_MANC_ARP_EN);
1578
1579                 /* enable receiving management packets to the host */
1580                 manc |= E1000_MANC_EN_MNG2HOST;
1581                 manc2h |= 1 << 5;  /* Mng Port 623 */
1582                 manc2h |= 1 << 6;  /* Mng Port 664 */
1583                 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h);
1584                 E1000_WRITE_REG(hw, E1000_MANC, manc);
1585         }
1586 }
1587
1588 static void
1589 igb_release_manageability(struct e1000_hw *hw)
1590 {
1591         if (e1000_enable_mng_pass_thru(hw)) {
1592                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
1593
1594                 manc |= E1000_MANC_ARP_EN;
1595                 manc &= ~E1000_MANC_EN_MNG2HOST;
1596
1597                 E1000_WRITE_REG(hw, E1000_MANC, manc);
1598         }
1599 }
1600
1601 static void
1602 eth_igb_promiscuous_enable(struct rte_eth_dev *dev)
1603 {
1604         struct e1000_hw *hw =
1605                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1606         uint32_t rctl;
1607
1608         rctl = E1000_READ_REG(hw, E1000_RCTL);
1609         rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1610         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1611 }
1612
1613 static void
1614 eth_igb_promiscuous_disable(struct rte_eth_dev *dev)
1615 {
1616         struct e1000_hw *hw =
1617                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1618         uint32_t rctl;
1619
1620         rctl = E1000_READ_REG(hw, E1000_RCTL);
1621         rctl &= (~E1000_RCTL_UPE);
1622         if (dev->data->all_multicast == 1)
1623                 rctl |= E1000_RCTL_MPE;
1624         else
1625                 rctl &= (~E1000_RCTL_MPE);
1626         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1627 }
1628
1629 static void
1630 eth_igb_allmulticast_enable(struct rte_eth_dev *dev)
1631 {
1632         struct e1000_hw *hw =
1633                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1634         uint32_t rctl;
1635
1636         rctl = E1000_READ_REG(hw, E1000_RCTL);
1637         rctl |= E1000_RCTL_MPE;
1638         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1639 }
1640
1641 static void
1642 eth_igb_allmulticast_disable(struct rte_eth_dev *dev)
1643 {
1644         struct e1000_hw *hw =
1645                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1646         uint32_t rctl;
1647
1648         if (dev->data->promiscuous == 1)
1649                 return; /* must remain in all_multicast mode */
1650         rctl = E1000_READ_REG(hw, E1000_RCTL);
1651         rctl &= (~E1000_RCTL_MPE);
1652         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1653 }
1654
1655 static int
1656 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1657 {
1658         struct e1000_hw *hw =
1659                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1660         struct e1000_vfta * shadow_vfta =
1661                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1662         uint32_t vfta;
1663         uint32_t vid_idx;
1664         uint32_t vid_bit;
1665
1666         vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) &
1667                               E1000_VFTA_ENTRY_MASK);
1668         vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK));
1669         vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx);
1670         if (on)
1671                 vfta |= vid_bit;
1672         else
1673                 vfta &= ~vid_bit;
1674         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta);
1675
1676         /* update local VFTA copy */
1677         shadow_vfta->vfta[vid_idx] = vfta;
1678
1679         return 0;
1680 }
1681
1682 static void
1683 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1684 {
1685         struct e1000_hw *hw =
1686                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1687         uint32_t reg = ETHER_TYPE_VLAN ;
1688
1689         reg |= (tpid << 16);
1690         E1000_WRITE_REG(hw, E1000_VET, reg);
1691 }
1692
1693 static void
1694 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1695 {
1696         struct e1000_hw *hw =
1697                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1698         uint32_t reg;
1699
1700         /* Filter Table Disable */
1701         reg = E1000_READ_REG(hw, E1000_RCTL);
1702         reg &= ~E1000_RCTL_CFIEN;
1703         reg &= ~E1000_RCTL_VFE;
1704         E1000_WRITE_REG(hw, E1000_RCTL, reg);
1705 }
1706
1707 static void
1708 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1709 {
1710         struct e1000_hw *hw =
1711                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1712         struct e1000_vfta * shadow_vfta =
1713                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1714         uint32_t reg;
1715         int i;
1716
1717         /* Filter Table Enable, CFI not used for packet acceptance */
1718         reg = E1000_READ_REG(hw, E1000_RCTL);
1719         reg &= ~E1000_RCTL_CFIEN;
1720         reg |= E1000_RCTL_VFE;
1721         E1000_WRITE_REG(hw, E1000_RCTL, reg);
1722
1723         /* restore VFTA table */
1724         for (i = 0; i < IGB_VFTA_SIZE; i++)
1725                 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]);
1726 }
1727
1728 static void
1729 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev)
1730 {
1731         struct e1000_hw *hw =
1732                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1733         uint32_t reg;
1734
1735         /* VLAN Mode Disable */
1736         reg = E1000_READ_REG(hw, E1000_CTRL);
1737         reg &= ~E1000_CTRL_VME;
1738         E1000_WRITE_REG(hw, E1000_CTRL, reg);
1739 }
1740
1741 static void
1742 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev)
1743 {
1744         struct e1000_hw *hw =
1745                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1746         uint32_t reg;
1747
1748         /* VLAN Mode Enable */
1749         reg = E1000_READ_REG(hw, E1000_CTRL);
1750         reg |= E1000_CTRL_VME;
1751         E1000_WRITE_REG(hw, E1000_CTRL, reg);
1752 }
1753
1754 static void
1755 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1756 {
1757         struct e1000_hw *hw =
1758                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1759         uint32_t reg;
1760
1761         /* CTRL_EXT: Extended VLAN */
1762         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
1763         reg &= ~E1000_CTRL_EXT_EXTEND_VLAN;
1764         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
1765
1766         /* Update maximum packet length */
1767         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
1768                 E1000_WRITE_REG(hw, E1000_RLPML,
1769                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
1770                                                 VLAN_TAG_SIZE);
1771 }
1772
1773 static void
1774 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1775 {
1776         struct e1000_hw *hw =
1777                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1778         uint32_t reg;
1779
1780         /* CTRL_EXT: Extended VLAN */
1781         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
1782         reg |= E1000_CTRL_EXT_EXTEND_VLAN;
1783         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
1784
1785         /* Update maximum packet length */
1786         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
1787                 E1000_WRITE_REG(hw, E1000_RLPML,
1788                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
1789                                                 2 * VLAN_TAG_SIZE);
1790 }
1791
1792 static void
1793 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1794 {
1795         if(mask & ETH_VLAN_STRIP_MASK){
1796                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1797                         igb_vlan_hw_strip_enable(dev);
1798                 else
1799                         igb_vlan_hw_strip_disable(dev);
1800         }
1801
1802         if(mask & ETH_VLAN_FILTER_MASK){
1803                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1804                         igb_vlan_hw_filter_enable(dev);
1805                 else
1806                         igb_vlan_hw_filter_disable(dev);
1807         }
1808
1809         if(mask & ETH_VLAN_EXTEND_MASK){
1810                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1811                         igb_vlan_hw_extend_enable(dev);
1812                 else
1813                         igb_vlan_hw_extend_disable(dev);
1814         }
1815 }
1816
1817
1818 /**
1819  * It enables the interrupt mask and then enable the interrupt.
1820  *
1821  * @param dev
1822  *  Pointer to struct rte_eth_dev.
1823  *
1824  * @return
1825  *  - On success, zero.
1826  *  - On failure, a negative value.
1827  */
1828 static int
1829 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev)
1830 {
1831         struct e1000_interrupt *intr =
1832                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1833
1834         intr->mask |= E1000_ICR_LSC;
1835
1836         return 0;
1837 }
1838
1839 /*
1840  * It reads ICR and gets interrupt causes, check it and set a bit flag
1841  * to update link status.
1842  *
1843  * @param dev
1844  *  Pointer to struct rte_eth_dev.
1845  *
1846  * @return
1847  *  - On success, zero.
1848  *  - On failure, a negative value.
1849  */
1850 static int
1851 eth_igb_interrupt_get_status(struct rte_eth_dev *dev)
1852 {
1853         uint32_t icr;
1854         struct e1000_hw *hw =
1855                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1856         struct e1000_interrupt *intr =
1857                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1858
1859         igb_intr_disable(hw);
1860
1861         /* read-on-clear nic registers here */
1862         icr = E1000_READ_REG(hw, E1000_ICR);
1863
1864         intr->flags = 0;
1865         if (icr & E1000_ICR_LSC) {
1866                 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
1867         }
1868
1869         if (icr & E1000_ICR_VMMB)
1870                 intr->flags |= E1000_FLAG_MAILBOX;
1871
1872         return 0;
1873 }
1874
1875 /*
1876  * It executes link_update after knowing an interrupt is prsent.
1877  *
1878  * @param dev
1879  *  Pointer to struct rte_eth_dev.
1880  *
1881  * @return
1882  *  - On success, zero.
1883  *  - On failure, a negative value.
1884  */
1885 static int
1886 eth_igb_interrupt_action(struct rte_eth_dev *dev)
1887 {
1888         struct e1000_hw *hw =
1889                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1890         struct e1000_interrupt *intr =
1891                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1892         uint32_t tctl, rctl;
1893         struct rte_eth_link link;
1894         int ret;
1895
1896         if (intr->flags & E1000_FLAG_MAILBOX) {
1897                 igb_pf_mbx_process(dev);
1898                 intr->flags &= ~E1000_FLAG_MAILBOX;
1899         }
1900
1901         igb_intr_enable(dev);
1902         rte_intr_enable(&(dev->pci_dev->intr_handle));
1903
1904         if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) {
1905                 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE;
1906
1907                 /* set get_link_status to check register later */
1908                 hw->mac.get_link_status = 1;
1909                 ret = eth_igb_link_update(dev, 0);
1910
1911                 /* check if link has changed */
1912                 if (ret < 0)
1913                         return 0;
1914
1915                 memset(&link, 0, sizeof(link));
1916                 rte_igb_dev_atomic_read_link_status(dev, &link);
1917                 if (link.link_status) {
1918                         PMD_INIT_LOG(INFO,
1919                                      " Port %d: Link Up - speed %u Mbps - %s",
1920                                      dev->data->port_id,
1921                                      (unsigned)link.link_speed,
1922                                      link.link_duplex == ETH_LINK_FULL_DUPLEX ?
1923                                      "full-duplex" : "half-duplex");
1924                 } else {
1925                         PMD_INIT_LOG(INFO, " Port %d: Link Down",
1926                                      dev->data->port_id);
1927                 }
1928                 PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
1929                              dev->pci_dev->addr.domain,
1930                              dev->pci_dev->addr.bus,
1931                              dev->pci_dev->addr.devid,
1932                              dev->pci_dev->addr.function);
1933                 tctl = E1000_READ_REG(hw, E1000_TCTL);
1934                 rctl = E1000_READ_REG(hw, E1000_RCTL);
1935                 if (link.link_status) {
1936                         /* enable Tx/Rx */
1937                         tctl |= E1000_TCTL_EN;
1938                         rctl |= E1000_RCTL_EN;
1939                 } else {
1940                         /* disable Tx/Rx */
1941                         tctl &= ~E1000_TCTL_EN;
1942                         rctl &= ~E1000_RCTL_EN;
1943                 }
1944                 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
1945                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1946                 E1000_WRITE_FLUSH(hw);
1947                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
1948         }
1949
1950         return 0;
1951 }
1952
1953 /**
1954  * Interrupt handler which shall be registered at first.
1955  *
1956  * @param handle
1957  *  Pointer to interrupt handle.
1958  * @param param
1959  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1960  *
1961  * @return
1962  *  void
1963  */
1964 static void
1965 eth_igb_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
1966                                                         void *param)
1967 {
1968         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1969
1970         eth_igb_interrupt_get_status(dev);
1971         eth_igb_interrupt_action(dev);
1972 }
1973
1974 static int
1975 eth_igb_led_on(struct rte_eth_dev *dev)
1976 {
1977         struct e1000_hw *hw;
1978
1979         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1980         return (e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
1981 }
1982
1983 static int
1984 eth_igb_led_off(struct rte_eth_dev *dev)
1985 {
1986         struct e1000_hw *hw;
1987
1988         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1989         return (e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
1990 }
1991
1992 static int
1993 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1994 {
1995         struct e1000_hw *hw;
1996         uint32_t ctrl;
1997         int tx_pause;
1998         int rx_pause;
1999
2000         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2001         fc_conf->pause_time = hw->fc.pause_time;
2002         fc_conf->high_water = hw->fc.high_water;
2003         fc_conf->low_water = hw->fc.low_water;
2004         fc_conf->send_xon = hw->fc.send_xon;
2005         fc_conf->autoneg = hw->mac.autoneg;
2006
2007         /*
2008          * Return rx_pause and tx_pause status according to actual setting of
2009          * the TFCE and RFCE bits in the CTRL register.
2010          */
2011         ctrl = E1000_READ_REG(hw, E1000_CTRL);
2012         if (ctrl & E1000_CTRL_TFCE)
2013                 tx_pause = 1;
2014         else
2015                 tx_pause = 0;
2016
2017         if (ctrl & E1000_CTRL_RFCE)
2018                 rx_pause = 1;
2019         else
2020                 rx_pause = 0;
2021
2022         if (rx_pause && tx_pause)
2023                 fc_conf->mode = RTE_FC_FULL;
2024         else if (rx_pause)
2025                 fc_conf->mode = RTE_FC_RX_PAUSE;
2026         else if (tx_pause)
2027                 fc_conf->mode = RTE_FC_TX_PAUSE;
2028         else
2029                 fc_conf->mode = RTE_FC_NONE;
2030
2031         return 0;
2032 }
2033
2034 static int
2035 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2036 {
2037         struct e1000_hw *hw;
2038         int err;
2039         enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = {
2040                 e1000_fc_none,
2041                 e1000_fc_rx_pause,
2042                 e1000_fc_tx_pause,
2043                 e1000_fc_full
2044         };
2045         uint32_t rx_buf_size;
2046         uint32_t max_high_water;
2047         uint32_t rctl;
2048
2049         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2050         if (fc_conf->autoneg != hw->mac.autoneg)
2051                 return -ENOTSUP;
2052         rx_buf_size = igb_get_rx_buffer_size(hw);
2053         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2054
2055         /* At least reserve one Ethernet frame for watermark */
2056         max_high_water = rx_buf_size - ETHER_MAX_LEN;
2057         if ((fc_conf->high_water > max_high_water) ||
2058             (fc_conf->high_water < fc_conf->low_water)) {
2059                 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value");
2060                 PMD_INIT_LOG(ERR, "high water must <=  0x%x", max_high_water);
2061                 return (-EINVAL);
2062         }
2063
2064         hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode];
2065         hw->fc.pause_time     = fc_conf->pause_time;
2066         hw->fc.high_water     = fc_conf->high_water;
2067         hw->fc.low_water      = fc_conf->low_water;
2068         hw->fc.send_xon       = fc_conf->send_xon;
2069
2070         err = e1000_setup_link_generic(hw);
2071         if (err == E1000_SUCCESS) {
2072
2073                 /* check if we want to forward MAC frames - driver doesn't have native
2074                  * capability to do that, so we'll write the registers ourselves */
2075
2076                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2077
2078                 /* set or clear MFLCN.PMCF bit depending on configuration */
2079                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2080                         rctl |= E1000_RCTL_PMCF;
2081                 else
2082                         rctl &= ~E1000_RCTL_PMCF;
2083
2084                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2085                 E1000_WRITE_FLUSH(hw);
2086
2087                 return 0;
2088         }
2089
2090         PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err);
2091         return (-EIO);
2092 }
2093
2094 #define E1000_RAH_POOLSEL_SHIFT      (18)
2095 static void
2096 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2097                 uint32_t index, __rte_unused uint32_t pool)
2098 {
2099         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2100         uint32_t rah;
2101
2102         e1000_rar_set(hw, mac_addr->addr_bytes, index);
2103         rah = E1000_READ_REG(hw, E1000_RAH(index));
2104         rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool));
2105         E1000_WRITE_REG(hw, E1000_RAH(index), rah);
2106 }
2107
2108 static void
2109 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index)
2110 {
2111         uint8_t addr[ETHER_ADDR_LEN];
2112         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2113
2114         memset(addr, 0, sizeof(addr));
2115
2116         e1000_rar_set(hw, addr, index);
2117 }
2118
2119 /*
2120  * Virtual Function operations
2121  */
2122 static void
2123 igbvf_intr_disable(struct e1000_hw *hw)
2124 {
2125         PMD_INIT_FUNC_TRACE();
2126
2127         /* Clear interrupt mask to stop from interrupts being generated */
2128         E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF);
2129
2130         E1000_WRITE_FLUSH(hw);
2131 }
2132
2133 static void
2134 igbvf_stop_adapter(struct rte_eth_dev *dev)
2135 {
2136         u32 reg_val;
2137         u16 i;
2138         struct rte_eth_dev_info dev_info;
2139         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2140
2141         memset(&dev_info, 0, sizeof(dev_info));
2142         eth_igbvf_infos_get(dev, &dev_info);
2143
2144         /* Clear interrupt mask to stop from interrupts being generated */
2145         igbvf_intr_disable(hw);
2146
2147         /* Clear any pending interrupts, flush previous writes */
2148         E1000_READ_REG(hw, E1000_EICR);
2149
2150         /* Disable the transmit unit.  Each queue must be disabled. */
2151         for (i = 0; i < dev_info.max_tx_queues; i++)
2152                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH);
2153
2154         /* Disable the receive unit by stopping each queue */
2155         for (i = 0; i < dev_info.max_rx_queues; i++) {
2156                 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i));
2157                 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE;
2158                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val);
2159                 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE)
2160                         ;
2161         }
2162
2163         /* flush all queues disables */
2164         E1000_WRITE_FLUSH(hw);
2165         msec_delay(2);
2166 }
2167
2168 static int eth_igbvf_link_update(struct e1000_hw *hw)
2169 {
2170         struct e1000_mbx_info *mbx = &hw->mbx;
2171         struct e1000_mac_info *mac = &hw->mac;
2172         int ret_val = E1000_SUCCESS;
2173
2174         PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf");
2175
2176         /*
2177          * We only want to run this if there has been a rst asserted.
2178          * in this case that could mean a link change, device reset,
2179          * or a virtual function reset
2180          */
2181
2182         /* If we were hit with a reset or timeout drop the link */
2183         if (!e1000_check_for_rst(hw, 0) || !mbx->timeout)
2184                 mac->get_link_status = TRUE;
2185
2186         if (!mac->get_link_status)
2187                 goto out;
2188
2189         /* if link status is down no point in checking to see if pf is up */
2190         if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU))
2191                 goto out;
2192
2193         /* if we passed all the tests above then the link is up and we no
2194          * longer need to check for link */
2195         mac->get_link_status = FALSE;
2196
2197 out:
2198         return ret_val;
2199 }
2200
2201
2202 static int
2203 igbvf_dev_configure(struct rte_eth_dev *dev)
2204 {
2205         struct rte_eth_conf* conf = &dev->data->dev_conf;
2206
2207         PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
2208                      dev->data->port_id);
2209
2210         /*
2211          * VF has no ability to enable/disable HW CRC
2212          * Keep the persistent behavior the same as Host PF
2213          */
2214 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC
2215         if (!conf->rxmode.hw_strip_crc) {
2216                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip");
2217                 conf->rxmode.hw_strip_crc = 1;
2218         }
2219 #else
2220         if (conf->rxmode.hw_strip_crc) {
2221                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip");
2222                 conf->rxmode.hw_strip_crc = 0;
2223         }
2224 #endif
2225
2226         return 0;
2227 }
2228
2229 static int
2230 igbvf_dev_start(struct rte_eth_dev *dev)
2231 {
2232         struct e1000_hw *hw =
2233                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2234         int ret;
2235
2236         PMD_INIT_FUNC_TRACE();
2237
2238         hw->mac.ops.reset_hw(hw);
2239
2240         /* Set all vfta */
2241         igbvf_set_vfta_all(dev,1);
2242
2243         eth_igbvf_tx_init(dev);
2244
2245         /* This can fail when allocating mbufs for descriptor rings */
2246         ret = eth_igbvf_rx_init(dev);
2247         if (ret) {
2248                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
2249                 igb_dev_clear_queues(dev);
2250                 return ret;
2251         }
2252
2253         return 0;
2254 }
2255
2256 static void
2257 igbvf_dev_stop(struct rte_eth_dev *dev)
2258 {
2259         PMD_INIT_FUNC_TRACE();
2260
2261         igbvf_stop_adapter(dev);
2262
2263         /*
2264           * Clear what we set, but we still keep shadow_vfta to
2265           * restore after device starts
2266           */
2267         igbvf_set_vfta_all(dev,0);
2268
2269         igb_dev_clear_queues(dev);
2270 }
2271
2272 static void
2273 igbvf_dev_close(struct rte_eth_dev *dev)
2274 {
2275         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2276
2277         PMD_INIT_FUNC_TRACE();
2278
2279         e1000_reset_hw(hw);
2280
2281         igbvf_dev_stop(dev);
2282 }
2283
2284 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on)
2285 {
2286         struct e1000_mbx_info *mbx = &hw->mbx;
2287         uint32_t msgbuf[2];
2288
2289         /* After set vlan, vlan strip will also be enabled in igb driver*/
2290         msgbuf[0] = E1000_VF_SET_VLAN;
2291         msgbuf[1] = vid;
2292         /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */
2293         if (on)
2294                 msgbuf[0] |= E1000_VF_SET_VLAN_ADD;
2295
2296         return (mbx->ops.write_posted(hw, msgbuf, 2, 0));
2297 }
2298
2299 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2300 {
2301         struct e1000_hw *hw =
2302                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2303         struct e1000_vfta * shadow_vfta =
2304                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2305         int i = 0, j = 0, vfta = 0, mask = 1;
2306
2307         for (i = 0; i < IGB_VFTA_SIZE; i++){
2308                 vfta = shadow_vfta->vfta[i];
2309                 if(vfta){
2310                         mask = 1;
2311                         for (j = 0; j < 32; j++){
2312                                 if(vfta & mask)
2313                                         igbvf_set_vfta(hw,
2314                                                 (uint16_t)((i<<5)+j), on);
2315                                 mask<<=1;
2316                         }
2317                 }
2318         }
2319
2320 }
2321
2322 static int
2323 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2324 {
2325         struct e1000_hw *hw =
2326                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2327         struct e1000_vfta * shadow_vfta =
2328                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2329         uint32_t vid_idx = 0;
2330         uint32_t vid_bit = 0;
2331         int ret = 0;
2332
2333         PMD_INIT_FUNC_TRACE();
2334
2335         /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/
2336         ret = igbvf_set_vfta(hw, vlan_id, !!on);
2337         if(ret){
2338                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2339                 return ret;
2340         }
2341         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2342         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2343
2344         /*Save what we set and retore it after device reset*/
2345         if (on)
2346                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2347         else
2348                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2349
2350         return 0;
2351 }
2352
2353 static int
2354 eth_igb_rss_reta_update(struct rte_eth_dev *dev,
2355                         struct rte_eth_rss_reta_entry64 *reta_conf,
2356                         uint16_t reta_size)
2357 {
2358         uint8_t i, j, mask;
2359         uint32_t reta, r;
2360         uint16_t idx, shift;
2361         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2362
2363         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2364                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2365                         "(%d) doesn't match the number hardware can supported "
2366                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2367                 return -EINVAL;
2368         }
2369
2370         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
2371                 idx = i / RTE_RETA_GROUP_SIZE;
2372                 shift = i % RTE_RETA_GROUP_SIZE;
2373                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2374                                                 IGB_4_BIT_MASK);
2375                 if (!mask)
2376                         continue;
2377                 if (mask == IGB_4_BIT_MASK)
2378                         r = 0;
2379                 else
2380                         r = E1000_READ_REG(hw, E1000_RETA(i >> 2));
2381                 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) {
2382                         if (mask & (0x1 << j))
2383                                 reta |= reta_conf[idx].reta[shift + j] <<
2384                                                         (CHAR_BIT * j);
2385                         else
2386                                 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j));
2387                 }
2388                 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta);
2389         }
2390
2391         return 0;
2392 }
2393
2394 static int
2395 eth_igb_rss_reta_query(struct rte_eth_dev *dev,
2396                        struct rte_eth_rss_reta_entry64 *reta_conf,
2397                        uint16_t reta_size)
2398 {
2399         uint8_t i, j, mask;
2400         uint32_t reta;
2401         uint16_t idx, shift;
2402         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2403
2404         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2405                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2406                         "(%d) doesn't match the number hardware can supported "
2407                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2408                 return -EINVAL;
2409         }
2410
2411         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
2412                 idx = i / RTE_RETA_GROUP_SIZE;
2413                 shift = i % RTE_RETA_GROUP_SIZE;
2414                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2415                                                 IGB_4_BIT_MASK);
2416                 if (!mask)
2417                         continue;
2418                 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2));
2419                 for (j = 0; j < IGB_4_BIT_WIDTH; j++) {
2420                         if (mask & (0x1 << j))
2421                                 reta_conf[idx].reta[shift + j] =
2422                                         ((reta >> (CHAR_BIT * j)) &
2423                                                 IGB_8_BIT_MASK);
2424                 }
2425         }
2426
2427         return 0;
2428 }
2429
2430 #define MAC_TYPE_FILTER_SUP(type)    do {\
2431         if ((type) != e1000_82580 && (type) != e1000_i350 &&\
2432                 (type) != e1000_82576)\
2433                 return -ENOTSUP;\
2434 } while (0)
2435
2436 static int
2437 eth_igb_syn_filter_set(struct rte_eth_dev *dev,
2438                         struct rte_eth_syn_filter *filter,
2439                         bool add)
2440 {
2441         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2442         uint32_t synqf, rfctl;
2443
2444         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
2445                 return -EINVAL;
2446
2447         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
2448
2449         if (add) {
2450                 if (synqf & E1000_SYN_FILTER_ENABLE)
2451                         return -EINVAL;
2452
2453                 synqf = (uint32_t)(((filter->queue << E1000_SYN_FILTER_QUEUE_SHIFT) &
2454                         E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE);
2455
2456                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
2457                 if (filter->hig_pri)
2458                         rfctl |= E1000_RFCTL_SYNQFP;
2459                 else
2460                         rfctl &= ~E1000_RFCTL_SYNQFP;
2461
2462                 E1000_WRITE_REG(hw, E1000_RFCTL, rfctl);
2463         } else {
2464                 if (!(synqf & E1000_SYN_FILTER_ENABLE))
2465                         return -ENOENT;
2466                 synqf = 0;
2467         }
2468
2469         E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf);
2470         E1000_WRITE_FLUSH(hw);
2471         return 0;
2472 }
2473
2474 static int
2475 eth_igb_syn_filter_get(struct rte_eth_dev *dev,
2476                         struct rte_eth_syn_filter *filter)
2477 {
2478         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2479         uint32_t synqf, rfctl;
2480
2481         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
2482         if (synqf & E1000_SYN_FILTER_ENABLE) {
2483                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
2484                 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0;
2485                 filter->queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >>
2486                                 E1000_SYN_FILTER_QUEUE_SHIFT);
2487                 return 0;
2488         }
2489
2490         return -ENOENT;
2491 }
2492
2493 static int
2494 eth_igb_syn_filter_handle(struct rte_eth_dev *dev,
2495                         enum rte_filter_op filter_op,
2496                         void *arg)
2497 {
2498         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2499         int ret;
2500
2501         MAC_TYPE_FILTER_SUP(hw->mac.type);
2502
2503         if (filter_op == RTE_ETH_FILTER_NOP)
2504                 return 0;
2505
2506         if (arg == NULL) {
2507                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
2508                             filter_op);
2509                 return -EINVAL;
2510         }
2511
2512         switch (filter_op) {
2513         case RTE_ETH_FILTER_ADD:
2514                 ret = eth_igb_syn_filter_set(dev,
2515                                 (struct rte_eth_syn_filter *)arg,
2516                                 TRUE);
2517                 break;
2518         case RTE_ETH_FILTER_DELETE:
2519                 ret = eth_igb_syn_filter_set(dev,
2520                                 (struct rte_eth_syn_filter *)arg,
2521                                 FALSE);
2522                 break;
2523         case RTE_ETH_FILTER_GET:
2524                 ret = eth_igb_syn_filter_get(dev,
2525                                 (struct rte_eth_syn_filter *)arg);
2526                 break;
2527         default:
2528                 PMD_DRV_LOG(ERR, "unsupported operation %u\n", filter_op);
2529                 ret = -EINVAL;
2530                 break;
2531         }
2532
2533         return ret;
2534 }
2535
2536 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
2537         if ((type) != e1000_82580 && (type) != e1000_i350)\
2538                 return -ENOSYS; \
2539 } while (0)
2540
2541 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_2tuple_filter_info*/
2542 static inline int
2543 ntuple_filter_to_2tuple(struct rte_eth_ntuple_filter *filter,
2544                         struct e1000_2tuple_filter_info *filter_info)
2545 {
2546         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM)
2547                 return -EINVAL;
2548         if (filter->priority > E1000_2TUPLE_MAX_PRI)
2549                 return -EINVAL;  /* filter index is out of range. */
2550         if (filter->tcp_flags > TCP_FLAG_ALL)
2551                 return -EINVAL;  /* flags is invalid. */
2552
2553         switch (filter->dst_port_mask) {
2554         case UINT16_MAX:
2555                 filter_info->dst_port_mask = 0;
2556                 filter_info->dst_port = filter->dst_port;
2557                 break;
2558         case 0:
2559                 filter_info->dst_port_mask = 1;
2560                 break;
2561         default:
2562                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
2563                 return -EINVAL;
2564         }
2565
2566         switch (filter->proto_mask) {
2567         case UINT8_MAX:
2568                 filter_info->proto_mask = 0;
2569                 filter_info->proto = filter->proto;
2570                 break;
2571         case 0:
2572                 filter_info->proto_mask = 1;
2573                 break;
2574         default:
2575                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
2576                 return -EINVAL;
2577         }
2578
2579         filter_info->priority = (uint8_t)filter->priority;
2580         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
2581                 filter_info->tcp_flags = filter->tcp_flags;
2582         else
2583                 filter_info->tcp_flags = 0;
2584
2585         return 0;
2586 }
2587
2588 static inline struct e1000_2tuple_filter *
2589 igb_2tuple_filter_lookup(struct e1000_2tuple_filter_list *filter_list,
2590                         struct e1000_2tuple_filter_info *key)
2591 {
2592         struct e1000_2tuple_filter *it;
2593
2594         TAILQ_FOREACH(it, filter_list, entries) {
2595                 if (memcmp(key, &it->filter_info,
2596                         sizeof(struct e1000_2tuple_filter_info)) == 0) {
2597                         return it;
2598                 }
2599         }
2600         return NULL;
2601 }
2602
2603 /*
2604  * igb_add_2tuple_filter - add a 2tuple filter
2605  *
2606  * @param
2607  * dev: Pointer to struct rte_eth_dev.
2608  * ntuple_filter: ponter to the filter that will be added.
2609  *
2610  * @return
2611  *    - On success, zero.
2612  *    - On failure, a negative value.
2613  */
2614 static int
2615 igb_add_2tuple_filter(struct rte_eth_dev *dev,
2616                         struct rte_eth_ntuple_filter *ntuple_filter)
2617 {
2618         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2619         struct e1000_filter_info *filter_info =
2620                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
2621         struct e1000_2tuple_filter *filter;
2622         uint32_t ttqf = E1000_TTQF_DISABLE_MASK;
2623         uint32_t imir, imir_ext = E1000_IMIREXT_SIZE_BP;
2624         int i, ret;
2625
2626         filter = rte_zmalloc("e1000_2tuple_filter",
2627                         sizeof(struct e1000_2tuple_filter), 0);
2628         if (filter == NULL)
2629                 return -ENOMEM;
2630
2631         ret = ntuple_filter_to_2tuple(ntuple_filter,
2632                                       &filter->filter_info);
2633         if (ret < 0) {
2634                 rte_free(filter);
2635                 return ret;
2636         }
2637         if (igb_2tuple_filter_lookup(&filter_info->twotuple_list,
2638                                          &filter->filter_info) != NULL) {
2639                 PMD_DRV_LOG(ERR, "filter exists.");
2640                 rte_free(filter);
2641                 return -EEXIST;
2642         }
2643         filter->queue = ntuple_filter->queue;
2644
2645         /*
2646          * look for an unused 2tuple filter index,
2647          * and insert the filter to list.
2648          */
2649         for (i = 0; i < E1000_MAX_TTQF_FILTERS; i++) {
2650                 if (!(filter_info->twotuple_mask & (1 << i))) {
2651                         filter_info->twotuple_mask |= 1 << i;
2652                         filter->index = i;
2653                         TAILQ_INSERT_TAIL(&filter_info->twotuple_list,
2654                                           filter,
2655                                           entries);
2656                         break;
2657                 }
2658         }
2659         if (i >= E1000_MAX_TTQF_FILTERS) {
2660                 PMD_DRV_LOG(ERR, "2tuple filters are full.");
2661                 rte_free(filter);
2662                 return -ENOSYS;
2663         }
2664
2665         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
2666         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
2667                 imir |= E1000_IMIR_PORT_BP;
2668         else
2669                 imir &= ~E1000_IMIR_PORT_BP;
2670
2671         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
2672
2673         ttqf |= E1000_TTQF_QUEUE_ENABLE;
2674         ttqf |= (uint32_t)(filter->queue << E1000_TTQF_QUEUE_SHIFT);
2675         ttqf |= (uint32_t)(filter->filter_info.proto & E1000_TTQF_PROTOCOL_MASK);
2676         if (filter->filter_info.proto_mask == 0)
2677                 ttqf &= ~E1000_TTQF_MASK_ENABLE;
2678
2679         /* tcp flags bits setting. */
2680         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
2681                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
2682                         imir_ext |= E1000_IMIREXT_CTRL_URG;
2683                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
2684                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
2685                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
2686                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
2687                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
2688                         imir_ext |= E1000_IMIREXT_CTRL_RST;
2689                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
2690                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
2691                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
2692                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
2693         } else
2694                 imir_ext |= E1000_IMIREXT_CTRL_BP;
2695         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
2696         E1000_WRITE_REG(hw, E1000_TTQF(i), ttqf);
2697         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
2698         return 0;
2699 }
2700
2701 /*
2702  * igb_remove_2tuple_filter - remove a 2tuple filter
2703  *
2704  * @param
2705  * dev: Pointer to struct rte_eth_dev.
2706  * ntuple_filter: ponter to the filter that will be removed.
2707  *
2708  * @return
2709  *    - On success, zero.
2710  *    - On failure, a negative value.
2711  */
2712 static int
2713 igb_remove_2tuple_filter(struct rte_eth_dev *dev,
2714                         struct rte_eth_ntuple_filter *ntuple_filter)
2715 {
2716         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2717         struct e1000_filter_info *filter_info =
2718                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
2719         struct e1000_2tuple_filter_info filter_2tuple;
2720         struct e1000_2tuple_filter *filter;
2721         int ret;
2722
2723         memset(&filter_2tuple, 0, sizeof(struct e1000_2tuple_filter_info));
2724         ret = ntuple_filter_to_2tuple(ntuple_filter,
2725                                       &filter_2tuple);
2726         if (ret < 0)
2727                 return ret;
2728
2729         filter = igb_2tuple_filter_lookup(&filter_info->twotuple_list,
2730                                          &filter_2tuple);
2731         if (filter == NULL) {
2732                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
2733                 return -ENOENT;
2734         }
2735
2736         filter_info->twotuple_mask &= ~(1 << filter->index);
2737         TAILQ_REMOVE(&filter_info->twotuple_list, filter, entries);
2738         rte_free(filter);
2739
2740         E1000_WRITE_REG(hw, E1000_TTQF(filter->index), E1000_TTQF_DISABLE_MASK);
2741         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
2742         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
2743         return 0;
2744 }
2745
2746 static inline struct e1000_flex_filter *
2747 eth_igb_flex_filter_lookup(struct e1000_flex_filter_list *filter_list,
2748                         struct e1000_flex_filter_info *key)
2749 {
2750         struct e1000_flex_filter *it;
2751
2752         TAILQ_FOREACH(it, filter_list, entries) {
2753                 if (memcmp(key, &it->filter_info,
2754                         sizeof(struct e1000_flex_filter_info)) == 0)
2755                         return it;
2756         }
2757
2758         return NULL;
2759 }
2760
2761 static int
2762 eth_igb_add_del_flex_filter(struct rte_eth_dev *dev,
2763                         struct rte_eth_flex_filter *filter,
2764                         bool add)
2765 {
2766         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2767         struct e1000_filter_info *filter_info =
2768                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
2769         struct e1000_flex_filter *flex_filter, *it;
2770         uint32_t wufc, queueing, mask;
2771         uint32_t reg_off;
2772         uint8_t shift, i, j = 0;
2773
2774         flex_filter = rte_zmalloc("e1000_flex_filter",
2775                         sizeof(struct e1000_flex_filter), 0);
2776         if (flex_filter == NULL)
2777                 return -ENOMEM;
2778
2779         flex_filter->filter_info.len = filter->len;
2780         flex_filter->filter_info.priority = filter->priority;
2781         memcpy(flex_filter->filter_info.dwords, filter->bytes, filter->len);
2782         for (i = 0; i < RTE_ALIGN(filter->len, CHAR_BIT) / CHAR_BIT; i++) {
2783                 mask = 0;
2784                 /* reverse bits in flex filter's mask*/
2785                 for (shift = 0; shift < CHAR_BIT; shift++) {
2786                         if (filter->mask[i] & (0x01 << shift))
2787                                 mask |= (0x80 >> shift);
2788                 }
2789                 flex_filter->filter_info.mask[i] = mask;
2790         }
2791
2792         wufc = E1000_READ_REG(hw, E1000_WUFC);
2793         if (flex_filter->index < E1000_MAX_FHFT)
2794                 reg_off = E1000_FHFT(flex_filter->index);
2795         else
2796                 reg_off = E1000_FHFT_EXT(flex_filter->index - E1000_MAX_FHFT);
2797
2798         if (add) {
2799                 if (eth_igb_flex_filter_lookup(&filter_info->flex_list,
2800                                 &flex_filter->filter_info) != NULL) {
2801                         PMD_DRV_LOG(ERR, "filter exists.");
2802                         rte_free(flex_filter);
2803                         return -EEXIST;
2804                 }
2805                 flex_filter->queue = filter->queue;
2806                 /*
2807                  * look for an unused flex filter index
2808                  * and insert the filter into the list.
2809                  */
2810                 for (i = 0; i < E1000_MAX_FLEX_FILTERS; i++) {
2811                         if (!(filter_info->flex_mask & (1 << i))) {
2812                                 filter_info->flex_mask |= 1 << i;
2813                                 flex_filter->index = i;
2814                                 TAILQ_INSERT_TAIL(&filter_info->flex_list,
2815                                         flex_filter,
2816                                         entries);
2817                                 break;
2818                         }
2819                 }
2820                 if (i >= E1000_MAX_FLEX_FILTERS) {
2821                         PMD_DRV_LOG(ERR, "flex filters are full.");
2822                         rte_free(flex_filter);
2823                         return -ENOSYS;
2824                 }
2825
2826                 E1000_WRITE_REG(hw, E1000_WUFC, wufc | E1000_WUFC_FLEX_HQ |
2827                                 (E1000_WUFC_FLX0 << flex_filter->index));
2828                 queueing = filter->len |
2829                         (filter->queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) |
2830                         (filter->priority << E1000_FHFT_QUEUEING_PRIO_SHIFT);
2831                 E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET,
2832                                 queueing);
2833                 for (i = 0; i < E1000_FLEX_FILTERS_MASK_SIZE; i++) {
2834                         E1000_WRITE_REG(hw, reg_off,
2835                                         flex_filter->filter_info.dwords[j]);
2836                         reg_off += sizeof(uint32_t);
2837                         E1000_WRITE_REG(hw, reg_off,
2838                                         flex_filter->filter_info.dwords[++j]);
2839                         reg_off += sizeof(uint32_t);
2840                         E1000_WRITE_REG(hw, reg_off,
2841                                 (uint32_t)flex_filter->filter_info.mask[i]);
2842                         reg_off += sizeof(uint32_t) * 2;
2843                         ++j;
2844                 }
2845         } else {
2846                 it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
2847                                 &flex_filter->filter_info);
2848                 if (it == NULL) {
2849                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
2850                         rte_free(flex_filter);
2851                         return -ENOENT;
2852                 }
2853
2854                 for (i = 0; i < E1000_FHFT_SIZE_IN_DWD; i++)
2855                         E1000_WRITE_REG(hw, reg_off + i * sizeof(uint32_t), 0);
2856                 E1000_WRITE_REG(hw, E1000_WUFC, wufc &
2857                         (~(E1000_WUFC_FLX0 << it->index)));
2858
2859                 filter_info->flex_mask &= ~(1 << it->index);
2860                 TAILQ_REMOVE(&filter_info->flex_list, it, entries);
2861                 rte_free(it);
2862                 rte_free(flex_filter);
2863         }
2864
2865         return 0;
2866 }
2867
2868 static int
2869 eth_igb_get_flex_filter(struct rte_eth_dev *dev,
2870                         struct rte_eth_flex_filter *filter)
2871 {
2872         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2873         struct e1000_filter_info *filter_info =
2874                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
2875         struct e1000_flex_filter flex_filter, *it;
2876         uint32_t wufc, queueing, wufc_en = 0;
2877
2878         memset(&flex_filter, 0, sizeof(struct e1000_flex_filter));
2879         flex_filter.filter_info.len = filter->len;
2880         flex_filter.filter_info.priority = filter->priority;
2881         memcpy(flex_filter.filter_info.dwords, filter->bytes, filter->len);
2882         memcpy(flex_filter.filter_info.mask, filter->mask,
2883                         RTE_ALIGN(filter->len, sizeof(char)) / sizeof(char));
2884
2885         it = eth_igb_flex_filter_lookup(&filter_info->flex_list,
2886                                 &flex_filter.filter_info);
2887         if (it == NULL) {
2888                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
2889                 return -ENOENT;
2890         }
2891
2892         wufc = E1000_READ_REG(hw, E1000_WUFC);
2893         wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << it->index);
2894
2895         if ((wufc & wufc_en) == wufc_en) {
2896                 uint32_t reg_off = 0;
2897                 if (it->index < E1000_MAX_FHFT)
2898                         reg_off = E1000_FHFT(it->index);
2899                 else
2900                         reg_off = E1000_FHFT_EXT(it->index - E1000_MAX_FHFT);
2901
2902                 queueing = E1000_READ_REG(hw,
2903                                 reg_off + E1000_FHFT_QUEUEING_OFFSET);
2904                 filter->len = queueing & E1000_FHFT_QUEUEING_LEN;
2905                 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >>
2906                         E1000_FHFT_QUEUEING_PRIO_SHIFT;
2907                 filter->queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >>
2908                         E1000_FHFT_QUEUEING_QUEUE_SHIFT;
2909                 return 0;
2910         }
2911         return -ENOENT;
2912 }
2913
2914 static int
2915 eth_igb_flex_filter_handle(struct rte_eth_dev *dev,
2916                         enum rte_filter_op filter_op,
2917                         void *arg)
2918 {
2919         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2920         struct rte_eth_flex_filter *filter;
2921         int ret = 0;
2922
2923         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
2924
2925         if (filter_op == RTE_ETH_FILTER_NOP)
2926                 return ret;
2927
2928         if (arg == NULL) {
2929                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u",
2930                             filter_op);
2931                 return -EINVAL;
2932         }
2933
2934         filter = (struct rte_eth_flex_filter *)arg;
2935         if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN
2936             || filter->len % sizeof(uint64_t) != 0) {
2937                 PMD_DRV_LOG(ERR, "filter's length is out of range");
2938                 return -EINVAL;
2939         }
2940         if (filter->priority > E1000_MAX_FLEX_FILTER_PRI) {
2941                 PMD_DRV_LOG(ERR, "filter's priority is out of range");
2942                 return -EINVAL;
2943         }
2944
2945         switch (filter_op) {
2946         case RTE_ETH_FILTER_ADD:
2947                 ret = eth_igb_add_del_flex_filter(dev, filter, TRUE);
2948                 break;
2949         case RTE_ETH_FILTER_DELETE:
2950                 ret = eth_igb_add_del_flex_filter(dev, filter, FALSE);
2951                 break;
2952         case RTE_ETH_FILTER_GET:
2953                 ret = eth_igb_get_flex_filter(dev, filter);
2954                 break;
2955         default:
2956                 PMD_DRV_LOG(ERR, "unsupported operation %u", filter_op);
2957                 ret = -EINVAL;
2958                 break;
2959         }
2960
2961         return ret;
2962 }
2963
2964 /* translate elements in struct rte_eth_ntuple_filter to struct e1000_5tuple_filter_info*/
2965 static inline int
2966 ntuple_filter_to_5tuple_82576(struct rte_eth_ntuple_filter *filter,
2967                         struct e1000_5tuple_filter_info *filter_info)
2968 {
2969         if (filter->queue >= IGB_MAX_RX_QUEUE_NUM_82576)
2970                 return -EINVAL;
2971         if (filter->priority > E1000_2TUPLE_MAX_PRI)
2972                 return -EINVAL;  /* filter index is out of range. */
2973         if (filter->tcp_flags > TCP_FLAG_ALL)
2974                 return -EINVAL;  /* flags is invalid. */
2975
2976         switch (filter->dst_ip_mask) {
2977         case UINT32_MAX:
2978                 filter_info->dst_ip_mask = 0;
2979                 filter_info->dst_ip = filter->dst_ip;
2980                 break;
2981         case 0:
2982                 filter_info->dst_ip_mask = 1;
2983                 break;
2984         default:
2985                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
2986                 return -EINVAL;
2987         }
2988
2989         switch (filter->src_ip_mask) {
2990         case UINT32_MAX:
2991                 filter_info->src_ip_mask = 0;
2992                 filter_info->src_ip = filter->src_ip;
2993                 break;
2994         case 0:
2995                 filter_info->src_ip_mask = 1;
2996                 break;
2997         default:
2998                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
2999                 return -EINVAL;
3000         }
3001
3002         switch (filter->dst_port_mask) {
3003         case UINT16_MAX:
3004                 filter_info->dst_port_mask = 0;
3005                 filter_info->dst_port = filter->dst_port;
3006                 break;
3007         case 0:
3008                 filter_info->dst_port_mask = 1;
3009                 break;
3010         default:
3011                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
3012                 return -EINVAL;
3013         }
3014
3015         switch (filter->src_port_mask) {
3016         case UINT16_MAX:
3017                 filter_info->src_port_mask = 0;
3018                 filter_info->src_port = filter->src_port;
3019                 break;
3020         case 0:
3021                 filter_info->src_port_mask = 1;
3022                 break;
3023         default:
3024                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
3025                 return -EINVAL;
3026         }
3027
3028         switch (filter->proto_mask) {
3029         case UINT8_MAX:
3030                 filter_info->proto_mask = 0;
3031                 filter_info->proto = filter->proto;
3032                 break;
3033         case 0:
3034                 filter_info->proto_mask = 1;
3035                 break;
3036         default:
3037                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
3038                 return -EINVAL;
3039         }
3040
3041         filter_info->priority = (uint8_t)filter->priority;
3042         if (filter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG)
3043                 filter_info->tcp_flags = filter->tcp_flags;
3044         else
3045                 filter_info->tcp_flags = 0;
3046
3047         return 0;
3048 }
3049
3050 static inline struct e1000_5tuple_filter *
3051 igb_5tuple_filter_lookup_82576(struct e1000_5tuple_filter_list *filter_list,
3052                         struct e1000_5tuple_filter_info *key)
3053 {
3054         struct e1000_5tuple_filter *it;
3055
3056         TAILQ_FOREACH(it, filter_list, entries) {
3057                 if (memcmp(key, &it->filter_info,
3058                         sizeof(struct e1000_5tuple_filter_info)) == 0) {
3059                         return it;
3060                 }
3061         }
3062         return NULL;
3063 }
3064
3065 /*
3066  * igb_add_5tuple_filter_82576 - add a 5tuple filter
3067  *
3068  * @param
3069  * dev: Pointer to struct rte_eth_dev.
3070  * ntuple_filter: ponter to the filter that will be added.
3071  *
3072  * @return
3073  *    - On success, zero.
3074  *    - On failure, a negative value.
3075  */
3076 static int
3077 igb_add_5tuple_filter_82576(struct rte_eth_dev *dev,
3078                         struct rte_eth_ntuple_filter *ntuple_filter)
3079 {
3080         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3081         struct e1000_filter_info *filter_info =
3082                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3083         struct e1000_5tuple_filter *filter;
3084         uint32_t ftqf = E1000_FTQF_VF_BP | E1000_FTQF_MASK;
3085         uint32_t spqf, imir, imir_ext = E1000_IMIREXT_SIZE_BP;
3086         uint8_t i;
3087         int ret;
3088
3089         filter = rte_zmalloc("e1000_5tuple_filter",
3090                         sizeof(struct e1000_5tuple_filter), 0);
3091         if (filter == NULL)
3092                 return -ENOMEM;
3093
3094         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3095                                             &filter->filter_info);
3096         if (ret < 0) {
3097                 rte_free(filter);
3098                 return ret;
3099         }
3100
3101         if (igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
3102                                          &filter->filter_info) != NULL) {
3103                 PMD_DRV_LOG(ERR, "filter exists.");
3104                 rte_free(filter);
3105                 return -EEXIST;
3106         }
3107         filter->queue = ntuple_filter->queue;
3108
3109         /*
3110          * look for an unused 5tuple filter index,
3111          * and insert the filter to list.
3112          */
3113         for (i = 0; i < E1000_MAX_FTQF_FILTERS; i++) {
3114                 if (!(filter_info->fivetuple_mask & (1 << i))) {
3115                         filter_info->fivetuple_mask |= 1 << i;
3116                         filter->index = i;
3117                         TAILQ_INSERT_TAIL(&filter_info->fivetuple_list,
3118                                           filter,
3119                                           entries);
3120                         break;
3121                 }
3122         }
3123         if (i >= E1000_MAX_FTQF_FILTERS) {
3124                 PMD_DRV_LOG(ERR, "5tuple filters are full.");
3125                 rte_free(filter);
3126                 return -ENOSYS;
3127         }
3128
3129         ftqf |= filter->filter_info.proto & E1000_FTQF_PROTOCOL_MASK;
3130         if (filter->filter_info.src_ip_mask == 0) /* 0b means compare. */
3131                 ftqf &= ~E1000_FTQF_MASK_SOURCE_ADDR_BP;
3132         if (filter->filter_info.dst_ip_mask == 0)
3133                 ftqf &= ~E1000_FTQF_MASK_DEST_ADDR_BP;
3134         if (filter->filter_info.src_port_mask == 0)
3135                 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
3136         if (filter->filter_info.proto_mask == 0)
3137                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP;
3138         ftqf |= (filter->queue << E1000_FTQF_QUEUE_SHIFT) &
3139                 E1000_FTQF_QUEUE_MASK;
3140         ftqf |= E1000_FTQF_QUEUE_ENABLE;
3141         E1000_WRITE_REG(hw, E1000_FTQF(i), ftqf);
3142         E1000_WRITE_REG(hw, E1000_DAQF(i), filter->filter_info.dst_ip);
3143         E1000_WRITE_REG(hw, E1000_SAQF(i), filter->filter_info.src_ip);
3144
3145         spqf = filter->filter_info.src_port & E1000_SPQF_SRCPORT;
3146         E1000_WRITE_REG(hw, E1000_SPQF(i), spqf);
3147
3148         imir = (uint32_t)(filter->filter_info.dst_port & E1000_IMIR_DSTPORT);
3149         if (filter->filter_info.dst_port_mask == 1) /* 1b means not compare. */
3150                 imir |= E1000_IMIR_PORT_BP;
3151         else
3152                 imir &= ~E1000_IMIR_PORT_BP;
3153         imir |= filter->filter_info.priority << E1000_IMIR_PRIORITY_SHIFT;
3154
3155         /* tcp flags bits setting. */
3156         if (filter->filter_info.tcp_flags & TCP_FLAG_ALL) {
3157                 if (filter->filter_info.tcp_flags & TCP_URG_FLAG)
3158                         imir_ext |= E1000_IMIREXT_CTRL_URG;
3159                 if (filter->filter_info.tcp_flags & TCP_ACK_FLAG)
3160                         imir_ext |= E1000_IMIREXT_CTRL_ACK;
3161                 if (filter->filter_info.tcp_flags & TCP_PSH_FLAG)
3162                         imir_ext |= E1000_IMIREXT_CTRL_PSH;
3163                 if (filter->filter_info.tcp_flags & TCP_RST_FLAG)
3164                         imir_ext |= E1000_IMIREXT_CTRL_RST;
3165                 if (filter->filter_info.tcp_flags & TCP_SYN_FLAG)
3166                         imir_ext |= E1000_IMIREXT_CTRL_SYN;
3167                 if (filter->filter_info.tcp_flags & TCP_FIN_FLAG)
3168                         imir_ext |= E1000_IMIREXT_CTRL_FIN;
3169         } else
3170                 imir_ext |= E1000_IMIREXT_CTRL_BP;
3171         E1000_WRITE_REG(hw, E1000_IMIR(i), imir);
3172         E1000_WRITE_REG(hw, E1000_IMIREXT(i), imir_ext);
3173         return 0;
3174 }
3175
3176 /*
3177  * igb_remove_5tuple_filter_82576 - remove a 5tuple filter
3178  *
3179  * @param
3180  * dev: Pointer to struct rte_eth_dev.
3181  * ntuple_filter: ponter to the filter that will be removed.
3182  *
3183  * @return
3184  *    - On success, zero.
3185  *    - On failure, a negative value.
3186  */
3187 static int
3188 igb_remove_5tuple_filter_82576(struct rte_eth_dev *dev,
3189                                 struct rte_eth_ntuple_filter *ntuple_filter)
3190 {
3191         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3192         struct e1000_filter_info *filter_info =
3193                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3194         struct e1000_5tuple_filter_info filter_5tuple;
3195         struct e1000_5tuple_filter *filter;
3196         int ret;
3197
3198         memset(&filter_5tuple, 0, sizeof(struct e1000_5tuple_filter_info));
3199         ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3200                                             &filter_5tuple);
3201         if (ret < 0)
3202                 return ret;
3203
3204         filter = igb_5tuple_filter_lookup_82576(&filter_info->fivetuple_list,
3205                                          &filter_5tuple);
3206         if (filter == NULL) {
3207                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3208                 return -ENOENT;
3209         }
3210
3211         filter_info->fivetuple_mask &= ~(1 << filter->index);
3212         TAILQ_REMOVE(&filter_info->fivetuple_list, filter, entries);
3213         rte_free(filter);
3214
3215         E1000_WRITE_REG(hw, E1000_FTQF(filter->index),
3216                         E1000_FTQF_VF_BP | E1000_FTQF_MASK);
3217         E1000_WRITE_REG(hw, E1000_DAQF(filter->index), 0);
3218         E1000_WRITE_REG(hw, E1000_SAQF(filter->index), 0);
3219         E1000_WRITE_REG(hw, E1000_SPQF(filter->index), 0);
3220         E1000_WRITE_REG(hw, E1000_IMIR(filter->index), 0);
3221         E1000_WRITE_REG(hw, E1000_IMIREXT(filter->index), 0);
3222         return 0;
3223 }
3224
3225 static int
3226 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
3227 {
3228         uint32_t rctl;
3229         struct e1000_hw *hw;
3230         struct rte_eth_dev_info dev_info;
3231         uint32_t frame_size = mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN +
3232                                      VLAN_TAG_SIZE);
3233
3234         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3235
3236 #ifdef RTE_LIBRTE_82571_SUPPORT
3237         /* XXX: not bigger than max_rx_pktlen */
3238         if (hw->mac.type == e1000_82571)
3239                 return -ENOTSUP;
3240 #endif
3241         eth_igb_infos_get(dev, &dev_info);
3242
3243         /* check that mtu is within the allowed range */
3244         if ((mtu < ETHER_MIN_MTU) ||
3245             (frame_size > dev_info.max_rx_pktlen))
3246                 return -EINVAL;
3247
3248         /* refuse mtu that requires the support of scattered packets when this
3249          * feature has not been enabled before. */
3250         if (!dev->data->scattered_rx &&
3251             frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)
3252                 return -EINVAL;
3253
3254         rctl = E1000_READ_REG(hw, E1000_RCTL);
3255
3256         /* switch to jumbo mode if needed */
3257         if (frame_size > ETHER_MAX_LEN) {
3258                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
3259                 rctl |= E1000_RCTL_LPE;
3260         } else {
3261                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
3262                 rctl &= ~E1000_RCTL_LPE;
3263         }
3264         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3265
3266         /* update max frame size */
3267         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
3268
3269         E1000_WRITE_REG(hw, E1000_RLPML,
3270                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
3271
3272         return 0;
3273 }
3274
3275 /*
3276  * igb_add_del_ntuple_filter - add or delete a ntuple filter
3277  *
3278  * @param
3279  * dev: Pointer to struct rte_eth_dev.
3280  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
3281  * add: if true, add filter, if false, remove filter
3282  *
3283  * @return
3284  *    - On success, zero.
3285  *    - On failure, a negative value.
3286  */
3287 static int
3288 igb_add_del_ntuple_filter(struct rte_eth_dev *dev,
3289                         struct rte_eth_ntuple_filter *ntuple_filter,
3290                         bool add)
3291 {
3292         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3293         int ret;
3294
3295         switch (ntuple_filter->flags) {
3296         case RTE_5TUPLE_FLAGS:
3297         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3298                 if (hw->mac.type != e1000_82576)
3299                         return -ENOTSUP;
3300                 if (add)
3301                         ret = igb_add_5tuple_filter_82576(dev,
3302                                                           ntuple_filter);
3303                 else
3304                         ret = igb_remove_5tuple_filter_82576(dev,
3305                                                              ntuple_filter);
3306                 break;
3307         case RTE_2TUPLE_FLAGS:
3308         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3309                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
3310                         return -ENOTSUP;
3311                 if (add)
3312                         ret = igb_add_2tuple_filter(dev, ntuple_filter);
3313                 else
3314                         ret = igb_remove_2tuple_filter(dev, ntuple_filter);
3315                 break;
3316         default:
3317                 ret = -EINVAL;
3318                 break;
3319         }
3320
3321         return ret;
3322 }
3323
3324 /*
3325  * igb_get_ntuple_filter - get a ntuple filter
3326  *
3327  * @param
3328  * dev: Pointer to struct rte_eth_dev.
3329  * ntuple_filter: Pointer to struct rte_eth_ntuple_filter
3330  *
3331  * @return
3332  *    - On success, zero.
3333  *    - On failure, a negative value.
3334  */
3335 static int
3336 igb_get_ntuple_filter(struct rte_eth_dev *dev,
3337                         struct rte_eth_ntuple_filter *ntuple_filter)
3338 {
3339         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3340         struct e1000_filter_info *filter_info =
3341                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3342         struct e1000_5tuple_filter_info filter_5tuple;
3343         struct e1000_2tuple_filter_info filter_2tuple;
3344         struct e1000_5tuple_filter *p_5tuple_filter;
3345         struct e1000_2tuple_filter *p_2tuple_filter;
3346         int ret;
3347
3348         switch (ntuple_filter->flags) {
3349         case RTE_5TUPLE_FLAGS:
3350         case (RTE_5TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3351                 if (hw->mac.type != e1000_82576)
3352                         return -ENOTSUP;
3353                 memset(&filter_5tuple,
3354                         0,
3355                         sizeof(struct e1000_5tuple_filter_info));
3356                 ret = ntuple_filter_to_5tuple_82576(ntuple_filter,
3357                                                     &filter_5tuple);
3358                 if (ret < 0)
3359                         return ret;
3360                 p_5tuple_filter = igb_5tuple_filter_lookup_82576(
3361                                         &filter_info->fivetuple_list,
3362                                         &filter_5tuple);
3363                 if (p_5tuple_filter == NULL) {
3364                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3365                         return -ENOENT;
3366                 }
3367                 ntuple_filter->queue = p_5tuple_filter->queue;
3368                 break;
3369         case RTE_2TUPLE_FLAGS:
3370         case (RTE_2TUPLE_FLAGS | RTE_NTUPLE_FLAGS_TCP_FLAG):
3371                 if (hw->mac.type != e1000_82580 && hw->mac.type != e1000_i350)
3372                         return -ENOTSUP;
3373                 memset(&filter_2tuple,
3374                         0,
3375                         sizeof(struct e1000_2tuple_filter_info));
3376                 ret = ntuple_filter_to_2tuple(ntuple_filter, &filter_2tuple);
3377                 if (ret < 0)
3378                         return ret;
3379                 p_2tuple_filter = igb_2tuple_filter_lookup(
3380                                         &filter_info->twotuple_list,
3381                                         &filter_2tuple);
3382                 if (p_2tuple_filter == NULL) {
3383                         PMD_DRV_LOG(ERR, "filter doesn't exist.");
3384                         return -ENOENT;
3385                 }
3386                 ntuple_filter->queue = p_2tuple_filter->queue;
3387                 break;
3388         default:
3389                 ret = -EINVAL;
3390                 break;
3391         }
3392
3393         return 0;
3394 }
3395
3396 /*
3397  * igb_ntuple_filter_handle - Handle operations for ntuple filter.
3398  * @dev: pointer to rte_eth_dev structure
3399  * @filter_op:operation will be taken.
3400  * @arg: a pointer to specific structure corresponding to the filter_op
3401  */
3402 static int
3403 igb_ntuple_filter_handle(struct rte_eth_dev *dev,
3404                                 enum rte_filter_op filter_op,
3405                                 void *arg)
3406 {
3407         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3408         int ret;
3409
3410         MAC_TYPE_FILTER_SUP(hw->mac.type);
3411
3412         if (filter_op == RTE_ETH_FILTER_NOP)
3413                 return 0;
3414
3415         if (arg == NULL) {
3416                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
3417                             filter_op);
3418                 return -EINVAL;
3419         }
3420
3421         switch (filter_op) {
3422         case RTE_ETH_FILTER_ADD:
3423                 ret = igb_add_del_ntuple_filter(dev,
3424                         (struct rte_eth_ntuple_filter *)arg,
3425                         TRUE);
3426                 break;
3427         case RTE_ETH_FILTER_DELETE:
3428                 ret = igb_add_del_ntuple_filter(dev,
3429                         (struct rte_eth_ntuple_filter *)arg,
3430                         FALSE);
3431                 break;
3432         case RTE_ETH_FILTER_GET:
3433                 ret = igb_get_ntuple_filter(dev,
3434                         (struct rte_eth_ntuple_filter *)arg);
3435                 break;
3436         default:
3437                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
3438                 ret = -EINVAL;
3439                 break;
3440         }
3441         return ret;
3442 }
3443
3444 static inline int
3445 igb_ethertype_filter_lookup(struct e1000_filter_info *filter_info,
3446                         uint16_t ethertype)
3447 {
3448         int i;
3449
3450         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
3451                 if (filter_info->ethertype_filters[i] == ethertype &&
3452                     (filter_info->ethertype_mask & (1 << i)))
3453                         return i;
3454         }
3455         return -1;
3456 }
3457
3458 static inline int
3459 igb_ethertype_filter_insert(struct e1000_filter_info *filter_info,
3460                         uint16_t ethertype)
3461 {
3462         int i;
3463
3464         for (i = 0; i < E1000_MAX_ETQF_FILTERS; i++) {
3465                 if (!(filter_info->ethertype_mask & (1 << i))) {
3466                         filter_info->ethertype_mask |= 1 << i;
3467                         filter_info->ethertype_filters[i] = ethertype;
3468                         return i;
3469                 }
3470         }
3471         return -1;
3472 }
3473
3474 static inline int
3475 igb_ethertype_filter_remove(struct e1000_filter_info *filter_info,
3476                         uint8_t idx)
3477 {
3478         if (idx >= E1000_MAX_ETQF_FILTERS)
3479                 return -1;
3480         filter_info->ethertype_mask &= ~(1 << idx);
3481         filter_info->ethertype_filters[idx] = 0;
3482         return idx;
3483 }
3484
3485
3486 static int
3487 igb_add_del_ethertype_filter(struct rte_eth_dev *dev,
3488                         struct rte_eth_ethertype_filter *filter,
3489                         bool add)
3490 {
3491         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3492         struct e1000_filter_info *filter_info =
3493                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3494         uint32_t etqf = 0;
3495         int ret;
3496
3497         if (filter->ether_type == ETHER_TYPE_IPv4 ||
3498                 filter->ether_type == ETHER_TYPE_IPv6) {
3499                 PMD_DRV_LOG(ERR, "unsupported ether_type(0x%04x) in"
3500                         " ethertype filter.", filter->ether_type);
3501                 return -EINVAL;
3502         }
3503
3504         if (filter->flags & RTE_ETHTYPE_FLAGS_MAC) {
3505                 PMD_DRV_LOG(ERR, "mac compare is unsupported.");
3506                 return -EINVAL;
3507         }
3508         if (filter->flags & RTE_ETHTYPE_FLAGS_DROP) {
3509                 PMD_DRV_LOG(ERR, "drop option is unsupported.");
3510                 return -EINVAL;
3511         }
3512
3513         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
3514         if (ret >= 0 && add) {
3515                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter exists.",
3516                             filter->ether_type);
3517                 return -EEXIST;
3518         }
3519         if (ret < 0 && !add) {
3520                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
3521                             filter->ether_type);
3522                 return -ENOENT;
3523         }
3524
3525         if (add) {
3526                 ret = igb_ethertype_filter_insert(filter_info,
3527                         filter->ether_type);
3528                 if (ret < 0) {
3529                         PMD_DRV_LOG(ERR, "ethertype filters are full.");
3530                         return -ENOSYS;
3531                 }
3532
3533                 etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE;
3534                 etqf |= (uint32_t)(filter->ether_type & E1000_ETQF_ETHERTYPE);
3535                 etqf |= filter->queue << E1000_ETQF_QUEUE_SHIFT;
3536         } else {
3537                 ret = igb_ethertype_filter_remove(filter_info, (uint8_t)ret);
3538                 if (ret < 0)
3539                         return -ENOSYS;
3540         }
3541         E1000_WRITE_REG(hw, E1000_ETQF(ret), etqf);
3542         E1000_WRITE_FLUSH(hw);
3543
3544         return 0;
3545 }
3546
3547 static int
3548 igb_get_ethertype_filter(struct rte_eth_dev *dev,
3549                         struct rte_eth_ethertype_filter *filter)
3550 {
3551         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3552         struct e1000_filter_info *filter_info =
3553                 E1000_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
3554         uint32_t etqf;
3555         int ret;
3556
3557         ret = igb_ethertype_filter_lookup(filter_info, filter->ether_type);
3558         if (ret < 0) {
3559                 PMD_DRV_LOG(ERR, "ethertype (0x%04x) filter doesn't exist.",
3560                             filter->ether_type);
3561                 return -ENOENT;
3562         }
3563
3564         etqf = E1000_READ_REG(hw, E1000_ETQF(ret));
3565         if (etqf & E1000_ETQF_FILTER_ENABLE) {
3566                 filter->ether_type = etqf & E1000_ETQF_ETHERTYPE;
3567                 filter->flags = 0;
3568                 filter->queue = (etqf & E1000_ETQF_QUEUE) >>
3569                                 E1000_ETQF_QUEUE_SHIFT;
3570                 return 0;
3571         }
3572
3573         return -ENOENT;
3574 }
3575
3576 /*
3577  * igb_ethertype_filter_handle - Handle operations for ethertype filter.
3578  * @dev: pointer to rte_eth_dev structure
3579  * @filter_op:operation will be taken.
3580  * @arg: a pointer to specific structure corresponding to the filter_op
3581  */
3582 static int
3583 igb_ethertype_filter_handle(struct rte_eth_dev *dev,
3584                                 enum rte_filter_op filter_op,
3585                                 void *arg)
3586 {
3587         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3588         int ret;
3589
3590         MAC_TYPE_FILTER_SUP(hw->mac.type);
3591
3592         if (filter_op == RTE_ETH_FILTER_NOP)
3593                 return 0;
3594
3595         if (arg == NULL) {
3596                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
3597                             filter_op);
3598                 return -EINVAL;
3599         }
3600
3601         switch (filter_op) {
3602         case RTE_ETH_FILTER_ADD:
3603                 ret = igb_add_del_ethertype_filter(dev,
3604                         (struct rte_eth_ethertype_filter *)arg,
3605                         TRUE);
3606                 break;
3607         case RTE_ETH_FILTER_DELETE:
3608                 ret = igb_add_del_ethertype_filter(dev,
3609                         (struct rte_eth_ethertype_filter *)arg,
3610                         FALSE);
3611                 break;
3612         case RTE_ETH_FILTER_GET:
3613                 ret = igb_get_ethertype_filter(dev,
3614                         (struct rte_eth_ethertype_filter *)arg);
3615                 break;
3616         default:
3617                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
3618                 ret = -EINVAL;
3619                 break;
3620         }
3621         return ret;
3622 }
3623
3624 static int
3625 eth_igb_filter_ctrl(struct rte_eth_dev *dev,
3626                      enum rte_filter_type filter_type,
3627                      enum rte_filter_op filter_op,
3628                      void *arg)
3629 {
3630         int ret = -EINVAL;
3631
3632         switch (filter_type) {
3633         case RTE_ETH_FILTER_NTUPLE:
3634                 ret = igb_ntuple_filter_handle(dev, filter_op, arg);
3635                 break;
3636         case RTE_ETH_FILTER_ETHERTYPE:
3637                 ret = igb_ethertype_filter_handle(dev, filter_op, arg);
3638                 break;
3639         case RTE_ETH_FILTER_SYN:
3640                 ret = eth_igb_syn_filter_handle(dev, filter_op, arg);
3641                 break;
3642         case RTE_ETH_FILTER_FLEXIBLE:
3643                 ret = eth_igb_flex_filter_handle(dev, filter_op, arg);
3644                 break;
3645         default:
3646                 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported",
3647                                                         filter_type);
3648                 break;
3649         }
3650
3651         return ret;
3652 }
3653
3654 static int
3655 eth_igb_set_mc_addr_list(struct rte_eth_dev *dev,
3656                          struct ether_addr *mc_addr_set,
3657                          uint32_t nb_mc_addr)
3658 {
3659         struct e1000_hw *hw;
3660
3661         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3662         e1000_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr);
3663         return 0;
3664 }
3665
3666 static struct rte_driver pmd_igb_drv = {
3667         .type = PMD_PDEV,
3668         .init = rte_igb_pmd_init,
3669 };
3670
3671 static struct rte_driver pmd_igbvf_drv = {
3672         .type = PMD_PDEV,
3673         .init = rte_igbvf_pmd_init,
3674 };
3675
3676 PMD_REGISTER_DRIVER(pmd_igb_drv);
3677 PMD_REGISTER_DRIVER(pmd_igbvf_drv);