lib/librte_pmd_e1000/igb_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <stdarg.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_interrupts.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_log.h>
  44 #include <rte_debug.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_memory.h>
  49 #include <rte_memzone.h>
  50 #include <rte_tailq.h>
  51 #include <rte_eal.h>
  52 #include <rte_atomic.h>
  53 #include <rte_malloc.h>
  54 #include <rte_dev.h>
  55
  56 #include "e1000_logs.h"
  57 #include "e1000/e1000_api.h"
  58 #include "e1000_ethdev.h"
  59
  60 /*
  61  * Default values for port configuration
  62  */
  63 #define IGB_DEFAULT_RX_FREE_THRESH  32
  64 #define IGB_DEFAULT_RX_PTHRESH      8
  65 #define IGB_DEFAULT_RX_HTHRESH      8
  66 #define IGB_DEFAULT_RX_WTHRESH      0
  67
  68 #define IGB_DEFAULT_TX_PTHRESH      32
  69 #define IGB_DEFAULT_TX_HTHRESH      0
  70 #define IGB_DEFAULT_TX_WTHRESH      0
  71
  72 /* Bit shift and mask */
  73 #define IGB_4_BIT_WIDTH  (CHAR_BIT / 2)
  74 #define IGB_4_BIT_MASK   RTE_LEN2MASK(IGB_4_BIT_WIDTH, uint8_t)
  75 #define IGB_8_BIT_WIDTH  CHAR_BIT
  76 #define IGB_8_BIT_MASK   UINT8_MAX
  77
  78 static int  eth_igb_configure(struct rte_eth_dev *dev);
  79 static int  eth_igb_start(struct rte_eth_dev *dev);
  80 static void eth_igb_stop(struct rte_eth_dev *dev);
  81 static void eth_igb_close(struct rte_eth_dev *dev);
  82 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev);
  83 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev);
  84 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev);
  85 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev);
  86 static int  eth_igb_link_update(struct rte_eth_dev *dev,
  87                                 int wait_to_complete);
  88 static void eth_igb_stats_get(struct rte_eth_dev *dev,
  89                                 struct rte_eth_stats *rte_stats);
  90 static void eth_igb_stats_reset(struct rte_eth_dev *dev);
  91 static void eth_igb_infos_get(struct rte_eth_dev *dev,
  92                               struct rte_eth_dev_info *dev_info);
  93 static void eth_igbvf_infos_get(struct rte_eth_dev *dev,
  94                                 struct rte_eth_dev_info *dev_info);
  95 static int  eth_igb_flow_ctrl_get(struct rte_eth_dev *dev,
  96                                 struct rte_eth_fc_conf *fc_conf);
  97 static int  eth_igb_flow_ctrl_set(struct rte_eth_dev *dev,
  98                                 struct rte_eth_fc_conf *fc_conf);
  99 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev);
 100 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev);
 101 static int eth_igb_interrupt_action(struct rte_eth_dev *dev);
 102 static void eth_igb_interrupt_handler(struct rte_intr_handle *handle,
 103                                                         void *param);
 104 static int  igb_hardware_init(struct e1000_hw *hw);
 105 static void igb_hw_control_acquire(struct e1000_hw *hw);
 106 static void igb_hw_control_release(struct e1000_hw *hw);
 107 static void igb_init_manageability(struct e1000_hw *hw);
 108 static void igb_release_manageability(struct e1000_hw *hw);
 109
 110 static int  eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 111
 112 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev,
 113                 uint16_t vlan_id, int on);
 114 static void eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
 115 static void eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 116
 117 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev);
 118 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev);
 119 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev);
 120 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev);
 121 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 122 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 123
 124 static int eth_igb_led_on(struct rte_eth_dev *dev);
 125 static int eth_igb_led_off(struct rte_eth_dev *dev);
 126
 127 static void igb_intr_disable(struct e1000_hw *hw);
 128 static int  igb_get_rx_buffer_size(struct e1000_hw *hw);
 129 static void eth_igb_rar_set(struct rte_eth_dev *dev,
 130                 struct ether_addr *mac_addr,
 131                 uint32_t index, uint32_t pool);
 132 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 133
 134 static void igbvf_intr_disable(struct e1000_hw *hw);
 135 static int igbvf_dev_configure(struct rte_eth_dev *dev);
 136 static int igbvf_dev_start(struct rte_eth_dev *dev);
 137 static void igbvf_dev_stop(struct rte_eth_dev *dev);
 138 static void igbvf_dev_close(struct rte_eth_dev *dev);
 139 static int eth_igbvf_link_update(struct e1000_hw *hw);
 140 static void eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats);
 141 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev);
 142 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev,
 143                 uint16_t vlan_id, int on);
 144 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on);
 145 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 146 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev,
 147                                    struct rte_eth_rss_reta_entry64 *reta_conf,
 148                                    uint16_t reta_size);
 149 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev,
 150                                   struct rte_eth_rss_reta_entry64 *reta_conf,
 151                                   uint16_t reta_size);
 152 static int eth_igb_add_syn_filter(struct rte_eth_dev *dev,
 153                         struct rte_syn_filter *filter, uint16_t rx_queue);
 154 static int eth_igb_remove_syn_filter(struct rte_eth_dev *dev);
 155 static int eth_igb_get_syn_filter(struct rte_eth_dev *dev,
 156                         struct rte_syn_filter *filter, uint16_t *rx_queue);
 157 static int eth_igb_add_ethertype_filter(struct rte_eth_dev *dev,
 158                         uint16_t index,
 159                         struct rte_ethertype_filter *filter, uint16_t rx_queue);
 160 static int eth_igb_remove_ethertype_filter(struct rte_eth_dev *dev,
 161                         uint16_t index);
 162 static int eth_igb_get_ethertype_filter(struct rte_eth_dev *dev,
 163                         uint16_t index,
 164                         struct rte_ethertype_filter *filter, uint16_t *rx_queue);
 165 static int eth_igb_add_2tuple_filter(struct rte_eth_dev *dev,
 166                         uint16_t index,
 167                         struct rte_2tuple_filter *filter, uint16_t rx_queue);
 168 static int eth_igb_remove_2tuple_filter(struct rte_eth_dev *dev,
 169                         uint16_t index);
 170 static int eth_igb_get_2tuple_filter(struct rte_eth_dev *dev,
 171                         uint16_t index,
 172                         struct rte_2tuple_filter *filter, uint16_t *rx_queue);
 173 static int eth_igb_add_flex_filter(struct rte_eth_dev *dev,
 174                         uint16_t index,
 175                         struct rte_flex_filter *filter, uint16_t rx_queue);
 176 static int eth_igb_remove_flex_filter(struct rte_eth_dev *dev,
 177                         uint16_t index);
 178 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev,
 179                         uint16_t index,
 180                         struct rte_flex_filter *filter, uint16_t *rx_queue);
 181 static int eth_igb_add_5tuple_filter(struct rte_eth_dev *dev,
 182                         uint16_t index,
 183                         struct rte_5tuple_filter *filter, uint16_t rx_queue);
 184 static int eth_igb_remove_5tuple_filter(struct rte_eth_dev *dev,
 185                         uint16_t index);
 186 static int eth_igb_get_5tuple_filter(struct rte_eth_dev *dev,
 187                         uint16_t index,
 188                         struct rte_5tuple_filter *filter, uint16_t *rx_queue);
 189
 190 /*
 191  * Define VF Stats MACRO for Non "cleared on read" register
 192  */
 193 #define UPDATE_VF_STAT(reg, last, cur)            \
 194 {                                                 \
 195         u32 latest = E1000_READ_REG(hw, reg);     \
 196         cur += latest - last;                     \
 197         last = latest;                            \
 198 }
 199
 200
 201 #define IGB_FC_PAUSE_TIME 0x0680
 202 #define IGB_LINK_UPDATE_CHECK_TIMEOUT  90  /* 9s */
 203 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */
 204
 205 #define IGBVF_PMD_NAME "rte_igbvf_pmd"     /* PMD name */
 206
 207 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full;
 208
 209 /*
 210  * The set of PCI devices this driver supports
 211  */
 212 static struct rte_pci_id pci_id_igb_map[] = {
 213
 214 #define RTE_PCI_DEV_ID_DECL_IGB(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 215 #include "rte_pci_dev_ids.h"
 216
 217 {.device_id = 0},
 218 };
 219
 220 /*
 221  * The set of PCI devices this driver supports (for 82576&I350 VF)
 222  */
 223 static struct rte_pci_id pci_id_igbvf_map[] = {
 224
 225 #define RTE_PCI_DEV_ID_DECL_IGBVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 226 #include "rte_pci_dev_ids.h"
 227
 228 {.device_id = 0},
 229 };
 230
 231 static struct eth_dev_ops eth_igb_ops = {
 232         .dev_configure        = eth_igb_configure,
 233         .dev_start            = eth_igb_start,
 234         .dev_stop             = eth_igb_stop,
 235         .dev_close            = eth_igb_close,
 236         .promiscuous_enable   = eth_igb_promiscuous_enable,
 237         .promiscuous_disable  = eth_igb_promiscuous_disable,
 238         .allmulticast_enable  = eth_igb_allmulticast_enable,
 239         .allmulticast_disable = eth_igb_allmulticast_disable,
 240         .link_update          = eth_igb_link_update,
 241         .stats_get            = eth_igb_stats_get,
 242         .stats_reset          = eth_igb_stats_reset,
 243         .dev_infos_get        = eth_igb_infos_get,
 244         .mtu_set              = eth_igb_mtu_set,
 245         .vlan_filter_set      = eth_igb_vlan_filter_set,
 246         .vlan_tpid_set        = eth_igb_vlan_tpid_set,
 247         .vlan_offload_set     = eth_igb_vlan_offload_set,
 248         .rx_queue_setup       = eth_igb_rx_queue_setup,
 249         .rx_queue_release     = eth_igb_rx_queue_release,
 250         .rx_queue_count       = eth_igb_rx_queue_count,
 251         .rx_descriptor_done   = eth_igb_rx_descriptor_done,
 252         .tx_queue_setup       = eth_igb_tx_queue_setup,
 253         .tx_queue_release     = eth_igb_tx_queue_release,
 254         .dev_led_on           = eth_igb_led_on,
 255         .dev_led_off          = eth_igb_led_off,
 256         .flow_ctrl_get        = eth_igb_flow_ctrl_get,
 257         .flow_ctrl_set        = eth_igb_flow_ctrl_set,
 258         .mac_addr_add         = eth_igb_rar_set,
 259         .mac_addr_remove      = eth_igb_rar_clear,
 260         .reta_update          = eth_igb_rss_reta_update,
 261         .reta_query           = eth_igb_rss_reta_query,
 262         .rss_hash_update      = eth_igb_rss_hash_update,
 263         .rss_hash_conf_get    = eth_igb_rss_hash_conf_get,
 264         .add_syn_filter          = eth_igb_add_syn_filter,
 265         .remove_syn_filter       = eth_igb_remove_syn_filter,
 266         .get_syn_filter          = eth_igb_get_syn_filter,
 267         .add_ethertype_filter    = eth_igb_add_ethertype_filter,
 268         .remove_ethertype_filter = eth_igb_remove_ethertype_filter,
 269         .get_ethertype_filter    = eth_igb_get_ethertype_filter,
 270         .add_2tuple_filter       = eth_igb_add_2tuple_filter,
 271         .remove_2tuple_filter    = eth_igb_remove_2tuple_filter,
 272         .get_2tuple_filter       = eth_igb_get_2tuple_filter,
 273         .add_flex_filter         = eth_igb_add_flex_filter,
 274         .remove_flex_filter      = eth_igb_remove_flex_filter,
 275         .get_flex_filter         = eth_igb_get_flex_filter,
 276         .add_5tuple_filter       = eth_igb_add_5tuple_filter,
 277         .remove_5tuple_filter    = eth_igb_remove_5tuple_filter,
 278         .get_5tuple_filter       = eth_igb_get_5tuple_filter,
 279 };
 280
 281 /*
 282  * dev_ops for virtual function, bare necessities for basic vf
 283  * operation have been implemented
 284  */
 285 static struct eth_dev_ops igbvf_eth_dev_ops = {
 286         .dev_configure        = igbvf_dev_configure,
 287         .dev_start            = igbvf_dev_start,
 288         .dev_stop             = igbvf_dev_stop,
 289         .dev_close            = igbvf_dev_close,
 290         .link_update          = eth_igb_link_update,
 291         .stats_get            = eth_igbvf_stats_get,
 292         .stats_reset          = eth_igbvf_stats_reset,
 293         .vlan_filter_set      = igbvf_vlan_filter_set,
 294         .dev_infos_get        = eth_igbvf_infos_get,
 295         .rx_queue_setup       = eth_igb_rx_queue_setup,
 296         .rx_queue_release     = eth_igb_rx_queue_release,
 297         .tx_queue_setup       = eth_igb_tx_queue_setup,
 298         .tx_queue_release     = eth_igb_tx_queue_release,
 299 };
 300
 301 /**
 302  * Atomically reads the link status information from global
 303  * structure rte_eth_dev.
 304  *
 305  * @param dev
 306  *   - Pointer to the structure rte_eth_dev to read from.
 307  *   - Pointer to the buffer to be saved with the link status.
 308  *
 309  * @return
 310  *   - On success, zero.
 311  *   - On failure, negative value.
 312  */
 313 static inline int
 314 rte_igb_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 315                                 struct rte_eth_link *link)
 316 {
 317         struct rte_eth_link *dst = link;
 318         struct rte_eth_link *src = &(dev->data->dev_link);
 319
 320         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 321                                         *(uint64_t *)src) == 0)
 322                 return -1;
 323
 324         return 0;
 325 }
 326
 327 /**
 328  * Atomically writes the link status information into global
 329  * structure rte_eth_dev.
 330  *
 331  * @param dev
 332  *   - Pointer to the structure rte_eth_dev to read from.
 333  *   - Pointer to the buffer to be saved with the link status.
 334  *
 335  * @return
 336  *   - On success, zero.
 337  *   - On failure, negative value.
 338  */
 339 static inline int
 340 rte_igb_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 341                                 struct rte_eth_link *link)
 342 {
 343         struct rte_eth_link *dst = &(dev->data->dev_link);
 344         struct rte_eth_link *src = link;
 345
 346         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 347                                         *(uint64_t *)src) == 0)
 348                 return -1;
 349
 350         return 0;
 351 }
 352
 353 static inline void
 354 igb_intr_enable(struct rte_eth_dev *dev)
 355 {
 356         struct e1000_interrupt *intr =
 357                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 358         struct e1000_hw *hw =
 359                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 360
 361         E1000_WRITE_REG(hw, E1000_IMS, intr->mask);
 362         E1000_WRITE_FLUSH(hw);
 363 }
 364
 365 static void
 366 igb_intr_disable(struct e1000_hw *hw)
 367 {
 368         E1000_WRITE_REG(hw, E1000_IMC, ~0);
 369         E1000_WRITE_FLUSH(hw);
 370 }
 371
 372 static inline int32_t
 373 igb_pf_reset_hw(struct e1000_hw *hw)
 374 {
 375         uint32_t ctrl_ext;
 376         int32_t status;
 377
 378         status = e1000_reset_hw(hw);
 379
 380         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 381         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 382         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 383         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 384         E1000_WRITE_FLUSH(hw);
 385
 386         return status;
 387 }
 388
 389 static void
 390 igb_identify_hardware(struct rte_eth_dev *dev)
 391 {
 392         struct e1000_hw *hw =
 393                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 394
 395         hw->vendor_id = dev->pci_dev->id.vendor_id;
 396         hw->device_id = dev->pci_dev->id.device_id;
 397         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
 398         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
 399
 400         e1000_set_mac_type(hw);
 401
 402         /* need to check if it is a vf device below */
 403 }
 404
 405 static int
 406 igb_reset_swfw_lock(struct e1000_hw *hw)
 407 {
 408         int ret_val;
 409
 410         /*
 411          * Do mac ops initialization manually here, since we will need
 412          * some function pointers set by this call.
 413          */
 414         ret_val = e1000_init_mac_params(hw);
 415         if (ret_val)
 416                 return ret_val;
 417
 418         /*
 419          * SMBI lock should not fail in this early stage. If this is the case,
 420          * it is due to an improper exit of the application.
 421          * So force the release of the faulty lock.
 422          */
 423         if (e1000_get_hw_semaphore_generic(hw) < 0) {
 424                 PMD_DRV_LOG(DEBUG, "SMBI lock released");
 425         }
 426         e1000_put_hw_semaphore_generic(hw);
 427
 428         if (hw->mac.ops.acquire_swfw_sync != NULL) {
 429                 uint16_t mask;
 430
 431                 /*
 432                  * Phy lock should not fail in this early stage. If this is the case,
 433                  * it is due to an improper exit of the application.
 434                  * So force the release of the faulty lock.
 435                  */
 436                 mask = E1000_SWFW_PHY0_SM << hw->bus.func;
 437                 if (hw->bus.func > E1000_FUNC_1)
 438                         mask <<= 2;
 439                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 440                         PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released",
 441                                     hw->bus.func);
 442                 }
 443                 hw->mac.ops.release_swfw_sync(hw, mask);
 444
 445                 /*
 446                  * This one is more tricky since it is common to all ports; but
 447                  * swfw_sync retries last long enough (1s) to be almost sure that if
 448                  * lock can not be taken it is due to an improper lock of the
 449                  * semaphore.
 450                  */
 451                 mask = E1000_SWFW_EEP_SM;
 452                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 453                         PMD_DRV_LOG(DEBUG, "SWFW common locks released");
 454                 }
 455                 hw->mac.ops.release_swfw_sync(hw, mask);
 456         }
 457
 458         return E1000_SUCCESS;
 459 }
 460
 461 static int
 462 eth_igb_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 463                    struct rte_eth_dev *eth_dev)
 464 {
 465         int error = 0;
 466         struct rte_pci_device *pci_dev;
 467         struct e1000_hw *hw =
 468                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 469         struct e1000_vfta * shadow_vfta =
 470                         E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 471         uint32_t ctrl_ext;
 472
 473         pci_dev = eth_dev->pci_dev;
 474         eth_dev->dev_ops = &eth_igb_ops;
 475         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 476         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 477
 478         /* for secondary processes, we don't initialise any further as primary
 479          * has already done this work. Only check we don't need a different
 480          * RX function */
 481         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 482                 if (eth_dev->data->scattered_rx)
 483                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 484                 return 0;
 485         }
 486
 487         hw->hw_addr= (void *)pci_dev->mem_resource[0].addr;
 488
 489         igb_identify_hardware(eth_dev);
 490         if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) {
 491                 error = -EIO;
 492                 goto err_late;
 493         }
 494
 495         e1000_get_bus_info(hw);
 496
 497         /* Reset any pending lock */
 498         if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) {
 499                 error = -EIO;
 500                 goto err_late;
 501         }
 502
 503         /* Finish initialization */
 504         if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) {
 505                 error = -EIO;
 506                 goto err_late;
 507         }
 508
 509         hw->mac.autoneg = 1;
 510         hw->phy.autoneg_wait_to_complete = 0;
 511         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 512
 513         /* Copper options */
 514         if (hw->phy.media_type == e1000_media_type_copper) {
 515                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
 516                 hw->phy.disable_polarity_correction = 0;
 517                 hw->phy.ms_type = e1000_ms_hw_default;
 518         }
 519
 520         /*
 521          * Start from a known state, this is important in reading the nvm
 522          * and mac from that.
 523          */
 524         igb_pf_reset_hw(hw);
 525
 526         /* Make sure we have a good EEPROM before we read from it */
 527         if (e1000_validate_nvm_checksum(hw) < 0) {
 528                 /*
 529                  * Some PCI-E parts fail the first check due to
 530                  * the link being in sleep state, call it again,
 531                  * if it fails a second time its a real issue.
 532                  */
 533                 if (e1000_validate_nvm_checksum(hw) < 0) {
 534                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
 535                         error = -EIO;
 536                         goto err_late;
 537                 }
 538         }
 539
 540         /* Read the permanent MAC address out of the EEPROM */
 541         if (e1000_read_mac_addr(hw) != 0) {
 542                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
 543                 error = -EIO;
 544                 goto err_late;
 545         }
 546
 547         /* Allocate memory for storing MAC addresses */
 548         eth_dev->data->mac_addrs = rte_zmalloc("e1000",
 549                 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
 550         if (eth_dev->data->mac_addrs == NULL) {
 551                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to "
 552                                                 "store MAC addresses",
 553                                 ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 554                 error = -ENOMEM;
 555                 goto err_late;
 556         }
 557
 558         /* Copy the permanent MAC address */
 559         ether_addr_copy((struct ether_addr *)hw->mac.addr, &eth_dev->data->mac_addrs[0]);
 560
 561         /* initialize the vfta */
 562         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 563
 564         /* Now initialize the hardware */
 565         if (igb_hardware_init(hw) != 0) {
 566                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
 567                 rte_free(eth_dev->data->mac_addrs);
 568                 eth_dev->data->mac_addrs = NULL;
 569                 error = -ENODEV;
 570                 goto err_late;
 571         }
 572         hw->mac.get_link_status = 1;
 573
 574         /* Indicate SOL/IDER usage */
 575         if (e1000_check_reset_block(hw) < 0) {
 576                 PMD_INIT_LOG(ERR, "PHY reset is blocked due to"
 577                                         "SOL/IDER session");
 578         }
 579
 580         /* initialize PF if max_vfs not zero */
 581         igb_pf_host_init(eth_dev);
 582
 583         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 584         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 585         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 586         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 587         E1000_WRITE_FLUSH(hw);
 588
 589         PMD_INIT_LOG(INFO, "port_id %d vendorID=0x%x deviceID=0x%x",
 590                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 591                      pci_dev->id.device_id);
 592
 593         rte_intr_callback_register(&(pci_dev->intr_handle),
 594                 eth_igb_interrupt_handler, (void *)eth_dev);
 595
 596         /* enable uio intr after callback register */
 597         rte_intr_enable(&(pci_dev->intr_handle));
 598
 599         /* enable support intr */
 600         igb_intr_enable(eth_dev);
 601
 602         return 0;
 603
 604 err_late:
 605         igb_hw_control_release(hw);
 606
 607         return (error);
 608 }
 609
 610 /*
 611  * Virtual Function device init
 612  */
 613 static int
 614 eth_igbvf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 615                 struct rte_eth_dev *eth_dev)
 616 {
 617         struct rte_pci_device *pci_dev;
 618         struct e1000_hw *hw =
 619                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 620         int diag;
 621
 622         PMD_INIT_FUNC_TRACE();
 623
 624         eth_dev->dev_ops = &igbvf_eth_dev_ops;
 625         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 626         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 627
 628         /* for secondary processes, we don't initialise any further as primary
 629          * has already done this work. Only check we don't need a different
 630          * RX function */
 631         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 632                 if (eth_dev->data->scattered_rx)
 633                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 634                 return 0;
 635         }
 636
 637         pci_dev = eth_dev->pci_dev;
 638
 639         hw->device_id = pci_dev->id.device_id;
 640         hw->vendor_id = pci_dev->id.vendor_id;
 641         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 642
 643         /* Initialize the shared code (base driver) */
 644         diag = e1000_setup_init_funcs(hw, TRUE);
 645         if (diag != 0) {
 646                 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d",
 647                         diag);
 648                 return -EIO;
 649         }
 650
 651         /* init_mailbox_params */
 652         hw->mbx.ops.init_params(hw);
 653
 654         /* Disable the interrupts for VF */
 655         igbvf_intr_disable(hw);
 656
 657         diag = hw->mac.ops.reset_hw(hw);
 658
 659         /* Allocate memory for storing MAC addresses */
 660         eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN *
 661                 hw->mac.rar_entry_count, 0);
 662         if (eth_dev->data->mac_addrs == NULL) {
 663                 PMD_INIT_LOG(ERR,
 664                         "Failed to allocate %d bytes needed to store MAC "
 665                         "addresses",
 666                         ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 667                 return -ENOMEM;
 668         }
 669
 670         /* Copy the permanent MAC address */
 671         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 672                         &eth_dev->data->mac_addrs[0]);
 673
 674         PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x "
 675                      "mac.type=%s",
 676                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 677                      pci_dev->id.device_id, "igb_mac_82576_vf");
 678
 679         return 0;
 680 }
 681
 682 static struct eth_driver rte_igb_pmd = {
 683         {
 684                 .name = "rte_igb_pmd",
 685                 .id_table = pci_id_igb_map,
 686                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
 687         },
 688         .eth_dev_init = eth_igb_dev_init,
 689         .dev_private_size = sizeof(struct e1000_adapter),
 690 };
 691
 692 /*
 693  * virtual function driver struct
 694  */
 695 static struct eth_driver rte_igbvf_pmd = {
 696         {
 697                 .name = "rte_igbvf_pmd",
 698                 .id_table = pci_id_igbvf_map,
 699                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 700         },
 701         .eth_dev_init = eth_igbvf_dev_init,
 702         .dev_private_size = sizeof(struct e1000_adapter),
 703 };
 704
 705 static int
 706 rte_igb_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 707 {
 708         rte_eth_driver_register(&rte_igb_pmd);
 709         return 0;
 710 }
 711
 712 static void
 713 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 714 {
 715         struct e1000_hw *hw =
 716                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 717         /* RCTL: enable VLAN filter since VMDq always use VLAN filter */
 718         uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL);
 719         rctl |= E1000_RCTL_VFE;
 720         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
 721 }
 722
 723 /*
 724  * VF Driver initialization routine.
 725  * Invoked one at EAL init time.
 726  * Register itself as the [Virtual Poll Mode] Driver of PCI IGB devices.
 727  */
 728 static int
 729 rte_igbvf_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 730 {
 731         PMD_INIT_FUNC_TRACE();
 732
 733         rte_eth_driver_register(&rte_igbvf_pmd);
 734         return (0);
 735 }
 736
 737 static int
 738 eth_igb_configure(struct rte_eth_dev *dev)
 739 {
 740         struct e1000_interrupt *intr =
 741                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 742
 743         PMD_INIT_FUNC_TRACE();
 744         intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
 745         PMD_INIT_FUNC_TRACE();
 746
 747         return (0);
 748 }
 749
 750 static int
 751 eth_igb_start(struct rte_eth_dev *dev)
 752 {
 753         struct e1000_hw *hw =
 754                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 755         int ret, i, mask;
 756         uint32_t ctrl_ext;
 757
 758         PMD_INIT_FUNC_TRACE();
 759
 760         /* Power up the phy. Needed to make the link go Up */
 761         e1000_power_up_phy(hw);
 762
 763         /*
 764          * Packet Buffer Allocation (PBA)
 765          * Writing PBA sets the receive portion of the buffer
 766          * the remainder is used for the transmit buffer.
 767          */
 768         if (hw->mac.type == e1000_82575) {
 769                 uint32_t pba;
 770
 771                 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
 772                 E1000_WRITE_REG(hw, E1000_PBA, pba);
 773         }
 774
 775         /* Put the address into the Receive Address Array */
 776         e1000_rar_set(hw, hw->mac.addr, 0);
 777
 778         /* Initialize the hardware */
 779         if (igb_hardware_init(hw)) {
 780                 PMD_INIT_LOG(ERR, "Unable to initialize the hardware");
 781                 return (-EIO);
 782         }
 783
 784         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
 785
 786         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 787         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 788         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 789         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 790         E1000_WRITE_FLUSH(hw);
 791
 792         /* configure PF module if SRIOV enabled */
 793         igb_pf_host_configure(dev);
 794
 795         /* Configure for OS presence */
 796         igb_init_manageability(hw);
 797
 798         eth_igb_tx_init(dev);
 799
 800         /* This can fail when allocating mbufs for descriptor rings */
 801         ret = eth_igb_rx_init(dev);
 802         if (ret) {
 803                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
 804                 igb_dev_clear_queues(dev);
 805                 return ret;
 806         }
 807
 808         e1000_clear_hw_cntrs_base_generic(hw);
 809
 810         /*
 811          * VLAN Offload Settings
 812          */
 813         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
 814                         ETH_VLAN_EXTEND_MASK;
 815         eth_igb_vlan_offload_set(dev, mask);
 816
 817         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
 818                 /* Enable VLAN filter since VMDq always use VLAN filter */
 819                 igb_vmdq_vlan_hw_filter_enable(dev);
 820         }
 821
 822         /*
 823          * Configure the Interrupt Moderation register (EITR) with the maximum
 824          * possible value (0xFFFF) to minimize "System Partial Write" issued by
 825          * spurious [DMA] memory updates of RX and TX ring descriptors.
 826          *
 827          * With a EITR granularity of 2 microseconds in the 82576, only 7/8
 828          * spurious memory updates per second should be expected.
 829          * ((65535 * 2) / 1000.1000 ~= 0.131 second).
 830          *
 831          * Because interrupts are not used at all, the MSI-X is not activated
 832          * and interrupt moderation is controlled by EITR[0].
 833          *
 834          * Note that having [almost] disabled memory updates of RX and TX ring
 835          * descriptors through the Interrupt Moderation mechanism, memory
 836          * updates of ring descriptors are now moderated by the configurable
 837          * value of Write-Back Threshold registers.
 838          */
 839         if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) ||
 840                 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210) ||
 841                 (hw->mac.type == e1000_i211)) {
 842                 uint32_t ivar;
 843
 844                 /* Enable all RX & TX queues in the IVAR registers */
 845                 ivar = (uint32_t) ((E1000_IVAR_VALID << 16) | E1000_IVAR_VALID);
 846                 for (i = 0; i < 8; i++)
 847                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, i, ivar);
 848
 849                 /* Configure EITR with the maximum possible value (0xFFFF) */
 850                 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF);
 851         }
 852
 853         /* Setup link speed and duplex */
 854         switch (dev->data->dev_conf.link_speed) {
 855         case ETH_LINK_SPEED_AUTONEG:
 856                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
 857                         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 858                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
 859                         hw->phy.autoneg_advertised = E1000_ALL_HALF_DUPLEX;
 860                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
 861                         hw->phy.autoneg_advertised = E1000_ALL_FULL_DUPLEX;
 862                 else
 863                         goto error_invalid_config;
 864                 break;
 865         case ETH_LINK_SPEED_10:
 866                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
 867                         hw->phy.autoneg_advertised = E1000_ALL_10_SPEED;
 868                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
 869                         hw->phy.autoneg_advertised = ADVERTISE_10_HALF;
 870                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
 871                         hw->phy.autoneg_advertised = ADVERTISE_10_FULL;
 872                 else
 873                         goto error_invalid_config;
 874                 break;
 875         case ETH_LINK_SPEED_100:
 876                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
 877                         hw->phy.autoneg_advertised = E1000_ALL_100_SPEED;
 878                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
 879                         hw->phy.autoneg_advertised = ADVERTISE_100_HALF;
 880                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
 881                         hw->phy.autoneg_advertised = ADVERTISE_100_FULL;
 882                 else
 883                         goto error_invalid_config;
 884                 break;
 885         case ETH_LINK_SPEED_1000:
 886                 if ((dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX) ||
 887                                 (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX))
 888                         hw->phy.autoneg_advertised = ADVERTISE_1000_FULL;
 889                 else
 890                         goto error_invalid_config;
 891                 break;
 892         case ETH_LINK_SPEED_10000:
 893         default:
 894                 goto error_invalid_config;
 895         }
 896         e1000_setup_link(hw);
 897
 898         /* check if lsc interrupt feature is enabled */
 899         if (dev->data->dev_conf.intr_conf.lsc != 0)
 900                 ret = eth_igb_lsc_interrupt_setup(dev);
 901
 902         /* resume enabled intr since hw reset */
 903         igb_intr_enable(dev);
 904
 905         PMD_INIT_LOG(DEBUG, "<<");
 906
 907         return (0);
 908
 909 error_invalid_config:
 910         PMD_INIT_LOG(ERR, "Invalid link_speed/link_duplex (%u/%u) for port %u",
 911                      dev->data->dev_conf.link_speed,
 912                      dev->data->dev_conf.link_duplex, dev->data->port_id);
 913         igb_dev_clear_queues(dev);
 914         return (-EINVAL);
 915 }
 916
 917 /*********************************************************************
 918  *
 919  *  This routine disables all traffic on the adapter by issuing a
 920  *  global reset on the MAC.
 921  *
 922  **********************************************************************/
 923 static void
 924 eth_igb_stop(struct rte_eth_dev *dev)
 925 {
 926         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 927         struct rte_eth_link link;
 928
 929         igb_intr_disable(hw);
 930         igb_pf_reset_hw(hw);
 931         E1000_WRITE_REG(hw, E1000_WUC, 0);
 932
 933         /* Set bit for Go Link disconnect */
 934         if (hw->mac.type >= e1000_82580) {
 935                 uint32_t phpm_reg;
 936
 937                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
 938                 phpm_reg |= E1000_82580_PM_GO_LINKD;
 939                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
 940         }
 941
 942         /* Power down the phy. Needed to make the link go Down */
 943         e1000_power_down_phy(hw);
 944
 945         igb_dev_clear_queues(dev);
 946
 947         /* clear the recorded link status */
 948         memset(&link, 0, sizeof(link));
 949         rte_igb_dev_atomic_write_link_status(dev, &link);
 950 }
 951
 952 static void
 953 eth_igb_close(struct rte_eth_dev *dev)
 954 {
 955         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 956         struct rte_eth_link link;
 957
 958         eth_igb_stop(dev);
 959         e1000_phy_hw_reset(hw);
 960         igb_release_manageability(hw);
 961         igb_hw_control_release(hw);
 962
 963         /* Clear bit for Go Link disconnect */
 964         if (hw->mac.type >= e1000_82580) {
 965                 uint32_t phpm_reg;
 966
 967                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
 968                 phpm_reg &= ~E1000_82580_PM_GO_LINKD;
 969                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
 970         }
 971
 972         igb_dev_clear_queues(dev);
 973
 974         memset(&link, 0, sizeof(link));
 975         rte_igb_dev_atomic_write_link_status(dev, &link);
 976 }
 977
 978 static int
 979 igb_get_rx_buffer_size(struct e1000_hw *hw)
 980 {
 981         uint32_t rx_buf_size;
 982         if (hw->mac.type == e1000_82576) {
 983                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10;
 984         } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) {
 985                 /* PBS needs to be translated according to a lookup table */
 986                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf);
 987                 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size);
 988                 rx_buf_size = (rx_buf_size << 10);
 989         } else if (hw->mac.type == e1000_i210 || hw->mac.type == e1000_i211) {
 990                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10;
 991         } else {
 992                 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10;
 993         }
 994
 995         return rx_buf_size;
 996 }
 997
 998 /*********************************************************************
 999  *
1000  *  Initialize the hardware
1001  *
1002  **********************************************************************/
1003 static int
1004 igb_hardware_init(struct e1000_hw *hw)
1005 {
1006         uint32_t rx_buf_size;
1007         int diag;
1008
1009         /* Let the firmware know the OS is in control */
1010         igb_hw_control_acquire(hw);
1011
1012         /*
1013          * These parameters control the automatic generation (Tx) and
1014          * response (Rx) to Ethernet PAUSE frames.
1015          * - High water mark should allow for at least two standard size (1518)
1016          *   frames to be received after sending an XOFF.
1017          * - Low water mark works best when it is very near the high water mark.
1018          *   This allows the receiver to restart by sending XON when it has
1019          *   drained a bit. Here we use an arbitrary value of 1500 which will
1020          *   restart after one full frame is pulled from the buffer. There
1021          *   could be several smaller frames in the buffer and if so they will
1022          *   not trigger the XON until their total number reduces the buffer
1023          *   by 1500.
1024          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
1025          */
1026         rx_buf_size = igb_get_rx_buffer_size(hw);
1027
1028         hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2);
1029         hw->fc.low_water = hw->fc.high_water - 1500;
1030         hw->fc.pause_time = IGB_FC_PAUSE_TIME;
1031         hw->fc.send_xon = 1;
1032
1033         /* Set Flow control, use the tunable location if sane */
1034         if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4))
1035                 hw->fc.requested_mode = igb_fc_setting;
1036         else
1037                 hw->fc.requested_mode = e1000_fc_none;
1038
1039         /* Issue a global reset */
1040         igb_pf_reset_hw(hw);
1041         E1000_WRITE_REG(hw, E1000_WUC, 0);
1042
1043         diag = e1000_init_hw(hw);
1044         if (diag < 0)
1045                 return (diag);
1046
1047         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1048         e1000_get_phy_info(hw);
1049         e1000_check_for_link(hw);
1050
1051         return (0);
1052 }
1053
1054 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */
1055 static void
1056 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1057 {
1058         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1059         struct e1000_hw_stats *stats =
1060                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1061         int pause_frames;
1062
1063         if(hw->phy.media_type == e1000_media_type_copper ||
1064             (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
1065                 stats->symerrs +=
1066                     E1000_READ_REG(hw,E1000_SYMERRS);
1067                 stats->sec += E1000_READ_REG(hw, E1000_SEC);
1068         }
1069
1070         stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
1071         stats->mpc += E1000_READ_REG(hw, E1000_MPC);
1072         stats->scc += E1000_READ_REG(hw, E1000_SCC);
1073         stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
1074
1075         stats->mcc += E1000_READ_REG(hw, E1000_MCC);
1076         stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
1077         stats->colc += E1000_READ_REG(hw, E1000_COLC);
1078         stats->dc += E1000_READ_REG(hw, E1000_DC);
1079         stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
1080         stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
1081         stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
1082         /*
1083         ** For watchdog management we need to know if we have been
1084         ** paused during the last interval, so capture that here.
1085         */
1086         pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC);
1087         stats->xoffrxc += pause_frames;
1088         stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
1089         stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
1090         stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
1091         stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
1092         stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
1093         stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
1094         stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
1095         stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
1096         stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
1097         stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
1098         stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
1099         stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
1100
1101         /* For the 64-bit byte counters the low dword must be read first. */
1102         /* Both registers clear on the read of the high dword */
1103
1104         stats->gorc += E1000_READ_REG(hw, E1000_GORCL);
1105         stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32);
1106         stats->gotc += E1000_READ_REG(hw, E1000_GOTCL);
1107         stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32);
1108
1109         stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
1110         stats->ruc += E1000_READ_REG(hw, E1000_RUC);
1111         stats->rfc += E1000_READ_REG(hw, E1000_RFC);
1112         stats->roc += E1000_READ_REG(hw, E1000_ROC);
1113         stats->rjc += E1000_READ_REG(hw, E1000_RJC);
1114
1115         stats->tor += E1000_READ_REG(hw, E1000_TORH);
1116         stats->tot += E1000_READ_REG(hw, E1000_TOTH);
1117
1118         stats->tpr += E1000_READ_REG(hw, E1000_TPR);
1119         stats->tpt += E1000_READ_REG(hw, E1000_TPT);
1120         stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
1121         stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
1122         stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
1123         stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
1124         stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
1125         stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
1126         stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
1127         stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
1128
1129         /* Interrupt Counts */
1130
1131         stats->iac += E1000_READ_REG(hw, E1000_IAC);
1132         stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
1133         stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
1134         stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
1135         stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
1136         stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
1137         stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
1138         stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
1139         stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
1140
1141         /* Host to Card Statistics */
1142
1143         stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
1144         stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
1145         stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
1146         stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
1147         stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
1148         stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
1149         stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
1150         stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL);
1151         stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32);
1152         stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL);
1153         stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32);
1154         stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
1155         stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
1156         stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
1157
1158         stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
1159         stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
1160         stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
1161         stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
1162         stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
1163         stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
1164
1165         if (rte_stats == NULL)
1166                 return;
1167
1168         /* Rx Errors */
1169         rte_stats->ibadcrc = stats->crcerrs;
1170         rte_stats->ibadlen = stats->rlec + stats->ruc + stats->roc;
1171         rte_stats->imissed = stats->mpc;
1172         rte_stats->ierrors = rte_stats->ibadcrc +
1173                              rte_stats->ibadlen +
1174                              rte_stats->imissed +
1175                              stats->rxerrc + stats->algnerrc + stats->cexterr;
1176
1177         /* Tx Errors */
1178         rte_stats->oerrors = stats->ecol + stats->latecol;
1179
1180         /* XON/XOFF pause frames */
1181         rte_stats->tx_pause_xon  = stats->xontxc;
1182         rte_stats->rx_pause_xon  = stats->xonrxc;
1183         rte_stats->tx_pause_xoff = stats->xofftxc;
1184         rte_stats->rx_pause_xoff = stats->xoffrxc;
1185
1186         rte_stats->ipackets = stats->gprc;
1187         rte_stats->opackets = stats->gptc;
1188         rte_stats->ibytes   = stats->gorc;
1189         rte_stats->obytes   = stats->gotc;
1190 }
1191
1192 static void
1193 eth_igb_stats_reset(struct rte_eth_dev *dev)
1194 {
1195         struct e1000_hw_stats *hw_stats =
1196                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1197
1198         /* HW registers are cleared on read */
1199         eth_igb_stats_get(dev, NULL);
1200
1201         /* Reset software totals */
1202         memset(hw_stats, 0, sizeof(*hw_stats));
1203 }
1204
1205 static void
1206 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1207 {
1208         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1209         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1210                           E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1211
1212         /* Good Rx packets, include VF loopback */
1213         UPDATE_VF_STAT(E1000_VFGPRC,
1214             hw_stats->last_gprc, hw_stats->gprc);
1215
1216         /* Good Rx octets, include VF loopback */
1217         UPDATE_VF_STAT(E1000_VFGORC,
1218             hw_stats->last_gorc, hw_stats->gorc);
1219
1220         /* Good Tx packets, include VF loopback */
1221         UPDATE_VF_STAT(E1000_VFGPTC,
1222             hw_stats->last_gptc, hw_stats->gptc);
1223
1224         /* Good Tx octets, include VF loopback */
1225         UPDATE_VF_STAT(E1000_VFGOTC,
1226             hw_stats->last_gotc, hw_stats->gotc);
1227
1228         /* Rx Multicst packets */
1229         UPDATE_VF_STAT(E1000_VFMPRC,
1230             hw_stats->last_mprc, hw_stats->mprc);
1231
1232         /* Good Rx loopback packets */
1233         UPDATE_VF_STAT(E1000_VFGPRLBC,
1234             hw_stats->last_gprlbc, hw_stats->gprlbc);
1235
1236         /* Good Rx loopback octets */
1237         UPDATE_VF_STAT(E1000_VFGORLBC,
1238             hw_stats->last_gorlbc, hw_stats->gorlbc);
1239
1240         /* Good Tx loopback packets */
1241         UPDATE_VF_STAT(E1000_VFGPTLBC,
1242             hw_stats->last_gptlbc, hw_stats->gptlbc);
1243
1244         /* Good Tx loopback octets */
1245         UPDATE_VF_STAT(E1000_VFGOTLBC,
1246             hw_stats->last_gotlbc, hw_stats->gotlbc);
1247
1248         if (rte_stats == NULL)
1249                 return;
1250
1251         memset(rte_stats, 0, sizeof(*rte_stats));
1252         rte_stats->ipackets = hw_stats->gprc;
1253         rte_stats->ibytes = hw_stats->gorc;
1254         rte_stats->opackets = hw_stats->gptc;
1255         rte_stats->obytes = hw_stats->gotc;
1256         rte_stats->imcasts = hw_stats->mprc;
1257         rte_stats->ilbpackets = hw_stats->gprlbc;
1258         rte_stats->ilbbytes = hw_stats->gorlbc;
1259         rte_stats->olbpackets = hw_stats->gptlbc;
1260         rte_stats->olbbytes = hw_stats->gotlbc;
1261
1262 }
1263
1264 static void
1265 eth_igbvf_stats_reset(struct rte_eth_dev *dev)
1266 {
1267         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1268                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1269
1270         /* Sync HW register to the last stats */
1271         eth_igbvf_stats_get(dev, NULL);
1272
1273         /* reset HW current stats*/
1274         memset(&hw_stats->gprc, 0, sizeof(*hw_stats) -
1275                offsetof(struct e1000_vf_stats, gprc));
1276
1277 }
1278
1279 static void
1280 eth_igb_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1281 {
1282         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1283
1284         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1285         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1286         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1287         dev_info->rx_offload_capa =
1288                 DEV_RX_OFFLOAD_VLAN_STRIP |
1289                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1290                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1291                 DEV_RX_OFFLOAD_TCP_CKSUM;
1292         dev_info->tx_offload_capa =
1293                 DEV_TX_OFFLOAD_VLAN_INSERT |
1294                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1295                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1296                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1297                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1298
1299         switch (hw->mac.type) {
1300         case e1000_82575:
1301                 dev_info->max_rx_queues = 4;
1302                 dev_info->max_tx_queues = 4;
1303                 dev_info->max_vmdq_pools = 0;
1304                 break;
1305
1306         case e1000_82576:
1307                 dev_info->max_rx_queues = 16;
1308                 dev_info->max_tx_queues = 16;
1309                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1310                 dev_info->vmdq_queue_num = 16;
1311                 break;
1312
1313         case e1000_82580:
1314                 dev_info->max_rx_queues = 8;
1315                 dev_info->max_tx_queues = 8;
1316                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1317                 dev_info->vmdq_queue_num = 8;
1318                 break;
1319
1320         case e1000_i350:
1321                 dev_info->max_rx_queues = 8;
1322                 dev_info->max_tx_queues = 8;
1323                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1324                 dev_info->vmdq_queue_num = 8;
1325                 break;
1326
1327         case e1000_i354:
1328                 dev_info->max_rx_queues = 8;
1329                 dev_info->max_tx_queues = 8;
1330                 break;
1331
1332         case e1000_i210:
1333                 dev_info->max_rx_queues = 4;
1334                 dev_info->max_tx_queues = 4;
1335                 dev_info->max_vmdq_pools = 0;
1336                 break;
1337
1338         case e1000_i211:
1339                 dev_info->max_rx_queues = 2;
1340                 dev_info->max_tx_queues = 2;
1341                 dev_info->max_vmdq_pools = 0;
1342                 break;
1343
1344         default:
1345                 /* Should not happen */
1346                 break;
1347         }
1348         dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
1349
1350         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1351                 .rx_thresh = {
1352                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1353                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1354                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
1355                 },
1356                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
1357                 .rx_drop_en = 0,
1358         };
1359
1360         dev_info->default_txconf = (struct rte_eth_txconf) {
1361                 .tx_thresh = {
1362                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
1363                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
1364                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
1365                 },
1366                 .txq_flags = 0,
1367         };
1368 }
1369
1370 static void
1371 eth_igbvf_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1372 {
1373         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1374
1375         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1376         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1377         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1378         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
1379                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1380                                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1381                                 DEV_RX_OFFLOAD_TCP_CKSUM;
1382         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
1383                                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1384                                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1385                                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1386                                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1387         switch (hw->mac.type) {
1388         case e1000_vfadapt:
1389                 dev_info->max_rx_queues = 2;
1390                 dev_info->max_tx_queues = 2;
1391                 break;
1392         case e1000_vfadapt_i350:
1393                 dev_info->max_rx_queues = 1;
1394                 dev_info->max_tx_queues = 1;
1395                 break;
1396         default:
1397                 /* Should not happen */
1398                 break;
1399         }
1400
1401         dev_info->default_rxconf = (struct rte_eth_rxconf) {
1402                 .rx_thresh = {
1403                         .pthresh = IGB_DEFAULT_RX_PTHRESH,
1404                         .hthresh = IGB_DEFAULT_RX_HTHRESH,
1405                         .wthresh = IGB_DEFAULT_RX_WTHRESH,
1406                 },
1407                 .rx_free_thresh = IGB_DEFAULT_RX_FREE_THRESH,
1408                 .rx_drop_en = 0,
1409         };
1410
1411         dev_info->default_txconf = (struct rte_eth_txconf) {
1412                 .tx_thresh = {
1413                         .pthresh = IGB_DEFAULT_TX_PTHRESH,
1414                         .hthresh = IGB_DEFAULT_TX_HTHRESH,
1415                         .wthresh = IGB_DEFAULT_TX_WTHRESH,
1416                 },
1417                 .txq_flags = 0,
1418         };
1419 }
1420
1421 /* return 0 means link status changed, -1 means not changed */
1422 static int
1423 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1424 {
1425         struct e1000_hw *hw =
1426                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1427         struct rte_eth_link link, old;
1428         int link_check, count;
1429
1430         link_check = 0;
1431         hw->mac.get_link_status = 1;
1432
1433         /* possible wait-to-complete in up to 9 seconds */
1434         for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) {
1435                 /* Read the real link status */
1436                 switch (hw->phy.media_type) {
1437                 case e1000_media_type_copper:
1438                         /* Do the work to read phy */
1439                         e1000_check_for_link(hw);
1440                         link_check = !hw->mac.get_link_status;
1441                         break;
1442
1443                 case e1000_media_type_fiber:
1444                         e1000_check_for_link(hw);
1445                         link_check = (E1000_READ_REG(hw, E1000_STATUS) &
1446                                       E1000_STATUS_LU);
1447                         break;
1448
1449                 case e1000_media_type_internal_serdes:
1450                         e1000_check_for_link(hw);
1451                         link_check = hw->mac.serdes_has_link;
1452                         break;
1453
1454                 /* VF device is type_unknown */
1455                 case e1000_media_type_unknown:
1456                         eth_igbvf_link_update(hw);
1457                         link_check = !hw->mac.get_link_status;
1458                         break;
1459
1460                 default:
1461                         break;
1462                 }
1463                 if (link_check || wait_to_complete == 0)
1464                         break;
1465                 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL);
1466         }
1467         memset(&link, 0, sizeof(link));
1468         rte_igb_dev_atomic_read_link_status(dev, &link);
1469         old = link;
1470
1471         /* Now we check if a transition has happened */
1472         if (link_check) {
1473                 hw->mac.ops.get_link_up_info(hw, &link.link_speed,
1474                                           &link.link_duplex);
1475                 link.link_status = 1;
1476         } else if (!link_check) {
1477                 link.link_speed = 0;
1478                 link.link_duplex = 0;
1479                 link.link_status = 0;
1480         }
1481         rte_igb_dev_atomic_write_link_status(dev, &link);
1482
1483         /* not changed */
1484         if (old.link_status == link.link_status)
1485                 return -1;
1486
1487         /* changed */
1488         return 0;
1489 }
1490
1491 /*
1492  * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
1493  * For ASF and Pass Through versions of f/w this means
1494  * that the driver is loaded.
1495  */
1496 static void
1497 igb_hw_control_acquire(struct e1000_hw *hw)
1498 {
1499         uint32_t ctrl_ext;
1500
1501         /* Let firmware know the driver has taken over */
1502         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1503         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1504 }
1505
1506 /*
1507  * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
1508  * For ASF and Pass Through versions of f/w this means that the
1509  * driver is no longer loaded.
1510  */
1511 static void
1512 igb_hw_control_release(struct e1000_hw *hw)
1513 {
1514         uint32_t ctrl_ext;
1515
1516         /* Let firmware taken over control of h/w */
1517         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1518         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1519                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1520 }
1521
1522 /*
1523  * Bit of a misnomer, what this really means is
1524  * to enable OS management of the system... aka
1525  * to disable special hardware management features.
1526  */
1527 static void
1528 igb_init_manageability(struct e1000_hw *hw)
1529 {
1530         if (e1000_enable_mng_pass_thru(hw)) {
1531                 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H);
1532                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
1533
1534                 /* disable hardware interception of ARP */
1535                 manc &= ~(E1000_MANC_ARP_EN);
1536
1537                 /* enable receiving management packets to the host */
1538                 manc |= E1000_MANC_EN_MNG2HOST;
1539                 manc2h |= 1 << 5;  /* Mng Port 623 */
1540                 manc2h |= 1 << 6;  /* Mng Port 664 */
1541                 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h);
1542                 E1000_WRITE_REG(hw, E1000_MANC, manc);
1543         }
1544 }
1545
1546 static void
1547 igb_release_manageability(struct e1000_hw *hw)
1548 {
1549         if (e1000_enable_mng_pass_thru(hw)) {
1550                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
1551
1552                 manc |= E1000_MANC_ARP_EN;
1553                 manc &= ~E1000_MANC_EN_MNG2HOST;
1554
1555                 E1000_WRITE_REG(hw, E1000_MANC, manc);
1556         }
1557 }
1558
1559 static void
1560 eth_igb_promiscuous_enable(struct rte_eth_dev *dev)
1561 {
1562         struct e1000_hw *hw =
1563                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1564         uint32_t rctl;
1565
1566         rctl = E1000_READ_REG(hw, E1000_RCTL);
1567         rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1568         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1569 }
1570
1571 static void
1572 eth_igb_promiscuous_disable(struct rte_eth_dev *dev)
1573 {
1574         struct e1000_hw *hw =
1575                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1576         uint32_t rctl;
1577
1578         rctl = E1000_READ_REG(hw, E1000_RCTL);
1579         rctl &= (~E1000_RCTL_UPE);
1580         if (dev->data->all_multicast == 1)
1581                 rctl |= E1000_RCTL_MPE;
1582         else
1583                 rctl &= (~E1000_RCTL_MPE);
1584         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1585 }
1586
1587 static void
1588 eth_igb_allmulticast_enable(struct rte_eth_dev *dev)
1589 {
1590         struct e1000_hw *hw =
1591                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1592         uint32_t rctl;
1593
1594         rctl = E1000_READ_REG(hw, E1000_RCTL);
1595         rctl |= E1000_RCTL_MPE;
1596         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1597 }
1598
1599 static void
1600 eth_igb_allmulticast_disable(struct rte_eth_dev *dev)
1601 {
1602         struct e1000_hw *hw =
1603                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1604         uint32_t rctl;
1605
1606         if (dev->data->promiscuous == 1)
1607                 return; /* must remain in all_multicast mode */
1608         rctl = E1000_READ_REG(hw, E1000_RCTL);
1609         rctl &= (~E1000_RCTL_MPE);
1610         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1611 }
1612
1613 static int
1614 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1615 {
1616         struct e1000_hw *hw =
1617                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1618         struct e1000_vfta * shadow_vfta =
1619                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1620         uint32_t vfta;
1621         uint32_t vid_idx;
1622         uint32_t vid_bit;
1623
1624         vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) &
1625                               E1000_VFTA_ENTRY_MASK);
1626         vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK));
1627         vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx);
1628         if (on)
1629                 vfta |= vid_bit;
1630         else
1631                 vfta &= ~vid_bit;
1632         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta);
1633
1634         /* update local VFTA copy */
1635         shadow_vfta->vfta[vid_idx] = vfta;
1636
1637         return 0;
1638 }
1639
1640 static void
1641 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1642 {
1643         struct e1000_hw *hw =
1644                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1645         uint32_t reg = ETHER_TYPE_VLAN ;
1646
1647         reg |= (tpid << 16);
1648         E1000_WRITE_REG(hw, E1000_VET, reg);
1649 }
1650
1651 static void
1652 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1653 {
1654         struct e1000_hw *hw =
1655                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1656         uint32_t reg;
1657
1658         /* Filter Table Disable */
1659         reg = E1000_READ_REG(hw, E1000_RCTL);
1660         reg &= ~E1000_RCTL_CFIEN;
1661         reg &= ~E1000_RCTL_VFE;
1662         E1000_WRITE_REG(hw, E1000_RCTL, reg);
1663 }
1664
1665 static void
1666 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1667 {
1668         struct e1000_hw *hw =
1669                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1670         struct e1000_vfta * shadow_vfta =
1671                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1672         uint32_t reg;
1673         int i;
1674
1675         /* Filter Table Enable, CFI not used for packet acceptance */
1676         reg = E1000_READ_REG(hw, E1000_RCTL);
1677         reg &= ~E1000_RCTL_CFIEN;
1678         reg |= E1000_RCTL_VFE;
1679         E1000_WRITE_REG(hw, E1000_RCTL, reg);
1680
1681         /* restore VFTA table */
1682         for (i = 0; i < IGB_VFTA_SIZE; i++)
1683                 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]);
1684 }
1685
1686 static void
1687 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev)
1688 {
1689         struct e1000_hw *hw =
1690                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1691         uint32_t reg;
1692
1693         /* VLAN Mode Disable */
1694         reg = E1000_READ_REG(hw, E1000_CTRL);
1695         reg &= ~E1000_CTRL_VME;
1696         E1000_WRITE_REG(hw, E1000_CTRL, reg);
1697 }
1698
1699 static void
1700 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev)
1701 {
1702         struct e1000_hw *hw =
1703                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1704         uint32_t reg;
1705
1706         /* VLAN Mode Enable */
1707         reg = E1000_READ_REG(hw, E1000_CTRL);
1708         reg |= E1000_CTRL_VME;
1709         E1000_WRITE_REG(hw, E1000_CTRL, reg);
1710 }
1711
1712 static void
1713 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1714 {
1715         struct e1000_hw *hw =
1716                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1717         uint32_t reg;
1718
1719         /* CTRL_EXT: Extended VLAN */
1720         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
1721         reg &= ~E1000_CTRL_EXT_EXTEND_VLAN;
1722         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
1723
1724         /* Update maximum packet length */
1725         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
1726                 E1000_WRITE_REG(hw, E1000_RLPML,
1727                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
1728                                                 VLAN_TAG_SIZE);
1729 }
1730
1731 static void
1732 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1733 {
1734         struct e1000_hw *hw =
1735                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1736         uint32_t reg;
1737
1738         /* CTRL_EXT: Extended VLAN */
1739         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
1740         reg |= E1000_CTRL_EXT_EXTEND_VLAN;
1741         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
1742
1743         /* Update maximum packet length */
1744         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
1745                 E1000_WRITE_REG(hw, E1000_RLPML,
1746                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
1747                                                 2 * VLAN_TAG_SIZE);
1748 }
1749
1750 static void
1751 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1752 {
1753         if(mask & ETH_VLAN_STRIP_MASK){
1754                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1755                         igb_vlan_hw_strip_enable(dev);
1756                 else
1757                         igb_vlan_hw_strip_disable(dev);
1758         }
1759
1760         if(mask & ETH_VLAN_FILTER_MASK){
1761                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1762                         igb_vlan_hw_filter_enable(dev);
1763                 else
1764                         igb_vlan_hw_filter_disable(dev);
1765         }
1766
1767         if(mask & ETH_VLAN_EXTEND_MASK){
1768                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1769                         igb_vlan_hw_extend_enable(dev);
1770                 else
1771                         igb_vlan_hw_extend_disable(dev);
1772         }
1773 }
1774
1775
1776 /**
1777  * It enables the interrupt mask and then enable the interrupt.
1778  *
1779  * @param dev
1780  *  Pointer to struct rte_eth_dev.
1781  *
1782  * @return
1783  *  - On success, zero.
1784  *  - On failure, a negative value.
1785  */
1786 static int
1787 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev)
1788 {
1789         struct e1000_interrupt *intr =
1790                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1791
1792         intr->mask |= E1000_ICR_LSC;
1793
1794         return 0;
1795 }
1796
1797 /*
1798  * It reads ICR and gets interrupt causes, check it and set a bit flag
1799  * to update link status.
1800  *
1801  * @param dev
1802  *  Pointer to struct rte_eth_dev.
1803  *
1804  * @return
1805  *  - On success, zero.
1806  *  - On failure, a negative value.
1807  */
1808 static int
1809 eth_igb_interrupt_get_status(struct rte_eth_dev *dev)
1810 {
1811         uint32_t icr;
1812         struct e1000_hw *hw =
1813                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1814         struct e1000_interrupt *intr =
1815                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1816
1817         igb_intr_disable(hw);
1818
1819         /* read-on-clear nic registers here */
1820         icr = E1000_READ_REG(hw, E1000_ICR);
1821
1822         intr->flags = 0;
1823         if (icr & E1000_ICR_LSC) {
1824                 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
1825         }
1826
1827         if (icr & E1000_ICR_VMMB)
1828                 intr->flags |= E1000_FLAG_MAILBOX;
1829
1830         return 0;
1831 }
1832
1833 /*
1834  * It executes link_update after knowing an interrupt is prsent.
1835  *
1836  * @param dev
1837  *  Pointer to struct rte_eth_dev.
1838  *
1839  * @return
1840  *  - On success, zero.
1841  *  - On failure, a negative value.
1842  */
1843 static int
1844 eth_igb_interrupt_action(struct rte_eth_dev *dev)
1845 {
1846         struct e1000_hw *hw =
1847                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1848         struct e1000_interrupt *intr =
1849                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1850         uint32_t tctl, rctl;
1851         struct rte_eth_link link;
1852         int ret;
1853
1854         if (intr->flags & E1000_FLAG_MAILBOX) {
1855                 igb_pf_mbx_process(dev);
1856                 intr->flags &= ~E1000_FLAG_MAILBOX;
1857         }
1858
1859         igb_intr_enable(dev);
1860         rte_intr_enable(&(dev->pci_dev->intr_handle));
1861
1862         if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) {
1863                 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE;
1864
1865                 /* set get_link_status to check register later */
1866                 hw->mac.get_link_status = 1;
1867                 ret = eth_igb_link_update(dev, 0);
1868
1869                 /* check if link has changed */
1870                 if (ret < 0)
1871                         return 0;
1872
1873                 memset(&link, 0, sizeof(link));
1874                 rte_igb_dev_atomic_read_link_status(dev, &link);
1875                 if (link.link_status) {
1876                         PMD_INIT_LOG(INFO,
1877                                      " Port %d: Link Up - speed %u Mbps - %s",
1878                                      dev->data->port_id,
1879                                      (unsigned)link.link_speed,
1880                                      link.link_duplex == ETH_LINK_FULL_DUPLEX ?
1881                                      "full-duplex" : "half-duplex");
1882                 } else {
1883                         PMD_INIT_LOG(INFO, " Port %d: Link Down",
1884                                      dev->data->port_id);
1885                 }
1886                 PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
1887                              dev->pci_dev->addr.domain,
1888                              dev->pci_dev->addr.bus,
1889                              dev->pci_dev->addr.devid,
1890                              dev->pci_dev->addr.function);
1891                 tctl = E1000_READ_REG(hw, E1000_TCTL);
1892                 rctl = E1000_READ_REG(hw, E1000_RCTL);
1893                 if (link.link_status) {
1894                         /* enable Tx/Rx */
1895                         tctl |= E1000_TCTL_EN;
1896                         rctl |= E1000_RCTL_EN;
1897                 } else {
1898                         /* disable Tx/Rx */
1899                         tctl &= ~E1000_TCTL_EN;
1900                         rctl &= ~E1000_RCTL_EN;
1901                 }
1902                 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
1903                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1904                 E1000_WRITE_FLUSH(hw);
1905                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
1906         }
1907
1908         return 0;
1909 }
1910
1911 /**
1912  * Interrupt handler which shall be registered at first.
1913  *
1914  * @param handle
1915  *  Pointer to interrupt handle.
1916  * @param param
1917  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1918  *
1919  * @return
1920  *  void
1921  */
1922 static void
1923 eth_igb_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
1924                                                         void *param)
1925 {
1926         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1927
1928         eth_igb_interrupt_get_status(dev);
1929         eth_igb_interrupt_action(dev);
1930 }
1931
1932 static int
1933 eth_igb_led_on(struct rte_eth_dev *dev)
1934 {
1935         struct e1000_hw *hw;
1936
1937         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1938         return (e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
1939 }
1940
1941 static int
1942 eth_igb_led_off(struct rte_eth_dev *dev)
1943 {
1944         struct e1000_hw *hw;
1945
1946         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1947         return (e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
1948 }
1949
1950 static int
1951 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1952 {
1953         struct e1000_hw *hw;
1954         uint32_t ctrl;
1955         int tx_pause;
1956         int rx_pause;
1957
1958         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1959         fc_conf->pause_time = hw->fc.pause_time;
1960         fc_conf->high_water = hw->fc.high_water;
1961         fc_conf->low_water = hw->fc.low_water;
1962         fc_conf->send_xon = hw->fc.send_xon;
1963         fc_conf->autoneg = hw->mac.autoneg;
1964
1965         /*
1966          * Return rx_pause and tx_pause status according to actual setting of
1967          * the TFCE and RFCE bits in the CTRL register.
1968          */
1969         ctrl = E1000_READ_REG(hw, E1000_CTRL);
1970         if (ctrl & E1000_CTRL_TFCE)
1971                 tx_pause = 1;
1972         else
1973                 tx_pause = 0;
1974
1975         if (ctrl & E1000_CTRL_RFCE)
1976                 rx_pause = 1;
1977         else
1978                 rx_pause = 0;
1979
1980         if (rx_pause && tx_pause)
1981                 fc_conf->mode = RTE_FC_FULL;
1982         else if (rx_pause)
1983                 fc_conf->mode = RTE_FC_RX_PAUSE;
1984         else if (tx_pause)
1985                 fc_conf->mode = RTE_FC_TX_PAUSE;
1986         else
1987                 fc_conf->mode = RTE_FC_NONE;
1988
1989         return 0;
1990 }
1991
1992 static int
1993 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1994 {
1995         struct e1000_hw *hw;
1996         int err;
1997         enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = {
1998                 e1000_fc_none,
1999                 e1000_fc_rx_pause,
2000                 e1000_fc_tx_pause,
2001                 e1000_fc_full
2002         };
2003         uint32_t rx_buf_size;
2004         uint32_t max_high_water;
2005         uint32_t rctl;
2006
2007         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2008         if (fc_conf->autoneg != hw->mac.autoneg)
2009                 return -ENOTSUP;
2010         rx_buf_size = igb_get_rx_buffer_size(hw);
2011         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
2012
2013         /* At least reserve one Ethernet frame for watermark */
2014         max_high_water = rx_buf_size - ETHER_MAX_LEN;
2015         if ((fc_conf->high_water > max_high_water) ||
2016             (fc_conf->high_water < fc_conf->low_water)) {
2017                 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value");
2018                 PMD_INIT_LOG(ERR, "high water must <=  0x%x", max_high_water);
2019                 return (-EINVAL);
2020         }
2021
2022         hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode];
2023         hw->fc.pause_time     = fc_conf->pause_time;
2024         hw->fc.high_water     = fc_conf->high_water;
2025         hw->fc.low_water      = fc_conf->low_water;
2026         hw->fc.send_xon       = fc_conf->send_xon;
2027
2028         err = e1000_setup_link_generic(hw);
2029         if (err == E1000_SUCCESS) {
2030
2031                 /* check if we want to forward MAC frames - driver doesn't have native
2032                  * capability to do that, so we'll write the registers ourselves */
2033
2034                 rctl = E1000_READ_REG(hw, E1000_RCTL);
2035
2036                 /* set or clear MFLCN.PMCF bit depending on configuration */
2037                 if (fc_conf->mac_ctrl_frame_fwd != 0)
2038                         rctl |= E1000_RCTL_PMCF;
2039                 else
2040                         rctl &= ~E1000_RCTL_PMCF;
2041
2042                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
2043                 E1000_WRITE_FLUSH(hw);
2044
2045                 return 0;
2046         }
2047
2048         PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x", err);
2049         return (-EIO);
2050 }
2051
2052 #define E1000_RAH_POOLSEL_SHIFT      (18)
2053 static void
2054 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
2055                 uint32_t index, __rte_unused uint32_t pool)
2056 {
2057         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2058         uint32_t rah;
2059
2060         e1000_rar_set(hw, mac_addr->addr_bytes, index);
2061         rah = E1000_READ_REG(hw, E1000_RAH(index));
2062         rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool));
2063         E1000_WRITE_REG(hw, E1000_RAH(index), rah);
2064 }
2065
2066 static void
2067 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index)
2068 {
2069         uint8_t addr[ETHER_ADDR_LEN];
2070         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2071
2072         memset(addr, 0, sizeof(addr));
2073
2074         e1000_rar_set(hw, addr, index);
2075 }
2076
2077 /*
2078  * Virtual Function operations
2079  */
2080 static void
2081 igbvf_intr_disable(struct e1000_hw *hw)
2082 {
2083         PMD_INIT_FUNC_TRACE();
2084
2085         /* Clear interrupt mask to stop from interrupts being generated */
2086         E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF);
2087
2088         E1000_WRITE_FLUSH(hw);
2089 }
2090
2091 static void
2092 igbvf_stop_adapter(struct rte_eth_dev *dev)
2093 {
2094         u32 reg_val;
2095         u16 i;
2096         struct rte_eth_dev_info dev_info;
2097         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2098
2099         memset(&dev_info, 0, sizeof(dev_info));
2100         eth_igbvf_infos_get(dev, &dev_info);
2101
2102         /* Clear interrupt mask to stop from interrupts being generated */
2103         igbvf_intr_disable(hw);
2104
2105         /* Clear any pending interrupts, flush previous writes */
2106         E1000_READ_REG(hw, E1000_EICR);
2107
2108         /* Disable the transmit unit.  Each queue must be disabled. */
2109         for (i = 0; i < dev_info.max_tx_queues; i++)
2110                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH);
2111
2112         /* Disable the receive unit by stopping each queue */
2113         for (i = 0; i < dev_info.max_rx_queues; i++) {
2114                 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i));
2115                 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE;
2116                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val);
2117                 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE)
2118                         ;
2119         }
2120
2121         /* flush all queues disables */
2122         E1000_WRITE_FLUSH(hw);
2123         msec_delay(2);
2124 }
2125
2126 static int eth_igbvf_link_update(struct e1000_hw *hw)
2127 {
2128         struct e1000_mbx_info *mbx = &hw->mbx;
2129         struct e1000_mac_info *mac = &hw->mac;
2130         int ret_val = E1000_SUCCESS;
2131
2132         PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf");
2133
2134         /*
2135          * We only want to run this if there has been a rst asserted.
2136          * in this case that could mean a link change, device reset,
2137          * or a virtual function reset
2138          */
2139
2140         /* If we were hit with a reset or timeout drop the link */
2141         if (!e1000_check_for_rst(hw, 0) || !mbx->timeout)
2142                 mac->get_link_status = TRUE;
2143
2144         if (!mac->get_link_status)
2145                 goto out;
2146
2147         /* if link status is down no point in checking to see if pf is up */
2148         if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU))
2149                 goto out;
2150
2151         /* if we passed all the tests above then the link is up and we no
2152          * longer need to check for link */
2153         mac->get_link_status = FALSE;
2154
2155 out:
2156         return ret_val;
2157 }
2158
2159
2160 static int
2161 igbvf_dev_configure(struct rte_eth_dev *dev)
2162 {
2163         struct rte_eth_conf* conf = &dev->data->dev_conf;
2164
2165         PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
2166                      dev->data->port_id);
2167
2168         /*
2169          * VF has no ability to enable/disable HW CRC
2170          * Keep the persistent behavior the same as Host PF
2171          */
2172 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC
2173         if (!conf->rxmode.hw_strip_crc) {
2174                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip");
2175                 conf->rxmode.hw_strip_crc = 1;
2176         }
2177 #else
2178         if (conf->rxmode.hw_strip_crc) {
2179                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip");
2180                 conf->rxmode.hw_strip_crc = 0;
2181         }
2182 #endif
2183
2184         return 0;
2185 }
2186
2187 static int
2188 igbvf_dev_start(struct rte_eth_dev *dev)
2189 {
2190         struct e1000_hw *hw =
2191                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2192         int ret;
2193
2194         PMD_INIT_FUNC_TRACE();
2195
2196         hw->mac.ops.reset_hw(hw);
2197
2198         /* Set all vfta */
2199         igbvf_set_vfta_all(dev,1);
2200
2201         eth_igbvf_tx_init(dev);
2202
2203         /* This can fail when allocating mbufs for descriptor rings */
2204         ret = eth_igbvf_rx_init(dev);
2205         if (ret) {
2206                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
2207                 igb_dev_clear_queues(dev);
2208                 return ret;
2209         }
2210
2211         return 0;
2212 }
2213
2214 static void
2215 igbvf_dev_stop(struct rte_eth_dev *dev)
2216 {
2217         PMD_INIT_FUNC_TRACE();
2218
2219         igbvf_stop_adapter(dev);
2220
2221         /*
2222           * Clear what we set, but we still keep shadow_vfta to
2223           * restore after device starts
2224           */
2225         igbvf_set_vfta_all(dev,0);
2226
2227         igb_dev_clear_queues(dev);
2228 }
2229
2230 static void
2231 igbvf_dev_close(struct rte_eth_dev *dev)
2232 {
2233         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2234
2235         PMD_INIT_FUNC_TRACE();
2236
2237         e1000_reset_hw(hw);
2238
2239         igbvf_dev_stop(dev);
2240 }
2241
2242 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on)
2243 {
2244         struct e1000_mbx_info *mbx = &hw->mbx;
2245         uint32_t msgbuf[2];
2246
2247         /* After set vlan, vlan strip will also be enabled in igb driver*/
2248         msgbuf[0] = E1000_VF_SET_VLAN;
2249         msgbuf[1] = vid;
2250         /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */
2251         if (on)
2252                 msgbuf[0] |= E1000_VF_SET_VLAN_ADD;
2253
2254         return (mbx->ops.write_posted(hw, msgbuf, 2, 0));
2255 }
2256
2257 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2258 {
2259         struct e1000_hw *hw =
2260                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2261         struct e1000_vfta * shadow_vfta =
2262                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2263         int i = 0, j = 0, vfta = 0, mask = 1;
2264
2265         for (i = 0; i < IGB_VFTA_SIZE; i++){
2266                 vfta = shadow_vfta->vfta[i];
2267                 if(vfta){
2268                         mask = 1;
2269                         for (j = 0; j < 32; j++){
2270                                 if(vfta & mask)
2271                                         igbvf_set_vfta(hw,
2272                                                 (uint16_t)((i<<5)+j), on);
2273                                 mask<<=1;
2274                         }
2275                 }
2276         }
2277
2278 }
2279
2280 static int
2281 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2282 {
2283         struct e1000_hw *hw =
2284                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2285         struct e1000_vfta * shadow_vfta =
2286                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2287         uint32_t vid_idx = 0;
2288         uint32_t vid_bit = 0;
2289         int ret = 0;
2290
2291         PMD_INIT_FUNC_TRACE();
2292
2293         /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/
2294         ret = igbvf_set_vfta(hw, vlan_id, !!on);
2295         if(ret){
2296                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2297                 return ret;
2298         }
2299         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2300         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2301
2302         /*Save what we set and retore it after device reset*/
2303         if (on)
2304                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2305         else
2306                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2307
2308         return 0;
2309 }
2310
2311 static int
2312 eth_igb_rss_reta_update(struct rte_eth_dev *dev,
2313                         struct rte_eth_rss_reta_entry64 *reta_conf,
2314                         uint16_t reta_size)
2315 {
2316         uint8_t i, j, mask;
2317         uint32_t reta, r;
2318         uint16_t idx, shift;
2319         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2320
2321         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2322                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2323                         "(%d) doesn't match the number hardware can supported "
2324                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2325                 return -EINVAL;
2326         }
2327
2328         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
2329                 idx = i / RTE_RETA_GROUP_SIZE;
2330                 shift = i % RTE_RETA_GROUP_SIZE;
2331                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2332                                                 IGB_4_BIT_MASK);
2333                 if (!mask)
2334                         continue;
2335                 if (mask == IGB_4_BIT_MASK)
2336                         r = 0;
2337                 else
2338                         r = E1000_READ_REG(hw, E1000_RETA(i >> 2));
2339                 for (j = 0, reta = 0; j < IGB_4_BIT_WIDTH; j++) {
2340                         if (mask & (0x1 << j))
2341                                 reta |= reta_conf[idx].reta[shift + j] <<
2342                                                         (CHAR_BIT * j);
2343                         else
2344                                 reta |= r & (IGB_8_BIT_MASK << (CHAR_BIT * j));
2345                 }
2346                 E1000_WRITE_REG(hw, E1000_RETA(i >> 2), reta);
2347         }
2348
2349         return 0;
2350 }
2351
2352 static int
2353 eth_igb_rss_reta_query(struct rte_eth_dev *dev,
2354                        struct rte_eth_rss_reta_entry64 *reta_conf,
2355                        uint16_t reta_size)
2356 {
2357         uint8_t i, j, mask;
2358         uint32_t reta;
2359         uint16_t idx, shift;
2360         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2361
2362         if (reta_size != ETH_RSS_RETA_SIZE_128) {
2363                 PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2364                         "(%d) doesn't match the number hardware can supported "
2365                         "(%d)\n", reta_size, ETH_RSS_RETA_SIZE_128);
2366                 return -EINVAL;
2367         }
2368
2369         for (i = 0; i < reta_size; i += IGB_4_BIT_WIDTH) {
2370                 idx = i / RTE_RETA_GROUP_SIZE;
2371                 shift = i % RTE_RETA_GROUP_SIZE;
2372                 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2373                                                 IGB_4_BIT_MASK);
2374                 if (!mask)
2375                         continue;
2376                 reta = E1000_READ_REG(hw, E1000_RETA(i >> 2));
2377                 for (j = 0; j < IGB_4_BIT_WIDTH; j++) {
2378                         if (mask & (0x1 << j))
2379                                 reta_conf[idx].reta[shift + j] =
2380                                         ((reta >> (CHAR_BIT * j)) &
2381                                                 IGB_8_BIT_MASK);
2382                 }
2383         }
2384
2385         return 0;
2386 }
2387
2388 #define MAC_TYPE_FILTER_SUP(type)    do {\
2389         if ((type) != e1000_82580 && (type) != e1000_i350 &&\
2390                 (type) != e1000_82576)\
2391                 return -ENOSYS;\
2392 } while (0)
2393
2394 /*
2395  * add the syn filter
2396  *
2397  * @param
2398  * dev: Pointer to struct rte_eth_dev.
2399  * filter: ponter to the filter that will be added.
2400  * rx_queue: the queue id the filter assigned to.
2401  *
2402  * @return
2403  *    - On success, zero.
2404  *    - On failure, a negative value.
2405  */
2406 static int
2407 eth_igb_add_syn_filter(struct rte_eth_dev *dev,
2408                         struct rte_syn_filter *filter, uint16_t rx_queue)
2409 {
2410         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2411         uint32_t synqf, rfctl;
2412
2413         MAC_TYPE_FILTER_SUP(hw->mac.type);
2414
2415         if (rx_queue >= IGB_MAX_RX_QUEUE_NUM)
2416                 return -EINVAL;
2417
2418         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
2419         if (synqf & E1000_SYN_FILTER_ENABLE)
2420                 return -EINVAL;
2421
2422         synqf = (uint32_t)(((rx_queue << E1000_SYN_FILTER_QUEUE_SHIFT) &
2423                 E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE);
2424
2425         rfctl = E1000_READ_REG(hw, E1000_RFCTL);
2426         if (filter->hig_pri)
2427                 rfctl |= E1000_RFCTL_SYNQFP;
2428         else
2429                 rfctl &= ~E1000_RFCTL_SYNQFP;
2430
2431         E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf);
2432         E1000_WRITE_REG(hw, E1000_RFCTL, rfctl);
2433         return 0;
2434 }
2435
2436 /*
2437  * remove the syn filter
2438  *
2439  * @param
2440  * dev: Pointer to struct rte_eth_dev.
2441  *
2442  * @return
2443  *    - On success, zero.
2444  *    - On failure, a negative value.
2445  */
2446 static int
2447 eth_igb_remove_syn_filter(struct rte_eth_dev *dev)
2448 {
2449         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2450
2451         MAC_TYPE_FILTER_SUP(hw->mac.type);
2452
2453         E1000_WRITE_REG(hw, E1000_SYNQF(0), 0);
2454         return 0;
2455 }
2456
2457 /*
2458  * get the syn filter's info
2459  *
2460  * @param
2461  * dev: Pointer to struct rte_eth_dev.
2462  * filter: ponter to the filter that returns.
2463  * *rx_queue: pointer to the queue id the filter assigned to.
2464  *
2465  * @return
2466  *    - On success, zero.
2467  *    - On failure, a negative value.
2468  */
2469 static int
2470 eth_igb_get_syn_filter(struct rte_eth_dev *dev,
2471                         struct rte_syn_filter *filter, uint16_t *rx_queue)
2472 {
2473         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2474         uint32_t synqf, rfctl;
2475
2476         MAC_TYPE_FILTER_SUP(hw->mac.type);
2477         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
2478         if (synqf & E1000_SYN_FILTER_ENABLE) {
2479                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
2480                 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0;
2481                 *rx_queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >>
2482                                 E1000_SYN_FILTER_QUEUE_SHIFT);
2483                 return 0;
2484         }
2485         return -ENOENT;
2486 }
2487
2488 /*
2489  * add an ethertype filter
2490  *
2491  * @param
2492  * dev: Pointer to struct rte_eth_dev.
2493  * index: the index the filter allocates.
2494  * filter: ponter to the filter that will be added.
2495  * rx_queue: the queue id the filter assigned to.
2496  *
2497  * @return
2498  *    - On success, zero.
2499  *    - On failure, a negative value.
2500  */
2501 static int
2502 eth_igb_add_ethertype_filter(struct rte_eth_dev *dev, uint16_t index,
2503                         struct rte_ethertype_filter *filter, uint16_t rx_queue)
2504 {
2505         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2506         uint32_t etqf;
2507
2508         MAC_TYPE_FILTER_SUP(hw->mac.type);
2509
2510         if (index >= E1000_MAX_ETQF_FILTERS || rx_queue >= IGB_MAX_RX_QUEUE_NUM)
2511                 return -EINVAL;
2512
2513         etqf = E1000_READ_REG(hw, E1000_ETQF(index));
2514         if (etqf & E1000_ETQF_FILTER_ENABLE)
2515                 return -EINVAL;  /* filter index is in use. */
2516         else
2517                 etqf = 0;
2518
2519         etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE;
2520         etqf |= (uint32_t)(filter->ethertype & E1000_ETQF_ETHERTYPE);
2521         etqf |= rx_queue << E1000_ETQF_QUEUE_SHIFT;
2522
2523         if (filter->priority_en) {
2524                 PMD_INIT_LOG(ERR, "vlan and priority (%d) is not supported"
2525                         " in E1000.", filter->priority);
2526                 return -EINVAL;
2527         }
2528
2529         E1000_WRITE_REG(hw, E1000_ETQF(index), etqf);
2530         return 0;
2531 }
2532
2533 /*
2534  * remove an ethertype filter
2535  *
2536  * @param
2537  * dev: Pointer to struct rte_eth_dev.
2538  * index: the index the filter allocates.
2539  *
2540  * @return
2541  *    - On success, zero.
2542  *    - On failure, a negative value.
2543  */
2544 static int
2545 eth_igb_remove_ethertype_filter(struct rte_eth_dev *dev, uint16_t index)
2546 {
2547         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2548
2549         MAC_TYPE_FILTER_SUP(hw->mac.type);
2550
2551         if (index >= E1000_MAX_ETQF_FILTERS)
2552                 return -EINVAL;
2553
2554         E1000_WRITE_REG(hw, E1000_ETQF(index), 0);
2555         return 0;
2556 }
2557
2558 /*
2559  * get an ethertype filter
2560  *
2561  * @param
2562  * dev: Pointer to struct rte_eth_dev.
2563  * index: the index the filter allocates.
2564  * filter: ponter to the filter that will be gotten.
2565  * *rx_queue: the ponited of the queue id the filter assigned to.
2566  *
2567  * @return
2568  *    - On success, zero.
2569  *    - On failure, a negative value.
2570  */
2571 static int
2572 eth_igb_get_ethertype_filter(struct rte_eth_dev *dev, uint16_t index,
2573                         struct rte_ethertype_filter *filter, uint16_t *rx_queue)
2574 {
2575         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2576         uint32_t etqf;
2577
2578         MAC_TYPE_FILTER_SUP(hw->mac.type);
2579
2580         if (index >= E1000_MAX_ETQF_FILTERS)
2581                 return -EINVAL;
2582
2583         etqf = E1000_READ_REG(hw, E1000_ETQF(index));
2584         if (etqf & E1000_ETQF_FILTER_ENABLE) {
2585                 filter->ethertype = etqf & E1000_ETQF_ETHERTYPE;
2586                 filter->priority_en = 0;
2587                 *rx_queue = (etqf & E1000_ETQF_QUEUE) >> E1000_ETQF_QUEUE_SHIFT;
2588                 return 0;
2589         }
2590         return -ENOENT;
2591 }
2592
2593 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
2594         if ((type) != e1000_82580 && (type) != e1000_i350)\
2595                 return -ENOSYS; \
2596 } while (0)
2597
2598 /*
2599  * add a 2tuple filter
2600  *
2601  * @param
2602  * dev: Pointer to struct rte_eth_dev.
2603  * index: the index the filter allocates.
2604  * filter: ponter to the filter that will be added.
2605  * rx_queue: the queue id the filter assigned to.
2606  *
2607  * @return
2608  *    - On success, zero.
2609  *    - On failure, a negative value.
2610  */
2611 static int
2612 eth_igb_add_2tuple_filter(struct rte_eth_dev *dev, uint16_t index,
2613                         struct rte_2tuple_filter *filter, uint16_t rx_queue)
2614 {
2615         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2616         uint32_t ttqf, imir = 0;
2617         uint32_t imir_ext = 0;
2618
2619         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
2620
2621         if (index >= E1000_MAX_TTQF_FILTERS ||
2622                 rx_queue >= IGB_MAX_RX_QUEUE_NUM ||
2623                 filter->priority > E1000_2TUPLE_MAX_PRI)
2624                 return -EINVAL;  /* filter index is out of range. */
2625         if  (filter->tcp_flags > TCP_FLAG_ALL)
2626                 return -EINVAL;  /* flags is invalid. */
2627
2628         ttqf = E1000_READ_REG(hw, E1000_TTQF(index));
2629         if (ttqf & E1000_TTQF_QUEUE_ENABLE)
2630                 return -EINVAL;  /* filter index is in use. */
2631
2632         imir = (uint32_t)(filter->dst_port & E1000_IMIR_DSTPORT);
2633         if (filter->dst_port_mask == 1) /* 1b means not compare. */
2634                 imir |= E1000_IMIR_PORT_BP;
2635         else
2636                 imir &= ~E1000_IMIR_PORT_BP;
2637
2638         imir |= filter->priority << E1000_IMIR_PRIORITY_SHIFT;
2639
2640         ttqf = 0;
2641         ttqf |= E1000_TTQF_QUEUE_ENABLE;
2642         ttqf |= (uint32_t)(rx_queue << E1000_TTQF_QUEUE_SHIFT);
2643         ttqf |= (uint32_t)(filter->protocol & E1000_TTQF_PROTOCOL_MASK);
2644         if (filter->protocol_mask == 1)
2645                 ttqf |= E1000_TTQF_MASK_ENABLE;
2646         else
2647                 ttqf &= ~E1000_TTQF_MASK_ENABLE;
2648
2649         imir_ext |= E1000_IMIR_EXT_SIZE_BP;
2650         /* tcp flags bits setting. */
2651         if (filter->tcp_flags & TCP_FLAG_ALL) {
2652                 if (filter->tcp_flags & TCP_UGR_FLAG)
2653                         imir_ext |= E1000_IMIR_EXT_CTRL_UGR;
2654                 if (filter->tcp_flags & TCP_ACK_FLAG)
2655                         imir_ext |= E1000_IMIR_EXT_CTRL_ACK;
2656                 if (filter->tcp_flags & TCP_PSH_FLAG)
2657                         imir_ext |= E1000_IMIR_EXT_CTRL_PSH;
2658                 if (filter->tcp_flags & TCP_RST_FLAG)
2659                         imir_ext |= E1000_IMIR_EXT_CTRL_RST;
2660                 if (filter->tcp_flags & TCP_SYN_FLAG)
2661                         imir_ext |= E1000_IMIR_EXT_CTRL_SYN;
2662                 if (filter->tcp_flags & TCP_FIN_FLAG)
2663                         imir_ext |= E1000_IMIR_EXT_CTRL_FIN;
2664                 imir_ext &= ~E1000_IMIR_EXT_CTRL_BP;
2665         } else
2666                 imir_ext |= E1000_IMIR_EXT_CTRL_BP;
2667         E1000_WRITE_REG(hw, E1000_IMIR(index), imir);
2668         E1000_WRITE_REG(hw, E1000_TTQF(index), ttqf);
2669         E1000_WRITE_REG(hw, E1000_IMIREXT(index), imir_ext);
2670         return 0;
2671 }
2672
2673 /*
2674  * remove a 2tuple filter
2675  *
2676  * @param
2677  * dev: Pointer to struct rte_eth_dev.
2678  * index: the index the filter allocates.
2679  *
2680  * @return
2681  *    - On success, zero.
2682  *    - On failure, a negative value.
2683  */
2684 static int
2685 eth_igb_remove_2tuple_filter(struct rte_eth_dev *dev,
2686                         uint16_t index)
2687 {
2688         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2689
2690         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
2691
2692         if (index >= E1000_MAX_TTQF_FILTERS)
2693                 return -EINVAL;  /* filter index is out of range */
2694
2695         E1000_WRITE_REG(hw, E1000_TTQF(index), 0);
2696         E1000_WRITE_REG(hw, E1000_IMIR(index), 0);
2697         E1000_WRITE_REG(hw, E1000_IMIREXT(index), 0);
2698         return 0;
2699 }
2700
2701 /*
2702  * get a 2tuple filter
2703  *
2704  * @param
2705  * dev: Pointer to struct rte_eth_dev.
2706  * index: the index the filter allocates.
2707  * filter: ponter to the filter that returns.
2708  * *rx_queue: pointer of the queue id the filter assigned to.
2709  *
2710  * @return
2711  *    - On success, zero.
2712  *    - On failure, a negative value.
2713  */
2714 static int
2715 eth_igb_get_2tuple_filter(struct rte_eth_dev *dev, uint16_t index,
2716                         struct rte_2tuple_filter *filter, uint16_t *rx_queue)
2717 {
2718         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2719         uint32_t imir, ttqf, imir_ext;
2720
2721         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
2722
2723         if (index >= E1000_MAX_TTQF_FILTERS)
2724                 return -EINVAL;  /* filter index is out of range. */
2725
2726         ttqf = E1000_READ_REG(hw, E1000_TTQF(index));
2727         if (ttqf & E1000_TTQF_QUEUE_ENABLE) {
2728                 imir = E1000_READ_REG(hw, E1000_IMIR(index));
2729                 filter->protocol = ttqf & E1000_TTQF_PROTOCOL_MASK;
2730                 filter->protocol_mask = (ttqf & E1000_TTQF_MASK_ENABLE) ? 1 : 0;
2731                 *rx_queue = (ttqf & E1000_TTQF_RX_QUEUE_MASK) >>
2732                                 E1000_TTQF_QUEUE_SHIFT;
2733                 filter->dst_port = (uint16_t)(imir & E1000_IMIR_DSTPORT);
2734                 filter->dst_port_mask = (imir & E1000_IMIR_PORT_BP) ? 1 : 0;
2735                 filter->priority = (imir & E1000_IMIR_PRIORITY) >>
2736                         E1000_IMIR_PRIORITY_SHIFT;
2737
2738                 imir_ext = E1000_READ_REG(hw, E1000_IMIREXT(index));
2739                 if (!(imir_ext & E1000_IMIR_EXT_CTRL_BP)) {
2740                         if (imir_ext & E1000_IMIR_EXT_CTRL_UGR)
2741                                 filter->tcp_flags |= TCP_UGR_FLAG;
2742                         if (imir_ext & E1000_IMIR_EXT_CTRL_ACK)
2743                                 filter->tcp_flags |= TCP_ACK_FLAG;
2744                         if (imir_ext & E1000_IMIR_EXT_CTRL_PSH)
2745                                 filter->tcp_flags |= TCP_PSH_FLAG;
2746                         if (imir_ext & E1000_IMIR_EXT_CTRL_RST)
2747                                 filter->tcp_flags |= TCP_RST_FLAG;
2748                         if (imir_ext & E1000_IMIR_EXT_CTRL_SYN)
2749                                 filter->tcp_flags |= TCP_SYN_FLAG;
2750                         if (imir_ext & E1000_IMIR_EXT_CTRL_FIN)
2751                                 filter->tcp_flags |= TCP_FIN_FLAG;
2752                 } else
2753                         filter->tcp_flags = 0;
2754                 return 0;
2755         }
2756         return -ENOENT;
2757 }
2758
2759 /*
2760  * add a flex filter
2761  *
2762  * @param
2763  * dev: Pointer to struct rte_eth_dev.
2764  * index: the index the filter allocates.
2765  * filter: ponter to the filter that will be added.
2766  * rx_queue: the queue id the filter assigned to.
2767  *
2768  * @return
2769  *    - On success, zero.
2770  *     - On failure, a negative value.
2771  */
2772 static int
2773 eth_igb_add_flex_filter(struct rte_eth_dev *dev, uint16_t index,
2774                         struct rte_flex_filter *filter, uint16_t rx_queue)
2775 {
2776         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2777         uint32_t wufc, en_bits = 0;
2778         uint32_t queueing = 0;
2779         uint32_t reg_off = 0;
2780         uint8_t i, j = 0;
2781
2782         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
2783
2784         if (index >= E1000_MAX_FLEXIBLE_FILTERS)
2785                 return -EINVAL;  /* filter index is out of range. */
2786
2787         if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN ||
2788                 filter->len % 8 != 0 ||
2789                 filter->priority > E1000_MAX_FLEX_FILTER_PRI)
2790                 return -EINVAL;
2791
2792         wufc = E1000_READ_REG(hw, E1000_WUFC);
2793         en_bits = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << index);
2794         if ((wufc & en_bits) == en_bits)
2795                 return -EINVAL;  /* the filter is in use. */
2796
2797         E1000_WRITE_REG(hw, E1000_WUFC,
2798                 wufc | E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << index));
2799
2800         j = 0;
2801         if (index < E1000_MAX_FHFT)
2802                 reg_off = E1000_FHFT(index);
2803         else
2804                 reg_off = E1000_FHFT_EXT(index - E1000_MAX_FHFT);
2805
2806         for (i = 0; i < 16; i++) {
2807                 E1000_WRITE_REG(hw, reg_off + i*4*4, filter->dwords[j]);
2808                 E1000_WRITE_REG(hw, reg_off + (i*4+1)*4, filter->dwords[++j]);
2809                 E1000_WRITE_REG(hw, reg_off + (i*4+2)*4,
2810                                 (uint32_t)filter->mask[i]);
2811                 ++j;
2812         }
2813         queueing |= filter->len |
2814                 (rx_queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) |
2815                 (filter->priority << E1000_FHFT_QUEUEING_PRIO_SHIFT);
2816         E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET, queueing);
2817         return 0;
2818 }
2819
2820 /*
2821  * remove a flex filter
2822  *
2823  * @param
2824  * dev: Pointer to struct rte_eth_dev.
2825  * index: the index the filter allocates.
2826  *
2827  * @return
2828  *    - On success, zero.
2829  *    - On failure, a negative value.
2830  */
2831 static int
2832 eth_igb_remove_flex_filter(struct rte_eth_dev *dev,
2833                                 uint16_t index)
2834 {
2835         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2836         uint32_t wufc, reg_off = 0;
2837         uint8_t i;
2838
2839         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
2840
2841         if (index >= E1000_MAX_FLEXIBLE_FILTERS)
2842                 return -EINVAL;  /* filter index is out of range. */
2843
2844         wufc = E1000_READ_REG(hw, E1000_WUFC);
2845         E1000_WRITE_REG(hw, E1000_WUFC, wufc & (~(E1000_WUFC_FLX0 << index)));
2846
2847         if (index < E1000_MAX_FHFT)
2848                 reg_off = E1000_FHFT(index);
2849         else
2850                 reg_off = E1000_FHFT_EXT(index - E1000_MAX_FHFT);
2851
2852         for (i = 0; i < 64; i++)
2853                 E1000_WRITE_REG(hw, reg_off + i*4, 0);
2854         return 0;
2855 }
2856
2857 /*
2858  * get a flex filter
2859  *
2860  * @param
2861  * dev: Pointer to struct rte_eth_dev.
2862  * index: the index the filter allocates.
2863  * filter: ponter to the filter that returns.
2864  * *rx_queue: the pointer of the queue id the filter assigned to.
2865  *
2866  * @return
2867  *    - On success, zero.
2868  *    - On failure, a negative value.
2869  */
2870 static int
2871 eth_igb_get_flex_filter(struct rte_eth_dev *dev, uint16_t index,
2872                         struct rte_flex_filter *filter, uint16_t *rx_queue)
2873 {
2874         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2875         uint32_t wufc, queueing, wufc_en = 0;
2876         uint8_t i, j;
2877
2878         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
2879
2880         if (index >= E1000_MAX_FLEXIBLE_FILTERS)
2881                 return -EINVAL;  /* filter index is out of range. */
2882
2883         wufc = E1000_READ_REG(hw, E1000_WUFC);
2884         wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << index);
2885
2886         if ((wufc & wufc_en) == wufc_en) {
2887                 uint32_t reg_off = 0;
2888                 j = 0;
2889                 if (index < E1000_MAX_FHFT)
2890                         reg_off = E1000_FHFT(index);
2891                 else
2892                         reg_off = E1000_FHFT_EXT(index - E1000_MAX_FHFT);
2893
2894                 for (i = 0; i < 16; i++, j = i * 2) {
2895                         filter->dwords[j] =
2896                                 E1000_READ_REG(hw, reg_off + i*4*4);
2897                         filter->dwords[j+1] =
2898                                 E1000_READ_REG(hw, reg_off + (i*4+1)*4);
2899                         filter->mask[i] =
2900                                 E1000_READ_REG(hw, reg_off + (i*4+2)*4);
2901                 }
2902                 queueing = E1000_READ_REG(hw,
2903                                 reg_off + E1000_FHFT_QUEUEING_OFFSET);
2904                 filter->len = queueing & E1000_FHFT_QUEUEING_LEN;
2905                 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >>
2906                         E1000_FHFT_QUEUEING_PRIO_SHIFT;
2907                 *rx_queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >>
2908                         E1000_FHFT_QUEUEING_QUEUE_SHIFT;
2909                 return 0;
2910         }
2911         return -ENOENT;
2912 }
2913
2914 /*
2915  * add a 5tuple filter
2916  *
2917  * @param
2918  * dev: Pointer to struct rte_eth_dev.
2919  * index: the index the filter allocates.
2920  * filter: ponter to the filter that will be added.
2921  * rx_queue: the queue id the filter assigned to.
2922  *
2923  * @return
2924  *    - On success, zero.
2925  *    - On failure, a negative value.
2926  */
2927 static int
2928 eth_igb_add_5tuple_filter(struct rte_eth_dev *dev, uint16_t index,
2929                         struct rte_5tuple_filter *filter, uint16_t rx_queue)
2930 {
2931         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2932         uint32_t ftqf, spqf = 0;
2933         uint32_t imir = 0;
2934         uint32_t imir_ext = 0;
2935
2936         if (hw->mac.type != e1000_82576)
2937                 return -ENOSYS;
2938
2939         if (index >= E1000_MAX_FTQF_FILTERS ||
2940                 rx_queue >= IGB_MAX_RX_QUEUE_NUM_82576)
2941                 return -EINVAL;  /* filter index is out of range. */
2942
2943         ftqf = E1000_READ_REG(hw, E1000_FTQF(index));
2944         if (ftqf & E1000_FTQF_QUEUE_ENABLE)
2945                 return -EINVAL;  /* filter index is in use. */
2946
2947         ftqf = 0;
2948         ftqf |= filter->protocol & E1000_FTQF_PROTOCOL_MASK;
2949         if (filter->src_ip_mask == 1) /* 1b means not compare. */
2950                 ftqf |= E1000_FTQF_SOURCE_ADDR_MASK;
2951         if (filter->dst_ip_mask == 1)
2952                 ftqf |= E1000_FTQF_DEST_ADDR_MASK;
2953         if (filter->src_port_mask == 1)
2954                 ftqf |= E1000_FTQF_SOURCE_PORT_MASK;
2955         if (filter->protocol_mask == 1)
2956                 ftqf |= E1000_FTQF_PROTOCOL_COMP_MASK;
2957         ftqf |= (rx_queue << E1000_FTQF_QUEUE_SHIFT) & E1000_FTQF_QUEUE_MASK;
2958         ftqf |= E1000_FTQF_VF_MASK_EN;
2959         ftqf |= E1000_FTQF_QUEUE_ENABLE;
2960         E1000_WRITE_REG(hw, E1000_FTQF(index), ftqf);
2961         E1000_WRITE_REG(hw, E1000_DAQF(index), filter->dst_ip);
2962         E1000_WRITE_REG(hw, E1000_SAQF(index), filter->src_ip);
2963
2964         spqf |= filter->src_port & E1000_SPQF_SRCPORT;
2965         E1000_WRITE_REG(hw, E1000_SPQF(index), spqf);
2966
2967         imir |= (uint32_t)(filter->dst_port & E1000_IMIR_DSTPORT);
2968         if (filter->dst_port_mask == 1) /* 1b means not compare. */
2969                 imir |= E1000_IMIR_PORT_BP;
2970         else
2971                 imir &= ~E1000_IMIR_PORT_BP;
2972         imir |= filter->priority << E1000_IMIR_PRIORITY_SHIFT;
2973
2974         imir_ext |= E1000_IMIR_EXT_SIZE_BP;
2975         /* tcp flags bits setting. */
2976         if (filter->tcp_flags & TCP_FLAG_ALL) {
2977                 if (filter->tcp_flags & TCP_UGR_FLAG)
2978                         imir_ext |= E1000_IMIR_EXT_CTRL_UGR;
2979                 if (filter->tcp_flags & TCP_ACK_FLAG)
2980                         imir_ext |= E1000_IMIR_EXT_CTRL_ACK;
2981                 if (filter->tcp_flags & TCP_PSH_FLAG)
2982                         imir_ext |= E1000_IMIR_EXT_CTRL_PSH;
2983                 if (filter->tcp_flags & TCP_RST_FLAG)
2984                         imir_ext |= E1000_IMIR_EXT_CTRL_RST;
2985                 if (filter->tcp_flags & TCP_SYN_FLAG)
2986                         imir_ext |= E1000_IMIR_EXT_CTRL_SYN;
2987                 if (filter->tcp_flags & TCP_FIN_FLAG)
2988                         imir_ext |= E1000_IMIR_EXT_CTRL_FIN;
2989         } else
2990                 imir_ext |= E1000_IMIR_EXT_CTRL_BP;
2991         E1000_WRITE_REG(hw, E1000_IMIR(index), imir);
2992         E1000_WRITE_REG(hw, E1000_IMIREXT(index), imir_ext);
2993         return 0;
2994 }
2995
2996 /*
2997  * remove a 5tuple filter
2998  *
2999  * @param
3000  * dev: Pointer to struct rte_eth_dev.
3001  * index: the index the filter allocates
3002  *
3003  * @return
3004  *    - On success, zero.
3005  *    - On failure, a negative value.
3006  */
3007 static int
3008 eth_igb_remove_5tuple_filter(struct rte_eth_dev *dev,
3009                                 uint16_t index)
3010 {
3011         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3012
3013         if (hw->mac.type != e1000_82576)
3014                 return -ENOSYS;
3015
3016         if (index >= E1000_MAX_FTQF_FILTERS)
3017                 return -EINVAL;  /* filter index is out of range. */
3018
3019         E1000_WRITE_REG(hw, E1000_FTQF(index), 0);
3020         E1000_WRITE_REG(hw, E1000_DAQF(index), 0);
3021         E1000_WRITE_REG(hw, E1000_SAQF(index), 0);
3022         E1000_WRITE_REG(hw, E1000_SPQF(index), 0);
3023         E1000_WRITE_REG(hw, E1000_IMIR(index), 0);
3024         E1000_WRITE_REG(hw, E1000_IMIREXT(index), 0);
3025         return 0;
3026 }
3027
3028 /*
3029  * get a 5tuple filter
3030  *
3031  * @param
3032  * dev: Pointer to struct rte_eth_dev.
3033  * index: the index the filter allocates
3034  * filter: ponter to the filter that returns
3035  * *rx_queue: pointer of the queue id the filter assigned to
3036  *
3037  * @return
3038  *    - On success, zero.
3039  *    - On failure, a negative value.
3040  */
3041 static int
3042 eth_igb_get_5tuple_filter(struct rte_eth_dev *dev, uint16_t index,
3043                         struct rte_5tuple_filter *filter, uint16_t *rx_queue)
3044 {
3045         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3046         uint32_t spqf, ftqf, imir, imir_ext;
3047
3048         if (hw->mac.type != e1000_82576)
3049                 return -ENOSYS;
3050
3051         if (index >= E1000_MAX_FTQF_FILTERS)
3052                 return -EINVAL;  /* filter index is out of range. */
3053
3054         ftqf = E1000_READ_REG(hw, E1000_FTQF(index));
3055         if (ftqf & E1000_FTQF_QUEUE_ENABLE) {
3056                 filter->src_ip_mask =
3057                         (ftqf & E1000_FTQF_SOURCE_ADDR_MASK) ? 1 : 0;
3058                 filter->dst_ip_mask =
3059                         (ftqf & E1000_FTQF_DEST_ADDR_MASK) ? 1 : 0;
3060                 filter->src_port_mask =
3061                         (ftqf & E1000_FTQF_SOURCE_PORT_MASK) ? 1 : 0;
3062                 filter->protocol_mask =
3063                         (ftqf & E1000_FTQF_PROTOCOL_COMP_MASK) ? 1 : 0;
3064                 filter->protocol =
3065                         (uint8_t)ftqf & E1000_FTQF_PROTOCOL_MASK;
3066                 *rx_queue = (uint16_t)((ftqf & E1000_FTQF_QUEUE_MASK) >>
3067                                 E1000_FTQF_QUEUE_SHIFT);
3068
3069                 spqf = E1000_READ_REG(hw, E1000_SPQF(index));
3070                 filter->src_port = spqf & E1000_SPQF_SRCPORT;
3071
3072                 filter->dst_ip = E1000_READ_REG(hw, E1000_DAQF(index));
3073                 filter->src_ip = E1000_READ_REG(hw, E1000_SAQF(index));
3074
3075                 imir = E1000_READ_REG(hw, E1000_IMIR(index));
3076                 filter->dst_port_mask = (imir & E1000_IMIR_PORT_BP) ? 1 : 0;
3077                 filter->dst_port = (uint16_t)(imir & E1000_IMIR_DSTPORT);
3078                 filter->priority = (imir & E1000_IMIR_PRIORITY) >>
3079                         E1000_IMIR_PRIORITY_SHIFT;
3080
3081                 imir_ext = E1000_READ_REG(hw, E1000_IMIREXT(index));
3082                 if (!(imir_ext & E1000_IMIR_EXT_CTRL_BP)) {
3083                         if (imir_ext & E1000_IMIR_EXT_CTRL_UGR)
3084                                 filter->tcp_flags |= TCP_UGR_FLAG;
3085                         if (imir_ext & E1000_IMIR_EXT_CTRL_ACK)
3086                                 filter->tcp_flags |= TCP_ACK_FLAG;
3087                         if (imir_ext & E1000_IMIR_EXT_CTRL_PSH)
3088                                 filter->tcp_flags |= TCP_PSH_FLAG;
3089                         if (imir_ext & E1000_IMIR_EXT_CTRL_RST)
3090                                 filter->tcp_flags |= TCP_RST_FLAG;
3091                         if (imir_ext & E1000_IMIR_EXT_CTRL_SYN)
3092                                 filter->tcp_flags |= TCP_SYN_FLAG;
3093                         if (imir_ext & E1000_IMIR_EXT_CTRL_FIN)
3094                                 filter->tcp_flags |= TCP_FIN_FLAG;
3095                 } else
3096                         filter->tcp_flags = 0;
3097                 return 0;
3098         }
3099         return -ENOENT;
3100 }
3101
3102 static int
3103 eth_igb_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
3104 {
3105         uint32_t rctl;
3106         struct e1000_hw *hw;
3107         struct rte_eth_dev_info dev_info;
3108         uint32_t frame_size = mtu + (ETHER_HDR_LEN + ETHER_CRC_LEN +
3109                                      VLAN_TAG_SIZE);
3110
3111         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3112
3113 #ifdef RTE_LIBRTE_82571_SUPPORT
3114         /* XXX: not bigger than max_rx_pktlen */
3115         if (hw->mac.type == e1000_82571)
3116                 return -ENOTSUP;
3117 #endif
3118         eth_igb_infos_get(dev, &dev_info);
3119
3120         /* check that mtu is within the allowed range */
3121         if ((mtu < ETHER_MIN_MTU) ||
3122             (frame_size > dev_info.max_rx_pktlen))
3123                 return -EINVAL;
3124
3125         /* refuse mtu that requires the support of scattered packets when this
3126          * feature has not been enabled before. */
3127         if (!dev->data->scattered_rx &&
3128             frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)
3129                 return -EINVAL;
3130
3131         rctl = E1000_READ_REG(hw, E1000_RCTL);
3132
3133         /* switch to jumbo mode if needed */
3134         if (frame_size > ETHER_MAX_LEN) {
3135                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
3136                 rctl |= E1000_RCTL_LPE;
3137         } else {
3138                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
3139                 rctl &= ~E1000_RCTL_LPE;
3140         }
3141         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3142
3143         /* update max frame size */
3144         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
3145
3146         E1000_WRITE_REG(hw, E1000_RLPML,
3147                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
3148
3149         return 0;
3150 }
3151
3152 static struct rte_driver pmd_igb_drv = {
3153         .type = PMD_PDEV,
3154         .init = rte_igb_pmd_init,
3155 };
3156
3157 static struct rte_driver pmd_igbvf_drv = {
3158         .type = PMD_PDEV,
3159         .init = rte_igbvf_pmd_init,
3160 };
3161
3162 PMD_REGISTER_DRIVER(pmd_igb_drv);
3163 PMD_REGISTER_DRIVER(pmd_igbvf_drv);