lib/librte_pmd_e1000/igb_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <stdarg.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_interrupts.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_log.h>
  44 #include <rte_debug.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_memory.h>
  49 #include <rte_memzone.h>
  50 #include <rte_tailq.h>
  51 #include <rte_eal.h>
  52 #include <rte_atomic.h>
  53 #include <rte_malloc.h>
  54 #include <rte_dev.h>
  55
  56 #include "e1000_logs.h"
  57 #include "e1000/e1000_api.h"
  58 #include "e1000_ethdev.h"
  59
  60 static int  eth_igb_configure(struct rte_eth_dev *dev);
  61 static int  eth_igb_start(struct rte_eth_dev *dev);
  62 static void eth_igb_stop(struct rte_eth_dev *dev);
  63 static void eth_igb_close(struct rte_eth_dev *dev);
  64 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev);
  65 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev);
  66 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev);
  67 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev);
  68 static int  eth_igb_link_update(struct rte_eth_dev *dev,
  69                                 int wait_to_complete);
  70 static void eth_igb_stats_get(struct rte_eth_dev *dev,
  71                                 struct rte_eth_stats *rte_stats);
  72 static void eth_igb_stats_reset(struct rte_eth_dev *dev);
  73 static void eth_igb_infos_get(struct rte_eth_dev *dev,
  74                                 struct rte_eth_dev_info *dev_info);
  75 static int  eth_igb_flow_ctrl_get(struct rte_eth_dev *dev,
  76                                 struct rte_eth_fc_conf *fc_conf);
  77 static int  eth_igb_flow_ctrl_set(struct rte_eth_dev *dev,
  78                                 struct rte_eth_fc_conf *fc_conf);
  79 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev);
  80 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev);
  81 static int eth_igb_interrupt_action(struct rte_eth_dev *dev);
  82 static void eth_igb_interrupt_handler(struct rte_intr_handle *handle,
  83                                                         void *param);
  84 static int  igb_hardware_init(struct e1000_hw *hw);
  85 static void igb_hw_control_acquire(struct e1000_hw *hw);
  86 static void igb_hw_control_release(struct e1000_hw *hw);
  87 static void igb_init_manageability(struct e1000_hw *hw);
  88 static void igb_release_manageability(struct e1000_hw *hw);
  89
  90 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev,
  91                 uint16_t vlan_id, int on);
  92 static void eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
  93 static void eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask);
  94
  95 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev);
  96 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev);
  97 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev);
  98 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev);
  99 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev);
 100 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev);
 101
 102 static int eth_igb_led_on(struct rte_eth_dev *dev);
 103 static int eth_igb_led_off(struct rte_eth_dev *dev);
 104
 105 static void igb_intr_disable(struct e1000_hw *hw);
 106 static int  igb_get_rx_buffer_size(struct e1000_hw *hw);
 107 static void eth_igb_rar_set(struct rte_eth_dev *dev,
 108                 struct ether_addr *mac_addr,
 109                 uint32_t index, uint32_t pool);
 110 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 111
 112 static void igbvf_intr_disable(struct e1000_hw *hw);
 113 static int igbvf_dev_configure(struct rte_eth_dev *dev);
 114 static int igbvf_dev_start(struct rte_eth_dev *dev);
 115 static void igbvf_dev_stop(struct rte_eth_dev *dev);
 116 static void igbvf_dev_close(struct rte_eth_dev *dev);
 117 static int eth_igbvf_link_update(struct e1000_hw *hw);
 118 static void eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats);
 119 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev);
 120 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev,
 121                 uint16_t vlan_id, int on);
 122 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on);
 123 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 124 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev,
 125                  struct rte_eth_rss_reta *reta_conf);
 126 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev,
 127                 struct rte_eth_rss_reta *reta_conf);
 128
 129 static int eth_igb_add_syn_filter(struct rte_eth_dev *dev,
 130                         struct rte_syn_filter *filter, uint16_t rx_queue);
 131 static int eth_igb_remove_syn_filter(struct rte_eth_dev *dev);
 132 static int eth_igb_get_syn_filter(struct rte_eth_dev *dev,
 133                         struct rte_syn_filter *filter, uint16_t *rx_queue);
 134 static int eth_igb_add_ethertype_filter(struct rte_eth_dev *dev,
 135                         uint16_t index,
 136                         struct rte_ethertype_filter *filter, uint16_t rx_queue);
 137 static int eth_igb_remove_ethertype_filter(struct rte_eth_dev *dev,
 138                         uint16_t index);
 139 static int eth_igb_get_ethertype_filter(struct rte_eth_dev *dev,
 140                         uint16_t index,
 141                         struct rte_ethertype_filter *filter, uint16_t *rx_queue);
 142 static int eth_igb_add_2tuple_filter(struct rte_eth_dev *dev,
 143                         uint16_t index,
 144                         struct rte_2tuple_filter *filter, uint16_t rx_queue);
 145 static int eth_igb_remove_2tuple_filter(struct rte_eth_dev *dev,
 146                         uint16_t index);
 147 static int eth_igb_get_2tuple_filter(struct rte_eth_dev *dev,
 148                         uint16_t index,
 149                         struct rte_2tuple_filter *filter, uint16_t *rx_queue);
 150 static int eth_igb_add_flex_filter(struct rte_eth_dev *dev,
 151                         uint16_t index,
 152                         struct rte_flex_filter *filter, uint16_t rx_queue);
 153 static int eth_igb_remove_flex_filter(struct rte_eth_dev *dev,
 154                         uint16_t index);
 155 static int eth_igb_get_flex_filter(struct rte_eth_dev *dev,
 156                         uint16_t index,
 157                         struct rte_flex_filter *filter, uint16_t *rx_queue);
 158 static int eth_igb_add_5tuple_filter(struct rte_eth_dev *dev,
 159                         uint16_t index,
 160                         struct rte_5tuple_filter *filter, uint16_t rx_queue);
 161 static int eth_igb_remove_5tuple_filter(struct rte_eth_dev *dev,
 162                         uint16_t index);
 163 static int eth_igb_get_5tuple_filter(struct rte_eth_dev *dev,
 164                         uint16_t index,
 165                         struct rte_5tuple_filter *filter, uint16_t *rx_queue);
 166
 167 /*
 168  * Define VF Stats MACRO for Non "cleared on read" register
 169  */
 170 #define UPDATE_VF_STAT(reg, last, cur)            \
 171 {                                                 \
 172         u32 latest = E1000_READ_REG(hw, reg);     \
 173         cur += latest - last;                     \
 174         last = latest;                            \
 175 }
 176
 177
 178 #define IGB_FC_PAUSE_TIME 0x0680
 179 #define IGB_LINK_UPDATE_CHECK_TIMEOUT  90  /* 9s */
 180 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */
 181
 182 #define IGBVF_PMD_NAME "rte_igbvf_pmd"     /* PMD name */
 183
 184 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full;
 185
 186 /*
 187  * The set of PCI devices this driver supports
 188  */
 189 static struct rte_pci_id pci_id_igb_map[] = {
 190
 191 #define RTE_PCI_DEV_ID_DECL_IGB(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 192 #include "rte_pci_dev_ids.h"
 193
 194 {.device_id = 0},
 195 };
 196
 197 /*
 198  * The set of PCI devices this driver supports (for 82576&I350 VF)
 199  */
 200 static struct rte_pci_id pci_id_igbvf_map[] = {
 201
 202 #define RTE_PCI_DEV_ID_DECL_IGBVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 203 #include "rte_pci_dev_ids.h"
 204
 205 {.device_id = 0},
 206 };
 207
 208 static struct eth_dev_ops eth_igb_ops = {
 209         .dev_configure        = eth_igb_configure,
 210         .dev_start            = eth_igb_start,
 211         .dev_stop             = eth_igb_stop,
 212         .dev_close            = eth_igb_close,
 213         .promiscuous_enable   = eth_igb_promiscuous_enable,
 214         .promiscuous_disable  = eth_igb_promiscuous_disable,
 215         .allmulticast_enable  = eth_igb_allmulticast_enable,
 216         .allmulticast_disable = eth_igb_allmulticast_disable,
 217         .link_update          = eth_igb_link_update,
 218         .stats_get            = eth_igb_stats_get,
 219         .stats_reset          = eth_igb_stats_reset,
 220         .dev_infos_get        = eth_igb_infos_get,
 221         .vlan_filter_set      = eth_igb_vlan_filter_set,
 222         .vlan_tpid_set        = eth_igb_vlan_tpid_set,
 223         .vlan_offload_set     = eth_igb_vlan_offload_set,
 224         .rx_queue_setup       = eth_igb_rx_queue_setup,
 225         .rx_queue_release     = eth_igb_rx_queue_release,
 226         .rx_queue_count       = eth_igb_rx_queue_count,
 227         .rx_descriptor_done   = eth_igb_rx_descriptor_done,
 228         .tx_queue_setup       = eth_igb_tx_queue_setup,
 229         .tx_queue_release     = eth_igb_tx_queue_release,
 230         .dev_led_on           = eth_igb_led_on,
 231         .dev_led_off          = eth_igb_led_off,
 232         .flow_ctrl_get        = eth_igb_flow_ctrl_get,
 233         .flow_ctrl_set        = eth_igb_flow_ctrl_set,
 234         .mac_addr_add         = eth_igb_rar_set,
 235         .mac_addr_remove      = eth_igb_rar_clear,
 236         .reta_update          = eth_igb_rss_reta_update,
 237         .reta_query           = eth_igb_rss_reta_query,
 238         .rss_hash_update      = eth_igb_rss_hash_update,
 239         .rss_hash_conf_get    = eth_igb_rss_hash_conf_get,
 240         .add_syn_filter          = eth_igb_add_syn_filter,
 241         .remove_syn_filter       = eth_igb_remove_syn_filter,
 242         .get_syn_filter          = eth_igb_get_syn_filter,
 243         .add_ethertype_filter    = eth_igb_add_ethertype_filter,
 244         .remove_ethertype_filter = eth_igb_remove_ethertype_filter,
 245         .get_ethertype_filter    = eth_igb_get_ethertype_filter,
 246         .add_2tuple_filter       = eth_igb_add_2tuple_filter,
 247         .remove_2tuple_filter    = eth_igb_remove_2tuple_filter,
 248         .get_2tuple_filter       = eth_igb_get_2tuple_filter,
 249         .add_flex_filter         = eth_igb_add_flex_filter,
 250         .remove_flex_filter      = eth_igb_remove_flex_filter,
 251         .get_flex_filter         = eth_igb_get_flex_filter,
 252         .add_5tuple_filter       = eth_igb_add_5tuple_filter,
 253         .remove_5tuple_filter    = eth_igb_remove_5tuple_filter,
 254         .get_5tuple_filter       = eth_igb_get_5tuple_filter,
 255 };
 256
 257 /*
 258  * dev_ops for virtual function, bare necessities for basic vf
 259  * operation have been implemented
 260  */
 261 static struct eth_dev_ops igbvf_eth_dev_ops = {
 262         .dev_configure        = igbvf_dev_configure,
 263         .dev_start            = igbvf_dev_start,
 264         .dev_stop             = igbvf_dev_stop,
 265         .dev_close            = igbvf_dev_close,
 266         .link_update          = eth_igb_link_update,
 267         .stats_get            = eth_igbvf_stats_get,
 268         .stats_reset          = eth_igbvf_stats_reset,
 269         .vlan_filter_set      = igbvf_vlan_filter_set,
 270         .dev_infos_get        = eth_igb_infos_get,
 271         .rx_queue_setup       = eth_igb_rx_queue_setup,
 272         .rx_queue_release     = eth_igb_rx_queue_release,
 273         .tx_queue_setup       = eth_igb_tx_queue_setup,
 274         .tx_queue_release     = eth_igb_tx_queue_release,
 275 };
 276
 277 /**
 278  * Atomically reads the link status information from global
 279  * structure rte_eth_dev.
 280  *
 281  * @param dev
 282  *   - Pointer to the structure rte_eth_dev to read from.
 283  *   - Pointer to the buffer to be saved with the link status.
 284  *
 285  * @return
 286  *   - On success, zero.
 287  *   - On failure, negative value.
 288  */
 289 static inline int
 290 rte_igb_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 291                                 struct rte_eth_link *link)
 292 {
 293         struct rte_eth_link *dst = link;
 294         struct rte_eth_link *src = &(dev->data->dev_link);
 295
 296         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 297                                         *(uint64_t *)src) == 0)
 298                 return -1;
 299
 300         return 0;
 301 }
 302
 303 /**
 304  * Atomically writes the link status information into global
 305  * structure rte_eth_dev.
 306  *
 307  * @param dev
 308  *   - Pointer to the structure rte_eth_dev to read from.
 309  *   - Pointer to the buffer to be saved with the link status.
 310  *
 311  * @return
 312  *   - On success, zero.
 313  *   - On failure, negative value.
 314  */
 315 static inline int
 316 rte_igb_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 317                                 struct rte_eth_link *link)
 318 {
 319         struct rte_eth_link *dst = &(dev->data->dev_link);
 320         struct rte_eth_link *src = link;
 321
 322         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 323                                         *(uint64_t *)src) == 0)
 324                 return -1;
 325
 326         return 0;
 327 }
 328
 329 static inline void
 330 igb_intr_enable(struct rte_eth_dev *dev)
 331 {
 332         struct e1000_interrupt *intr =
 333                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 334         struct e1000_hw *hw =
 335                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 336
 337         E1000_WRITE_REG(hw, E1000_IMS, intr->mask);
 338         E1000_WRITE_FLUSH(hw);
 339 }
 340
 341 static void
 342 igb_intr_disable(struct e1000_hw *hw)
 343 {
 344         E1000_WRITE_REG(hw, E1000_IMC, ~0);
 345         E1000_WRITE_FLUSH(hw);
 346 }
 347
 348 static inline int32_t
 349 igb_pf_reset_hw(struct e1000_hw *hw)
 350 {
 351         uint32_t ctrl_ext;
 352         int32_t status;
 353
 354         status = e1000_reset_hw(hw);
 355
 356         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 357         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 358         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 359         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 360         E1000_WRITE_FLUSH(hw);
 361
 362         return status;
 363 }
 364
 365 static void
 366 igb_identify_hardware(struct rte_eth_dev *dev)
 367 {
 368         struct e1000_hw *hw =
 369                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 370
 371         hw->vendor_id = dev->pci_dev->id.vendor_id;
 372         hw->device_id = dev->pci_dev->id.device_id;
 373         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
 374         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
 375
 376         e1000_set_mac_type(hw);
 377
 378         /* need to check if it is a vf device below */
 379 }
 380
 381 static int
 382 igb_reset_swfw_lock(struct e1000_hw *hw)
 383 {
 384         int ret_val;
 385
 386         /*
 387          * Do mac ops initialization manually here, since we will need
 388          * some function pointers set by this call.
 389          */
 390         ret_val = e1000_init_mac_params(hw);
 391         if (ret_val)
 392                 return ret_val;
 393
 394         /*
 395          * SMBI lock should not fail in this early stage. If this is the case,
 396          * it is due to an improper exit of the application.
 397          * So force the release of the faulty lock.
 398          */
 399         if (e1000_get_hw_semaphore_generic(hw) < 0) {
 400                 DEBUGOUT("SMBI lock released");
 401         }
 402         e1000_put_hw_semaphore_generic(hw);
 403
 404         if (hw->mac.ops.acquire_swfw_sync != NULL) {
 405                 uint16_t mask;
 406
 407                 /*
 408                  * Phy lock should not fail in this early stage. If this is the case,
 409                  * it is due to an improper exit of the application.
 410                  * So force the release of the faulty lock.
 411                  */
 412                 mask = E1000_SWFW_PHY0_SM << hw->bus.func;
 413                 if (hw->bus.func > E1000_FUNC_1)
 414                         mask <<= 2;
 415                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 416                         DEBUGOUT1("SWFW phy%d lock released", hw->bus.func);
 417                 }
 418                 hw->mac.ops.release_swfw_sync(hw, mask);
 419
 420                 /*
 421                  * This one is more tricky since it is common to all ports; but
 422                  * swfw_sync retries last long enough (1s) to be almost sure that if
 423                  * lock can not be taken it is due to an improper lock of the
 424                  * semaphore.
 425                  */
 426                 mask = E1000_SWFW_EEP_SM;
 427                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 428                         DEBUGOUT("SWFW common locks released");
 429                 }
 430                 hw->mac.ops.release_swfw_sync(hw, mask);
 431         }
 432
 433         return E1000_SUCCESS;
 434 }
 435
 436 static int
 437 eth_igb_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 438                    struct rte_eth_dev *eth_dev)
 439 {
 440         int error = 0;
 441         struct rte_pci_device *pci_dev;
 442         struct e1000_hw *hw =
 443                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 444         struct e1000_vfta * shadow_vfta =
 445                         E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 446         uint32_t ctrl_ext;
 447
 448         pci_dev = eth_dev->pci_dev;
 449         eth_dev->dev_ops = &eth_igb_ops;
 450         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 451         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 452
 453         /* for secondary processes, we don't initialise any further as primary
 454          * has already done this work. Only check we don't need a different
 455          * RX function */
 456         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 457                 if (eth_dev->data->scattered_rx)
 458                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 459                 return 0;
 460         }
 461
 462         hw->hw_addr= (void *)pci_dev->mem_resource[0].addr;
 463
 464         igb_identify_hardware(eth_dev);
 465         if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) {
 466                 error = -EIO;
 467                 goto err_late;
 468         }
 469
 470         e1000_get_bus_info(hw);
 471
 472         /* Reset any pending lock */
 473         if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) {
 474                 error = -EIO;
 475                 goto err_late;
 476         }
 477
 478         /* Finish initialization */
 479         if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) {
 480                 error = -EIO;
 481                 goto err_late;
 482         }
 483
 484         hw->mac.autoneg = 1;
 485         hw->phy.autoneg_wait_to_complete = 0;
 486         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 487
 488         /* Copper options */
 489         if (hw->phy.media_type == e1000_media_type_copper) {
 490                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
 491                 hw->phy.disable_polarity_correction = 0;
 492                 hw->phy.ms_type = e1000_ms_hw_default;
 493         }
 494
 495         /*
 496          * Start from a known state, this is important in reading the nvm
 497          * and mac from that.
 498          */
 499         igb_pf_reset_hw(hw);
 500
 501         /* Make sure we have a good EEPROM before we read from it */
 502         if (e1000_validate_nvm_checksum(hw) < 0) {
 503                 /*
 504                  * Some PCI-E parts fail the first check due to
 505                  * the link being in sleep state, call it again,
 506                  * if it fails a second time its a real issue.
 507                  */
 508                 if (e1000_validate_nvm_checksum(hw) < 0) {
 509                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
 510                         error = -EIO;
 511                         goto err_late;
 512                 }
 513         }
 514
 515         /* Read the permanent MAC address out of the EEPROM */
 516         if (e1000_read_mac_addr(hw) != 0) {
 517                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
 518                 error = -EIO;
 519                 goto err_late;
 520         }
 521
 522         /* Allocate memory for storing MAC addresses */
 523         eth_dev->data->mac_addrs = rte_zmalloc("e1000",
 524                 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
 525         if (eth_dev->data->mac_addrs == NULL) {
 526                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to "
 527                                                 "store MAC addresses",
 528                                 ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 529                 error = -ENOMEM;
 530                 goto err_late;
 531         }
 532
 533         /* Copy the permanent MAC address */
 534         ether_addr_copy((struct ether_addr *)hw->mac.addr, &eth_dev->data->mac_addrs[0]);
 535
 536         /* initialize the vfta */
 537         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 538
 539         /* Now initialize the hardware */
 540         if (igb_hardware_init(hw) != 0) {
 541                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
 542                 rte_free(eth_dev->data->mac_addrs);
 543                 eth_dev->data->mac_addrs = NULL;
 544                 error = -ENODEV;
 545                 goto err_late;
 546         }
 547         hw->mac.get_link_status = 1;
 548
 549         /* Indicate SOL/IDER usage */
 550         if (e1000_check_reset_block(hw) < 0) {
 551                 PMD_INIT_LOG(ERR, "PHY reset is blocked due to"
 552                                         "SOL/IDER session");
 553         }
 554
 555         /* initialize PF if max_vfs not zero */
 556         igb_pf_host_init(eth_dev);
 557
 558         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 559         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 560         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 561         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 562         E1000_WRITE_FLUSH(hw);
 563
 564         PMD_INIT_LOG(INFO, "port_id %d vendorID=0x%x deviceID=0x%x\n",
 565                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 566                      pci_dev->id.device_id);
 567
 568         rte_intr_callback_register(&(pci_dev->intr_handle),
 569                 eth_igb_interrupt_handler, (void *)eth_dev);
 570
 571         /* enable uio intr after callback register */
 572         rte_intr_enable(&(pci_dev->intr_handle));
 573
 574         /* enable support intr */
 575         igb_intr_enable(eth_dev);
 576
 577         return 0;
 578
 579 err_late:
 580         igb_hw_control_release(hw);
 581
 582         return (error);
 583 }
 584
 585 /*
 586  * Virtual Function device init
 587  */
 588 static int
 589 eth_igbvf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 590                 struct rte_eth_dev *eth_dev)
 591 {
 592         struct rte_pci_device *pci_dev;
 593         struct e1000_hw *hw =
 594                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 595         int diag;
 596
 597         PMD_INIT_LOG(DEBUG, "eth_igbvf_dev_init");
 598
 599         eth_dev->dev_ops = &igbvf_eth_dev_ops;
 600         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 601         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 602
 603         /* for secondary processes, we don't initialise any further as primary
 604          * has already done this work. Only check we don't need a different
 605          * RX function */
 606         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 607                 if (eth_dev->data->scattered_rx)
 608                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 609                 return 0;
 610         }
 611
 612         pci_dev = eth_dev->pci_dev;
 613
 614         hw->device_id = pci_dev->id.device_id;
 615         hw->vendor_id = pci_dev->id.vendor_id;
 616         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 617
 618         /* Initialize the shared code */
 619         diag = e1000_setup_init_funcs(hw, TRUE);
 620         if (diag != 0) {
 621                 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d",
 622                         diag);
 623                 return -EIO;
 624         }
 625
 626         /* init_mailbox_params */
 627         hw->mbx.ops.init_params(hw);
 628
 629         /* Disable the interrupts for VF */
 630         igbvf_intr_disable(hw);
 631
 632         diag = hw->mac.ops.reset_hw(hw);
 633
 634         /* Allocate memory for storing MAC addresses */
 635         eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN *
 636                 hw->mac.rar_entry_count, 0);
 637         if (eth_dev->data->mac_addrs == NULL) {
 638                 PMD_INIT_LOG(ERR,
 639                         "Failed to allocate %d bytes needed to store MAC "
 640                         "addresses",
 641                         ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 642                 return -ENOMEM;
 643         }
 644
 645         /* Copy the permanent MAC address */
 646         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 647                         &eth_dev->data->mac_addrs[0]);
 648
 649         PMD_INIT_LOG(DEBUG, "\nport %d vendorID=0x%x deviceID=0x%x "
 650                         "mac.type=%s\n",
 651                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 652                         pci_dev->id.device_id,
 653                         "igb_mac_82576_vf");
 654
 655         return 0;
 656 }
 657
 658 static struct eth_driver rte_igb_pmd = {
 659         {
 660                 .name = "rte_igb_pmd",
 661                 .id_table = pci_id_igb_map,
 662                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 663         },
 664         .eth_dev_init = eth_igb_dev_init,
 665         .dev_private_size = sizeof(struct e1000_adapter),
 666 };
 667
 668 /*
 669  * virtual function driver struct
 670  */
 671 static struct eth_driver rte_igbvf_pmd = {
 672         {
 673                 .name = "rte_igbvf_pmd",
 674                 .id_table = pci_id_igbvf_map,
 675                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 676         },
 677         .eth_dev_init = eth_igbvf_dev_init,
 678         .dev_private_size = sizeof(struct e1000_adapter),
 679 };
 680
 681 static int
 682 rte_igb_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 683 {
 684         rte_eth_driver_register(&rte_igb_pmd);
 685         return 0;
 686 }
 687
 688 static void
 689 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 690 {
 691         struct e1000_hw *hw =
 692                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 693         /* RCTL: enable VLAN filter since VMDq always use VLAN filter */
 694         uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL);
 695         rctl |= E1000_RCTL_VFE;
 696         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
 697 }
 698
 699 /*
 700  * VF Driver initialization routine.
 701  * Invoked one at EAL init time.
 702  * Register itself as the [Virtual Poll Mode] Driver of PCI IGB devices.
 703  */
 704 static int
 705 rte_igbvf_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 706 {
 707         DEBUGFUNC("rte_igbvf_pmd_init");
 708
 709         rte_eth_driver_register(&rte_igbvf_pmd);
 710         return (0);
 711 }
 712
 713 static int
 714 eth_igb_configure(struct rte_eth_dev *dev)
 715 {
 716         struct e1000_interrupt *intr =
 717                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 718
 719         PMD_INIT_LOG(DEBUG, ">>");
 720
 721         intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
 722
 723         PMD_INIT_LOG(DEBUG, "<<");
 724
 725         return (0);
 726 }
 727
 728 static int
 729 eth_igb_start(struct rte_eth_dev *dev)
 730 {
 731         struct e1000_hw *hw =
 732                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 733         int ret, i, mask;
 734         uint32_t ctrl_ext;
 735
 736         PMD_INIT_LOG(DEBUG, ">>");
 737
 738         /* Power up the phy. Needed to make the link go Up */
 739         e1000_power_up_phy(hw);
 740
 741         /*
 742          * Packet Buffer Allocation (PBA)
 743          * Writing PBA sets the receive portion of the buffer
 744          * the remainder is used for the transmit buffer.
 745          */
 746         if (hw->mac.type == e1000_82575) {
 747                 uint32_t pba;
 748
 749                 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
 750                 E1000_WRITE_REG(hw, E1000_PBA, pba);
 751         }
 752
 753         /* Put the address into the Receive Address Array */
 754         e1000_rar_set(hw, hw->mac.addr, 0);
 755
 756         /* Initialize the hardware */
 757         if (igb_hardware_init(hw)) {
 758                 PMD_INIT_LOG(ERR, "Unable to initialize the hardware");
 759                 return (-EIO);
 760         }
 761
 762         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
 763
 764         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 765         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 766         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 767         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 768         E1000_WRITE_FLUSH(hw);
 769
 770         /* configure PF module if SRIOV enabled */
 771         igb_pf_host_configure(dev);
 772
 773         /* Configure for OS presence */
 774         igb_init_manageability(hw);
 775
 776         eth_igb_tx_init(dev);
 777
 778         /* This can fail when allocating mbufs for descriptor rings */
 779         ret = eth_igb_rx_init(dev);
 780         if (ret) {
 781                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
 782                 igb_dev_clear_queues(dev);
 783                 return ret;
 784         }
 785
 786         e1000_clear_hw_cntrs_base_generic(hw);
 787
 788         /*
 789          * VLAN Offload Settings
 790          */
 791         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
 792                         ETH_VLAN_EXTEND_MASK;
 793         eth_igb_vlan_offload_set(dev, mask);
 794
 795         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
 796                 /* Enable VLAN filter since VMDq always use VLAN filter */
 797                 igb_vmdq_vlan_hw_filter_enable(dev);
 798         }
 799
 800         /*
 801          * Configure the Interrupt Moderation register (EITR) with the maximum
 802          * possible value (0xFFFF) to minimize "System Partial Write" issued by
 803          * spurious [DMA] memory updates of RX and TX ring descriptors.
 804          *
 805          * With a EITR granularity of 2 microseconds in the 82576, only 7/8
 806          * spurious memory updates per second should be expected.
 807          * ((65535 * 2) / 1000.1000 ~= 0.131 second).
 808          *
 809          * Because interrupts are not used at all, the MSI-X is not activated
 810          * and interrupt moderation is controlled by EITR[0].
 811          *
 812          * Note that having [almost] disabled memory updates of RX and TX ring
 813          * descriptors through the Interrupt Moderation mechanism, memory
 814          * updates of ring descriptors are now moderated by the configurable
 815          * value of Write-Back Threshold registers.
 816          */
 817         if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) ||
 818                 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210)) {
 819                 uint32_t ivar;
 820
 821                 /* Enable all RX & TX queues in the IVAR registers */
 822                 ivar = (uint32_t) ((E1000_IVAR_VALID << 16) | E1000_IVAR_VALID);
 823                 for (i = 0; i < 8; i++)
 824                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, i, ivar);
 825
 826                 /* Configure EITR with the maximum possible value (0xFFFF) */
 827                 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF);
 828         }
 829
 830         /* Setup link speed and duplex */
 831         switch (dev->data->dev_conf.link_speed) {
 832         case ETH_LINK_SPEED_AUTONEG:
 833                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
 834                         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 835                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
 836                         hw->phy.autoneg_advertised = E1000_ALL_HALF_DUPLEX;
 837                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
 838                         hw->phy.autoneg_advertised = E1000_ALL_FULL_DUPLEX;
 839                 else
 840                         goto error_invalid_config;
 841                 break;
 842         case ETH_LINK_SPEED_10:
 843                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
 844                         hw->phy.autoneg_advertised = E1000_ALL_10_SPEED;
 845                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
 846                         hw->phy.autoneg_advertised = ADVERTISE_10_HALF;
 847                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
 848                         hw->phy.autoneg_advertised = ADVERTISE_10_FULL;
 849                 else
 850                         goto error_invalid_config;
 851                 break;
 852         case ETH_LINK_SPEED_100:
 853                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
 854                         hw->phy.autoneg_advertised = E1000_ALL_100_SPEED;
 855                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
 856                         hw->phy.autoneg_advertised = ADVERTISE_100_HALF;
 857                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
 858                         hw->phy.autoneg_advertised = ADVERTISE_100_FULL;
 859                 else
 860                         goto error_invalid_config;
 861                 break;
 862         case ETH_LINK_SPEED_1000:
 863                 if ((dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX) ||
 864                                 (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX))
 865                         hw->phy.autoneg_advertised = ADVERTISE_1000_FULL;
 866                 else
 867                         goto error_invalid_config;
 868                 break;
 869         case ETH_LINK_SPEED_10000:
 870         default:
 871                 goto error_invalid_config;
 872         }
 873         e1000_setup_link(hw);
 874
 875         /* check if lsc interrupt feature is enabled */
 876         if (dev->data->dev_conf.intr_conf.lsc != 0)
 877                 ret = eth_igb_lsc_interrupt_setup(dev);
 878
 879         /* resume enabled intr since hw reset */
 880         igb_intr_enable(dev);
 881
 882         PMD_INIT_LOG(DEBUG, "<<");
 883
 884         return (0);
 885
 886 error_invalid_config:
 887         PMD_INIT_LOG(ERR, "Invalid link_speed/link_duplex (%u/%u) for port %u\n",
 888                         dev->data->dev_conf.link_speed,
 889                         dev->data->dev_conf.link_duplex, dev->data->port_id);
 890         igb_dev_clear_queues(dev);
 891         return (-EINVAL);
 892 }
 893
 894 /*********************************************************************
 895  *
 896  *  This routine disables all traffic on the adapter by issuing a
 897  *  global reset on the MAC.
 898  *
 899  **********************************************************************/
 900 static void
 901 eth_igb_stop(struct rte_eth_dev *dev)
 902 {
 903         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 904         struct rte_eth_link link;
 905
 906         igb_intr_disable(hw);
 907         igb_pf_reset_hw(hw);
 908         E1000_WRITE_REG(hw, E1000_WUC, 0);
 909
 910         /* Set bit for Go Link disconnect */
 911         if (hw->mac.type >= e1000_82580) {
 912                 uint32_t phpm_reg;
 913
 914                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
 915                 phpm_reg |= E1000_82580_PM_GO_LINKD;
 916                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
 917         }
 918
 919         /* Power down the phy. Needed to make the link go Down */
 920         e1000_power_down_phy(hw);
 921
 922         igb_dev_clear_queues(dev);
 923
 924         /* clear the recorded link status */
 925         memset(&link, 0, sizeof(link));
 926         rte_igb_dev_atomic_write_link_status(dev, &link);
 927 }
 928
 929 static void
 930 eth_igb_close(struct rte_eth_dev *dev)
 931 {
 932         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 933         struct rte_eth_link link;
 934
 935         eth_igb_stop(dev);
 936         e1000_phy_hw_reset(hw);
 937         igb_release_manageability(hw);
 938         igb_hw_control_release(hw);
 939
 940         /* Clear bit for Go Link disconnect */
 941         if (hw->mac.type >= e1000_82580) {
 942                 uint32_t phpm_reg;
 943
 944                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
 945                 phpm_reg &= ~E1000_82580_PM_GO_LINKD;
 946                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
 947         }
 948
 949         igb_dev_clear_queues(dev);
 950
 951         memset(&link, 0, sizeof(link));
 952         rte_igb_dev_atomic_write_link_status(dev, &link);
 953 }
 954
 955 static int
 956 igb_get_rx_buffer_size(struct e1000_hw *hw)
 957 {
 958         uint32_t rx_buf_size;
 959         if (hw->mac.type == e1000_82576) {
 960                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10;
 961         } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) {
 962                 /* PBS needs to be translated according to a lookup table */
 963                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf);
 964                 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size);
 965                 rx_buf_size = (rx_buf_size << 10);
 966         } else if (hw->mac.type == e1000_i210) {
 967                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10;
 968         } else {
 969                 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10;
 970         }
 971
 972         return rx_buf_size;
 973 }
 974
 975 /*********************************************************************
 976  *
 977  *  Initialize the hardware
 978  *
 979  **********************************************************************/
 980 static int
 981 igb_hardware_init(struct e1000_hw *hw)
 982 {
 983         uint32_t rx_buf_size;
 984         int diag;
 985
 986         /* Let the firmware know the OS is in control */
 987         igb_hw_control_acquire(hw);
 988
 989         /*
 990          * These parameters control the automatic generation (Tx) and
 991          * response (Rx) to Ethernet PAUSE frames.
 992          * - High water mark should allow for at least two standard size (1518)
 993          *   frames to be received after sending an XOFF.
 994          * - Low water mark works best when it is very near the high water mark.
 995          *   This allows the receiver to restart by sending XON when it has
 996          *   drained a bit. Here we use an arbitrary value of 1500 which will
 997          *   restart after one full frame is pulled from the buffer. There
 998          *   could be several smaller frames in the buffer and if so they will
 999          *   not trigger the XON until their total number reduces the buffer
1000          *   by 1500.
1001          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
1002          */
1003         rx_buf_size = igb_get_rx_buffer_size(hw);
1004
1005         hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2);
1006         hw->fc.low_water = hw->fc.high_water - 1500;
1007         hw->fc.pause_time = IGB_FC_PAUSE_TIME;
1008         hw->fc.send_xon = 1;
1009
1010         /* Set Flow control, use the tunable location if sane */
1011         if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4))
1012                 hw->fc.requested_mode = igb_fc_setting;
1013         else
1014                 hw->fc.requested_mode = e1000_fc_none;
1015
1016         /* Issue a global reset */
1017         igb_pf_reset_hw(hw);
1018         E1000_WRITE_REG(hw, E1000_WUC, 0);
1019
1020         diag = e1000_init_hw(hw);
1021         if (diag < 0)
1022                 return (diag);
1023
1024         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
1025         e1000_get_phy_info(hw);
1026         e1000_check_for_link(hw);
1027
1028         return (0);
1029 }
1030
1031 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */
1032 static void
1033 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1034 {
1035         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1036         struct e1000_hw_stats *stats =
1037                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1038         int pause_frames;
1039
1040         if(hw->phy.media_type == e1000_media_type_copper ||
1041             (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
1042                 stats->symerrs +=
1043                     E1000_READ_REG(hw,E1000_SYMERRS);
1044                 stats->sec += E1000_READ_REG(hw, E1000_SEC);
1045         }
1046
1047         stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
1048         stats->mpc += E1000_READ_REG(hw, E1000_MPC);
1049         stats->scc += E1000_READ_REG(hw, E1000_SCC);
1050         stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
1051
1052         stats->mcc += E1000_READ_REG(hw, E1000_MCC);
1053         stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
1054         stats->colc += E1000_READ_REG(hw, E1000_COLC);
1055         stats->dc += E1000_READ_REG(hw, E1000_DC);
1056         stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
1057         stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
1058         stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
1059         /*
1060         ** For watchdog management we need to know if we have been
1061         ** paused during the last interval, so capture that here.
1062         */
1063         pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC);
1064         stats->xoffrxc += pause_frames;
1065         stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
1066         stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
1067         stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
1068         stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
1069         stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
1070         stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
1071         stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
1072         stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
1073         stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
1074         stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
1075         stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
1076         stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
1077
1078         /* For the 64-bit byte counters the low dword must be read first. */
1079         /* Both registers clear on the read of the high dword */
1080
1081         stats->gorc += E1000_READ_REG(hw, E1000_GORCL);
1082         stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32);
1083         stats->gotc += E1000_READ_REG(hw, E1000_GOTCL);
1084         stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32);
1085
1086         stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
1087         stats->ruc += E1000_READ_REG(hw, E1000_RUC);
1088         stats->rfc += E1000_READ_REG(hw, E1000_RFC);
1089         stats->roc += E1000_READ_REG(hw, E1000_ROC);
1090         stats->rjc += E1000_READ_REG(hw, E1000_RJC);
1091
1092         stats->tor += E1000_READ_REG(hw, E1000_TORH);
1093         stats->tot += E1000_READ_REG(hw, E1000_TOTH);
1094
1095         stats->tpr += E1000_READ_REG(hw, E1000_TPR);
1096         stats->tpt += E1000_READ_REG(hw, E1000_TPT);
1097         stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
1098         stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
1099         stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
1100         stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
1101         stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
1102         stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
1103         stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
1104         stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
1105
1106         /* Interrupt Counts */
1107
1108         stats->iac += E1000_READ_REG(hw, E1000_IAC);
1109         stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
1110         stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
1111         stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
1112         stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
1113         stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
1114         stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
1115         stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
1116         stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
1117
1118         /* Host to Card Statistics */
1119
1120         stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
1121         stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
1122         stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
1123         stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
1124         stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
1125         stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
1126         stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
1127         stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL);
1128         stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32);
1129         stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL);
1130         stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32);
1131         stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
1132         stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
1133         stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
1134
1135         stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
1136         stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
1137         stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
1138         stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
1139         stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
1140         stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
1141
1142         if (rte_stats == NULL)
1143                 return;
1144
1145         /* Rx Errors */
1146         rte_stats->ibadcrc = stats->crcerrs;
1147         rte_stats->ibadlen = stats->rlec + stats->ruc + stats->roc;
1148         rte_stats->imissed = stats->mpc;
1149         rte_stats->ierrors = rte_stats->ibadcrc +
1150                              rte_stats->ibadlen +
1151                              rte_stats->imissed +
1152                              stats->rxerrc + stats->algnerrc + stats->cexterr;
1153
1154         /* Tx Errors */
1155         rte_stats->oerrors = stats->ecol + stats->latecol;
1156
1157         /* XON/XOFF pause frames */
1158         rte_stats->tx_pause_xon  = stats->xontxc;
1159         rte_stats->rx_pause_xon  = stats->xonrxc;
1160         rte_stats->tx_pause_xoff = stats->xofftxc;
1161         rte_stats->rx_pause_xoff = stats->xoffrxc;
1162
1163         rte_stats->ipackets = stats->gprc;
1164         rte_stats->opackets = stats->gptc;
1165         rte_stats->ibytes   = stats->gorc;
1166         rte_stats->obytes   = stats->gotc;
1167 }
1168
1169 static void
1170 eth_igb_stats_reset(struct rte_eth_dev *dev)
1171 {
1172         struct e1000_hw_stats *hw_stats =
1173                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1174
1175         /* HW registers are cleared on read */
1176         eth_igb_stats_get(dev, NULL);
1177
1178         /* Reset software totals */
1179         memset(hw_stats, 0, sizeof(*hw_stats));
1180 }
1181
1182 static void
1183 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1184 {
1185         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1186         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1187                           E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1188
1189         /* Good Rx packets, include VF loopback */
1190         UPDATE_VF_STAT(E1000_VFGPRC,
1191             hw_stats->last_gprc, hw_stats->gprc);
1192
1193         /* Good Rx octets, include VF loopback */
1194         UPDATE_VF_STAT(E1000_VFGORC,
1195             hw_stats->last_gorc, hw_stats->gorc);
1196
1197         /* Good Tx packets, include VF loopback */
1198         UPDATE_VF_STAT(E1000_VFGPTC,
1199             hw_stats->last_gptc, hw_stats->gptc);
1200
1201         /* Good Tx octets, include VF loopback */
1202         UPDATE_VF_STAT(E1000_VFGOTC,
1203             hw_stats->last_gotc, hw_stats->gotc);
1204
1205         /* Rx Multicst packets */
1206         UPDATE_VF_STAT(E1000_VFMPRC,
1207             hw_stats->last_mprc, hw_stats->mprc);
1208
1209         /* Good Rx loopback packets */
1210         UPDATE_VF_STAT(E1000_VFGPRLBC,
1211             hw_stats->last_gprlbc, hw_stats->gprlbc);
1212
1213         /* Good Rx loopback octets */
1214         UPDATE_VF_STAT(E1000_VFGORLBC,
1215             hw_stats->last_gorlbc, hw_stats->gorlbc);
1216
1217         /* Good Tx loopback packets */
1218         UPDATE_VF_STAT(E1000_VFGPTLBC,
1219             hw_stats->last_gptlbc, hw_stats->gptlbc);
1220
1221         /* Good Tx loopback octets */
1222         UPDATE_VF_STAT(E1000_VFGOTLBC,
1223             hw_stats->last_gotlbc, hw_stats->gotlbc);
1224
1225         if (rte_stats == NULL)
1226                 return;
1227
1228         memset(rte_stats, 0, sizeof(*rte_stats));
1229         rte_stats->ipackets = hw_stats->gprc;
1230         rte_stats->ibytes = hw_stats->gorc;
1231         rte_stats->opackets = hw_stats->gptc;
1232         rte_stats->obytes = hw_stats->gotc;
1233         rte_stats->imcasts = hw_stats->mprc;
1234         rte_stats->ilbpackets = hw_stats->gprlbc;
1235         rte_stats->ilbbytes = hw_stats->gorlbc;
1236         rte_stats->olbpackets = hw_stats->gptlbc;
1237         rte_stats->olbbytes = hw_stats->gotlbc;
1238
1239 }
1240
1241 static void
1242 eth_igbvf_stats_reset(struct rte_eth_dev *dev)
1243 {
1244         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1245                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1246
1247         /* Sync HW register to the last stats */
1248         eth_igbvf_stats_get(dev, NULL);
1249
1250         /* reset HW current stats*/
1251         memset(&hw_stats->gprc, 0, sizeof(*hw_stats) -
1252                offsetof(struct e1000_vf_stats, gprc));
1253
1254 }
1255
1256 static void
1257 eth_igb_infos_get(struct rte_eth_dev *dev,
1258                     struct rte_eth_dev_info *dev_info)
1259 {
1260         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1261
1262         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1263         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1264         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1265         dev_info->rx_offload_capa =
1266                 DEV_RX_OFFLOAD_VLAN_STRIP |
1267                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1268                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1269                 DEV_RX_OFFLOAD_TCP_CKSUM;
1270         dev_info->tx_offload_capa =
1271                 DEV_TX_OFFLOAD_VLAN_INSERT |
1272                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1273                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1274                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1275                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1276
1277         switch (hw->mac.type) {
1278         case e1000_82575:
1279                 dev_info->max_rx_queues = 4;
1280                 dev_info->max_tx_queues = 4;
1281                 dev_info->max_vmdq_pools = 0;
1282                 break;
1283
1284         case e1000_82576:
1285                 dev_info->max_rx_queues = 16;
1286                 dev_info->max_tx_queues = 16;
1287                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1288                 break;
1289
1290         case e1000_82580:
1291                 dev_info->max_rx_queues = 8;
1292                 dev_info->max_tx_queues = 8;
1293                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1294                 break;
1295
1296         case e1000_i350:
1297                 dev_info->max_rx_queues = 8;
1298                 dev_info->max_tx_queues = 8;
1299                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1300                 break;
1301
1302         case e1000_i354:
1303                 dev_info->max_rx_queues = 8;
1304                 dev_info->max_tx_queues = 8;
1305                 break;
1306
1307         case e1000_i210:
1308                 dev_info->max_rx_queues = 4;
1309                 dev_info->max_tx_queues = 4;
1310                 dev_info->max_vmdq_pools = 0;
1311                 break;
1312
1313         case e1000_vfadapt:
1314                 dev_info->max_rx_queues = 2;
1315                 dev_info->max_tx_queues = 2;
1316                 dev_info->max_vmdq_pools = 0;
1317                 break;
1318
1319         case e1000_vfadapt_i350:
1320                 dev_info->max_rx_queues = 1;
1321                 dev_info->max_tx_queues = 1;
1322                 dev_info->max_vmdq_pools = 0;
1323                 break;
1324
1325         default:
1326                 /* Should not happen */
1327                 dev_info->max_rx_queues = 0;
1328                 dev_info->max_tx_queues = 0;
1329                 dev_info->max_vmdq_pools = 0;
1330         }
1331 }
1332
1333 /* return 0 means link status changed, -1 means not changed */
1334 static int
1335 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1336 {
1337         struct e1000_hw *hw =
1338                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1339         struct rte_eth_link link, old;
1340         int link_check, count;
1341
1342         link_check = 0;
1343         hw->mac.get_link_status = 1;
1344
1345         /* possible wait-to-complete in up to 9 seconds */
1346         for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) {
1347                 /* Read the real link status */
1348                 switch (hw->phy.media_type) {
1349                 case e1000_media_type_copper:
1350                         /* Do the work to read phy */
1351                         e1000_check_for_link(hw);
1352                         link_check = !hw->mac.get_link_status;
1353                         break;
1354
1355                 case e1000_media_type_fiber:
1356                         e1000_check_for_link(hw);
1357                         link_check = (E1000_READ_REG(hw, E1000_STATUS) &
1358                                       E1000_STATUS_LU);
1359                         break;
1360
1361                 case e1000_media_type_internal_serdes:
1362                         e1000_check_for_link(hw);
1363                         link_check = hw->mac.serdes_has_link;
1364                         break;
1365
1366                 /* VF device is type_unknown */
1367                 case e1000_media_type_unknown:
1368                         eth_igbvf_link_update(hw);
1369                         link_check = !hw->mac.get_link_status;
1370                         break;
1371
1372                 default:
1373                         break;
1374                 }
1375                 if (link_check || wait_to_complete == 0)
1376                         break;
1377                 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL);
1378         }
1379         memset(&link, 0, sizeof(link));
1380         rte_igb_dev_atomic_read_link_status(dev, &link);
1381         old = link;
1382
1383         /* Now we check if a transition has happened */
1384         if (link_check) {
1385                 hw->mac.ops.get_link_up_info(hw, &link.link_speed,
1386                                           &link.link_duplex);
1387                 link.link_status = 1;
1388         } else if (!link_check) {
1389                 link.link_speed = 0;
1390                 link.link_duplex = 0;
1391                 link.link_status = 0;
1392         }
1393         rte_igb_dev_atomic_write_link_status(dev, &link);
1394
1395         /* not changed */
1396         if (old.link_status == link.link_status)
1397                 return -1;
1398
1399         /* changed */
1400         return 0;
1401 }
1402
1403 /*
1404  * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
1405  * For ASF and Pass Through versions of f/w this means
1406  * that the driver is loaded.
1407  */
1408 static void
1409 igb_hw_control_acquire(struct e1000_hw *hw)
1410 {
1411         uint32_t ctrl_ext;
1412
1413         /* Let firmware know the driver has taken over */
1414         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1415         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1416 }
1417
1418 /*
1419  * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
1420  * For ASF and Pass Through versions of f/w this means that the
1421  * driver is no longer loaded.
1422  */
1423 static void
1424 igb_hw_control_release(struct e1000_hw *hw)
1425 {
1426         uint32_t ctrl_ext;
1427
1428         /* Let firmware taken over control of h/w */
1429         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1430         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1431                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1432 }
1433
1434 /*
1435  * Bit of a misnomer, what this really means is
1436  * to enable OS management of the system... aka
1437  * to disable special hardware management features.
1438  */
1439 static void
1440 igb_init_manageability(struct e1000_hw *hw)
1441 {
1442         if (e1000_enable_mng_pass_thru(hw)) {
1443                 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H);
1444                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
1445
1446                 /* disable hardware interception of ARP */
1447                 manc &= ~(E1000_MANC_ARP_EN);
1448
1449                 /* enable receiving management packets to the host */
1450                 manc |= E1000_MANC_EN_MNG2HOST;
1451                 manc2h |= 1 << 5;  /* Mng Port 623 */
1452                 manc2h |= 1 << 6;  /* Mng Port 664 */
1453                 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h);
1454                 E1000_WRITE_REG(hw, E1000_MANC, manc);
1455         }
1456 }
1457
1458 static void
1459 igb_release_manageability(struct e1000_hw *hw)
1460 {
1461         if (e1000_enable_mng_pass_thru(hw)) {
1462                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
1463
1464                 manc |= E1000_MANC_ARP_EN;
1465                 manc &= ~E1000_MANC_EN_MNG2HOST;
1466
1467                 E1000_WRITE_REG(hw, E1000_MANC, manc);
1468         }
1469 }
1470
1471 static void
1472 eth_igb_promiscuous_enable(struct rte_eth_dev *dev)
1473 {
1474         struct e1000_hw *hw =
1475                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1476         uint32_t rctl;
1477
1478         rctl = E1000_READ_REG(hw, E1000_RCTL);
1479         rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1480         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1481 }
1482
1483 static void
1484 eth_igb_promiscuous_disable(struct rte_eth_dev *dev)
1485 {
1486         struct e1000_hw *hw =
1487                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1488         uint32_t rctl;
1489
1490         rctl = E1000_READ_REG(hw, E1000_RCTL);
1491         rctl &= (~E1000_RCTL_UPE);
1492         if (dev->data->all_multicast == 1)
1493                 rctl |= E1000_RCTL_MPE;
1494         else
1495                 rctl &= (~E1000_RCTL_MPE);
1496         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1497 }
1498
1499 static void
1500 eth_igb_allmulticast_enable(struct rte_eth_dev *dev)
1501 {
1502         struct e1000_hw *hw =
1503                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1504         uint32_t rctl;
1505
1506         rctl = E1000_READ_REG(hw, E1000_RCTL);
1507         rctl |= E1000_RCTL_MPE;
1508         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1509 }
1510
1511 static void
1512 eth_igb_allmulticast_disable(struct rte_eth_dev *dev)
1513 {
1514         struct e1000_hw *hw =
1515                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1516         uint32_t rctl;
1517
1518         if (dev->data->promiscuous == 1)
1519                 return; /* must remain in all_multicast mode */
1520         rctl = E1000_READ_REG(hw, E1000_RCTL);
1521         rctl &= (~E1000_RCTL_MPE);
1522         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1523 }
1524
1525 static int
1526 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1527 {
1528         struct e1000_hw *hw =
1529                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1530         struct e1000_vfta * shadow_vfta =
1531                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1532         uint32_t vfta;
1533         uint32_t vid_idx;
1534         uint32_t vid_bit;
1535
1536         vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) &
1537                               E1000_VFTA_ENTRY_MASK);
1538         vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK));
1539         vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx);
1540         if (on)
1541                 vfta |= vid_bit;
1542         else
1543                 vfta &= ~vid_bit;
1544         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta);
1545
1546         /* update local VFTA copy */
1547         shadow_vfta->vfta[vid_idx] = vfta;
1548
1549         return 0;
1550 }
1551
1552 static void
1553 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1554 {
1555         struct e1000_hw *hw =
1556                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1557         uint32_t reg = ETHER_TYPE_VLAN ;
1558
1559         reg |= (tpid << 16);
1560         E1000_WRITE_REG(hw, E1000_VET, reg);
1561 }
1562
1563 static void
1564 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1565 {
1566         struct e1000_hw *hw =
1567                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1568         uint32_t reg;
1569
1570         /* Filter Table Disable */
1571         reg = E1000_READ_REG(hw, E1000_RCTL);
1572         reg &= ~E1000_RCTL_CFIEN;
1573         reg &= ~E1000_RCTL_VFE;
1574         E1000_WRITE_REG(hw, E1000_RCTL, reg);
1575 }
1576
1577 static void
1578 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1579 {
1580         struct e1000_hw *hw =
1581                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1582         struct e1000_vfta * shadow_vfta =
1583                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1584         uint32_t reg;
1585         int i;
1586
1587         /* Filter Table Enable, CFI not used for packet acceptance */
1588         reg = E1000_READ_REG(hw, E1000_RCTL);
1589         reg &= ~E1000_RCTL_CFIEN;
1590         reg |= E1000_RCTL_VFE;
1591         E1000_WRITE_REG(hw, E1000_RCTL, reg);
1592
1593         /* restore VFTA table */
1594         for (i = 0; i < IGB_VFTA_SIZE; i++)
1595                 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]);
1596 }
1597
1598 static void
1599 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev)
1600 {
1601         struct e1000_hw *hw =
1602                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1603         uint32_t reg;
1604
1605         /* VLAN Mode Disable */
1606         reg = E1000_READ_REG(hw, E1000_CTRL);
1607         reg &= ~E1000_CTRL_VME;
1608         E1000_WRITE_REG(hw, E1000_CTRL, reg);
1609 }
1610
1611 static void
1612 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev)
1613 {
1614         struct e1000_hw *hw =
1615                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1616         uint32_t reg;
1617
1618         /* VLAN Mode Enable */
1619         reg = E1000_READ_REG(hw, E1000_CTRL);
1620         reg |= E1000_CTRL_VME;
1621         E1000_WRITE_REG(hw, E1000_CTRL, reg);
1622 }
1623
1624 static void
1625 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1626 {
1627         struct e1000_hw *hw =
1628                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1629         uint32_t reg;
1630
1631         /* CTRL_EXT: Extended VLAN */
1632         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
1633         reg &= ~E1000_CTRL_EXT_EXTEND_VLAN;
1634         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
1635
1636         /* Update maximum packet length */
1637         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
1638                 E1000_WRITE_REG(hw, E1000_RLPML,
1639                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
1640                                                 VLAN_TAG_SIZE);
1641 }
1642
1643 static void
1644 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1645 {
1646         struct e1000_hw *hw =
1647                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1648         uint32_t reg;
1649
1650         /* CTRL_EXT: Extended VLAN */
1651         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
1652         reg |= E1000_CTRL_EXT_EXTEND_VLAN;
1653         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
1654
1655         /* Update maximum packet length */
1656         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
1657                 E1000_WRITE_REG(hw, E1000_RLPML,
1658                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
1659                                                 2 * VLAN_TAG_SIZE);
1660 }
1661
1662 static void
1663 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1664 {
1665         if(mask & ETH_VLAN_STRIP_MASK){
1666                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1667                         igb_vlan_hw_strip_enable(dev);
1668                 else
1669                         igb_vlan_hw_strip_disable(dev);
1670         }
1671
1672         if(mask & ETH_VLAN_FILTER_MASK){
1673                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1674                         igb_vlan_hw_filter_enable(dev);
1675                 else
1676                         igb_vlan_hw_filter_disable(dev);
1677         }
1678
1679         if(mask & ETH_VLAN_EXTEND_MASK){
1680                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1681                         igb_vlan_hw_extend_enable(dev);
1682                 else
1683                         igb_vlan_hw_extend_disable(dev);
1684         }
1685 }
1686
1687
1688 /**
1689  * It enables the interrupt mask and then enable the interrupt.
1690  *
1691  * @param dev
1692  *  Pointer to struct rte_eth_dev.
1693  *
1694  * @return
1695  *  - On success, zero.
1696  *  - On failure, a negative value.
1697  */
1698 static int
1699 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev)
1700 {
1701         struct e1000_interrupt *intr =
1702                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1703
1704         intr->mask |= E1000_ICR_LSC;
1705
1706         return 0;
1707 }
1708
1709 /*
1710  * It reads ICR and gets interrupt causes, check it and set a bit flag
1711  * to update link status.
1712  *
1713  * @param dev
1714  *  Pointer to struct rte_eth_dev.
1715  *
1716  * @return
1717  *  - On success, zero.
1718  *  - On failure, a negative value.
1719  */
1720 static int
1721 eth_igb_interrupt_get_status(struct rte_eth_dev *dev)
1722 {
1723         uint32_t icr;
1724         struct e1000_hw *hw =
1725                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1726         struct e1000_interrupt *intr =
1727                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1728
1729         igb_intr_disable(hw);
1730
1731         /* read-on-clear nic registers here */
1732         icr = E1000_READ_REG(hw, E1000_ICR);
1733
1734         intr->flags = 0;
1735         if (icr & E1000_ICR_LSC) {
1736                 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
1737         }
1738
1739         if (icr & E1000_ICR_VMMB)
1740                 intr->flags |= E1000_FLAG_MAILBOX;
1741
1742         return 0;
1743 }
1744
1745 /*
1746  * It executes link_update after knowing an interrupt is prsent.
1747  *
1748  * @param dev
1749  *  Pointer to struct rte_eth_dev.
1750  *
1751  * @return
1752  *  - On success, zero.
1753  *  - On failure, a negative value.
1754  */
1755 static int
1756 eth_igb_interrupt_action(struct rte_eth_dev *dev)
1757 {
1758         struct e1000_hw *hw =
1759                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1760         struct e1000_interrupt *intr =
1761                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1762         uint32_t tctl, rctl;
1763         struct rte_eth_link link;
1764         int ret;
1765
1766         if (intr->flags & E1000_FLAG_MAILBOX) {
1767                 igb_pf_mbx_process(dev);
1768                 intr->flags &= ~E1000_FLAG_MAILBOX;
1769         }
1770
1771         igb_intr_enable(dev);
1772         rte_intr_enable(&(dev->pci_dev->intr_handle));
1773
1774         if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) {
1775                 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE;
1776
1777                 /* set get_link_status to check register later */
1778                 hw->mac.get_link_status = 1;
1779                 ret = eth_igb_link_update(dev, 0);
1780
1781                 /* check if link has changed */
1782                 if (ret < 0)
1783                         return 0;
1784
1785                 memset(&link, 0, sizeof(link));
1786                 rte_igb_dev_atomic_read_link_status(dev, &link);
1787                 if (link.link_status) {
1788                         PMD_INIT_LOG(INFO,
1789                                 " Port %d: Link Up - speed %u Mbps - %s\n",
1790                                 dev->data->port_id, (unsigned)link.link_speed,
1791                                 link.link_duplex == ETH_LINK_FULL_DUPLEX ?
1792                                         "full-duplex" : "half-duplex");
1793                 } else {
1794                         PMD_INIT_LOG(INFO, " Port %d: Link Down\n",
1795                                                 dev->data->port_id);
1796                 }
1797                 PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
1798                                         dev->pci_dev->addr.domain,
1799                                         dev->pci_dev->addr.bus,
1800                                         dev->pci_dev->addr.devid,
1801                                         dev->pci_dev->addr.function);
1802                 tctl = E1000_READ_REG(hw, E1000_TCTL);
1803                 rctl = E1000_READ_REG(hw, E1000_RCTL);
1804                 if (link.link_status) {
1805                         /* enable Tx/Rx */
1806                         tctl |= E1000_TCTL_EN;
1807                         rctl |= E1000_RCTL_EN;
1808                 } else {
1809                         /* disable Tx/Rx */
1810                         tctl &= ~E1000_TCTL_EN;
1811                         rctl &= ~E1000_RCTL_EN;
1812                 }
1813                 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
1814                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1815                 E1000_WRITE_FLUSH(hw);
1816                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
1817         }
1818
1819         return 0;
1820 }
1821
1822 /**
1823  * Interrupt handler which shall be registered at first.
1824  *
1825  * @param handle
1826  *  Pointer to interrupt handle.
1827  * @param param
1828  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1829  *
1830  * @return
1831  *  void
1832  */
1833 static void
1834 eth_igb_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
1835                                                         void *param)
1836 {
1837         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1838
1839         eth_igb_interrupt_get_status(dev);
1840         eth_igb_interrupt_action(dev);
1841 }
1842
1843 static int
1844 eth_igb_led_on(struct rte_eth_dev *dev)
1845 {
1846         struct e1000_hw *hw;
1847
1848         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1849         return (e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
1850 }
1851
1852 static int
1853 eth_igb_led_off(struct rte_eth_dev *dev)
1854 {
1855         struct e1000_hw *hw;
1856
1857         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1858         return (e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
1859 }
1860
1861 static int
1862 eth_igb_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1863 {
1864         struct e1000_hw *hw;
1865         uint32_t ctrl;
1866         int tx_pause;
1867         int rx_pause;
1868
1869         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1870         fc_conf->pause_time = hw->fc.pause_time;
1871         fc_conf->high_water = hw->fc.high_water;
1872         fc_conf->low_water = hw->fc.low_water;
1873         fc_conf->send_xon = hw->fc.send_xon;
1874         fc_conf->autoneg = hw->mac.autoneg;
1875
1876         /*
1877          * Return rx_pause and tx_pause status according to actual setting of
1878          * the TFCE and RFCE bits in the CTRL register.
1879          */
1880         ctrl = E1000_READ_REG(hw, E1000_CTRL);
1881         if (ctrl & E1000_CTRL_TFCE)
1882                 tx_pause = 1;
1883         else
1884                 tx_pause = 0;
1885
1886         if (ctrl & E1000_CTRL_RFCE)
1887                 rx_pause = 1;
1888         else
1889                 rx_pause = 0;
1890
1891         if (rx_pause && tx_pause)
1892                 fc_conf->mode = RTE_FC_FULL;
1893         else if (rx_pause)
1894                 fc_conf->mode = RTE_FC_RX_PAUSE;
1895         else if (tx_pause)
1896                 fc_conf->mode = RTE_FC_TX_PAUSE;
1897         else
1898                 fc_conf->mode = RTE_FC_NONE;
1899
1900         return 0;
1901 }
1902
1903 static int
1904 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1905 {
1906         struct e1000_hw *hw;
1907         int err;
1908         enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = {
1909                 e1000_fc_none,
1910                 e1000_fc_rx_pause,
1911                 e1000_fc_tx_pause,
1912                 e1000_fc_full
1913         };
1914         uint32_t rx_buf_size;
1915         uint32_t max_high_water;
1916         uint32_t rctl;
1917
1918         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1919         if (fc_conf->autoneg != hw->mac.autoneg)
1920                 return -ENOTSUP;
1921         rx_buf_size = igb_get_rx_buffer_size(hw);
1922         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
1923
1924         /* At least reserve one Ethernet frame for watermark */
1925         max_high_water = rx_buf_size - ETHER_MAX_LEN;
1926         if ((fc_conf->high_water > max_high_water) ||
1927                 (fc_conf->high_water < fc_conf->low_water)) {
1928                 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value \n");
1929                 PMD_INIT_LOG(ERR, "high water must <=  0x%x \n", max_high_water);
1930                 return (-EINVAL);
1931         }
1932
1933         hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode];
1934         hw->fc.pause_time     = fc_conf->pause_time;
1935         hw->fc.high_water     = fc_conf->high_water;
1936         hw->fc.low_water      = fc_conf->low_water;
1937         hw->fc.send_xon       = fc_conf->send_xon;
1938
1939         err = e1000_setup_link_generic(hw);
1940         if (err == E1000_SUCCESS) {
1941
1942                 /* check if we want to forward MAC frames - driver doesn't have native
1943                  * capability to do that, so we'll write the registers ourselves */
1944
1945                 rctl = E1000_READ_REG(hw, E1000_RCTL);
1946
1947                 /* set or clear MFLCN.PMCF bit depending on configuration */
1948                 if (fc_conf->mac_ctrl_frame_fwd != 0)
1949                         rctl |= E1000_RCTL_PMCF;
1950                 else
1951                         rctl &= ~E1000_RCTL_PMCF;
1952
1953                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1954                 E1000_WRITE_FLUSH(hw);
1955
1956                 return 0;
1957         }
1958
1959         PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x \n", err);
1960         return (-EIO);
1961 }
1962
1963 #define E1000_RAH_POOLSEL_SHIFT      (18)
1964 static void
1965 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
1966                 uint32_t index, __rte_unused uint32_t pool)
1967 {
1968         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1969         uint32_t rah;
1970
1971         e1000_rar_set(hw, mac_addr->addr_bytes, index);
1972         rah = E1000_READ_REG(hw, E1000_RAH(index));
1973         rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool));
1974         E1000_WRITE_REG(hw, E1000_RAH(index), rah);
1975 }
1976
1977 static void
1978 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index)
1979 {
1980         uint8_t addr[ETHER_ADDR_LEN];
1981         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1982
1983         memset(addr, 0, sizeof(addr));
1984
1985         e1000_rar_set(hw, addr, index);
1986 }
1987
1988 /*
1989  * Virtual Function operations
1990  */
1991 static void
1992 igbvf_intr_disable(struct e1000_hw *hw)
1993 {
1994         PMD_INIT_LOG(DEBUG, "igbvf_intr_disable");
1995
1996         /* Clear interrupt mask to stop from interrupts being generated */
1997         E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF);
1998
1999         E1000_WRITE_FLUSH(hw);
2000 }
2001
2002 static void
2003 igbvf_stop_adapter(struct rte_eth_dev *dev)
2004 {
2005         u32 reg_val;
2006         u16 i;
2007         struct rte_eth_dev_info dev_info;
2008         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2009
2010         memset(&dev_info, 0, sizeof(dev_info));
2011         eth_igb_infos_get(dev, &dev_info);
2012
2013         /* Clear interrupt mask to stop from interrupts being generated */
2014         igbvf_intr_disable(hw);
2015
2016         /* Clear any pending interrupts, flush previous writes */
2017         E1000_READ_REG(hw, E1000_EICR);
2018
2019         /* Disable the transmit unit.  Each queue must be disabled. */
2020         for (i = 0; i < dev_info.max_tx_queues; i++)
2021                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH);
2022
2023         /* Disable the receive unit by stopping each queue */
2024         for (i = 0; i < dev_info.max_rx_queues; i++) {
2025                 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i));
2026                 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE;
2027                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val);
2028                 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE)
2029                         ;
2030         }
2031
2032         /* flush all queues disables */
2033         E1000_WRITE_FLUSH(hw);
2034         msec_delay(2);
2035 }
2036
2037 static int eth_igbvf_link_update(struct e1000_hw *hw)
2038 {
2039         struct e1000_mbx_info *mbx = &hw->mbx;
2040         struct e1000_mac_info *mac = &hw->mac;
2041         int ret_val = E1000_SUCCESS;
2042
2043         PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf");
2044
2045         /*
2046          * We only want to run this if there has been a rst asserted.
2047          * in this case that could mean a link change, device reset,
2048          * or a virtual function reset
2049          */
2050
2051         /* If we were hit with a reset or timeout drop the link */
2052         if (!e1000_check_for_rst(hw, 0) || !mbx->timeout)
2053                 mac->get_link_status = TRUE;
2054
2055         if (!mac->get_link_status)
2056                 goto out;
2057
2058         /* if link status is down no point in checking to see if pf is up */
2059         if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU))
2060                 goto out;
2061
2062         /* if we passed all the tests above then the link is up and we no
2063          * longer need to check for link */
2064         mac->get_link_status = FALSE;
2065
2066 out:
2067         return ret_val;
2068 }
2069
2070
2071 static int
2072 igbvf_dev_configure(struct rte_eth_dev *dev)
2073 {
2074         struct rte_eth_conf* conf = &dev->data->dev_conf;
2075
2076         PMD_INIT_LOG(DEBUG, "\nConfigured Virtual Function port id: %d\n",
2077                 dev->data->port_id);
2078
2079         /*
2080          * VF has no ability to enable/disable HW CRC
2081          * Keep the persistent behavior the same as Host PF
2082          */
2083 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC
2084         if (!conf->rxmode.hw_strip_crc) {
2085                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip\n");
2086                 conf->rxmode.hw_strip_crc = 1;
2087         }
2088 #else
2089         if (conf->rxmode.hw_strip_crc) {
2090                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip\n");
2091                 conf->rxmode.hw_strip_crc = 0;
2092         }
2093 #endif
2094
2095         return 0;
2096 }
2097
2098 static int
2099 igbvf_dev_start(struct rte_eth_dev *dev)
2100 {
2101         struct e1000_hw *hw =
2102                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2103         int ret;
2104
2105         PMD_INIT_LOG(DEBUG, "igbvf_dev_start");
2106
2107         hw->mac.ops.reset_hw(hw);
2108
2109         /* Set all vfta */
2110         igbvf_set_vfta_all(dev,1);
2111
2112         eth_igbvf_tx_init(dev);
2113
2114         /* This can fail when allocating mbufs for descriptor rings */
2115         ret = eth_igbvf_rx_init(dev);
2116         if (ret) {
2117                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
2118                 igb_dev_clear_queues(dev);
2119                 return ret;
2120         }
2121
2122         return 0;
2123 }
2124
2125 static void
2126 igbvf_dev_stop(struct rte_eth_dev *dev)
2127 {
2128         PMD_INIT_LOG(DEBUG, "igbvf_dev_stop");
2129
2130         igbvf_stop_adapter(dev);
2131
2132         /*
2133           * Clear what we set, but we still keep shadow_vfta to
2134           * restore after device starts
2135           */
2136         igbvf_set_vfta_all(dev,0);
2137
2138         igb_dev_clear_queues(dev);
2139 }
2140
2141 static void
2142 igbvf_dev_close(struct rte_eth_dev *dev)
2143 {
2144         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2145
2146         PMD_INIT_LOG(DEBUG, "igbvf_dev_close");
2147
2148         e1000_reset_hw(hw);
2149
2150         igbvf_dev_stop(dev);
2151 }
2152
2153 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on)
2154 {
2155         struct e1000_mbx_info *mbx = &hw->mbx;
2156         uint32_t msgbuf[2];
2157
2158         /* After set vlan, vlan strip will also be enabled in igb driver*/
2159         msgbuf[0] = E1000_VF_SET_VLAN;
2160         msgbuf[1] = vid;
2161         /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */
2162         if (on)
2163                 msgbuf[0] |= E1000_VF_SET_VLAN_ADD;
2164
2165         return (mbx->ops.write_posted(hw, msgbuf, 2, 0));
2166 }
2167
2168 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2169 {
2170         struct e1000_hw *hw =
2171                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2172         struct e1000_vfta * shadow_vfta =
2173                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2174         int i = 0, j = 0, vfta = 0, mask = 1;
2175
2176         for (i = 0; i < IGB_VFTA_SIZE; i++){
2177                 vfta = shadow_vfta->vfta[i];
2178                 if(vfta){
2179                         mask = 1;
2180                         for (j = 0; j < 32; j++){
2181                                 if(vfta & mask)
2182                                         igbvf_set_vfta(hw,
2183                                                 (uint16_t)((i<<5)+j), on);
2184                                 mask<<=1;
2185                         }
2186                 }
2187         }
2188
2189 }
2190
2191 static int
2192 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2193 {
2194         struct e1000_hw *hw =
2195                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2196         struct e1000_vfta * shadow_vfta =
2197                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2198         uint32_t vid_idx = 0;
2199         uint32_t vid_bit = 0;
2200         int ret = 0;
2201
2202         PMD_INIT_LOG(DEBUG, "igbvf_vlan_filter_set");
2203
2204         /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/
2205         ret = igbvf_set_vfta(hw, vlan_id, !!on);
2206         if(ret){
2207                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2208                 return ret;
2209         }
2210         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2211         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2212
2213         /*Save what we set and retore it after device reset*/
2214         if (on)
2215                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2216         else
2217                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2218
2219         return 0;
2220 }
2221
2222 static int
2223 eth_igb_rss_reta_update(struct rte_eth_dev *dev,
2224                                 struct rte_eth_rss_reta *reta_conf)
2225 {
2226         uint8_t i,j,mask;
2227         uint32_t reta;
2228         struct e1000_hw *hw =
2229                         E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2230
2231         /*
2232          * Update Redirection Table RETA[n],n=0...31,The redirection table has
2233          * 128-entries in 32 registers
2234          */
2235         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2236                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2237                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2238                 else
2239                         mask = (uint8_t)((reta_conf->mask_hi >>
2240                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2241                 if (mask != 0) {
2242                         reta = 0;
2243                         /* If all 4 entries were set,don't need read RETA register */
2244                         if (mask != 0xF)
2245                                 reta = E1000_READ_REG(hw,E1000_RETA(i >> 2));
2246
2247                         for (j = 0; j < 4; j++) {
2248                                 if (mask & (0x1 << j)) {
2249                                         if (mask != 0xF)
2250                                                 reta &= ~(0xFF << 8 * j);
2251                                         reta |= reta_conf->reta[i + j] << 8 * j;
2252                                 }
2253                         }
2254                         E1000_WRITE_REG(hw, E1000_RETA(i >> 2),reta);
2255                 }
2256         }
2257
2258         return 0;
2259 }
2260
2261 static int
2262 eth_igb_rss_reta_query(struct rte_eth_dev *dev,
2263                                 struct rte_eth_rss_reta *reta_conf)
2264 {
2265         uint8_t i,j,mask;
2266         uint32_t reta;
2267         struct e1000_hw *hw =
2268                         E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2269
2270         /*
2271          * Read Redirection Table RETA[n],n=0...31,The redirection table has
2272          * 128-entries in 32 registers
2273          */
2274         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2275                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2276                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2277                 else
2278                         mask = (uint8_t)((reta_conf->mask_hi >>
2279                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2280
2281                 if (mask != 0) {
2282                         reta = E1000_READ_REG(hw,E1000_RETA(i >> 2));
2283                         for (j = 0; j < 4; j++) {
2284                                 if (mask & (0x1 << j))
2285                                         reta_conf->reta[i + j] =
2286                                                 (uint8_t)((reta >> 8 * j) & 0xFF);
2287                         }
2288                 }
2289         }
2290
2291         return 0;
2292 }
2293
2294 #define MAC_TYPE_FILTER_SUP(type)    do {\
2295         if ((type) != e1000_82580 && (type) != e1000_i350 &&\
2296                 (type) != e1000_82576)\
2297                 return -ENOSYS;\
2298 } while (0)
2299
2300 /*
2301  * add the syn filter
2302  *
2303  * @param
2304  * dev: Pointer to struct rte_eth_dev.
2305  * filter: ponter to the filter that will be added.
2306  * rx_queue: the queue id the filter assigned to.
2307  *
2308  * @return
2309  *    - On success, zero.
2310  *    - On failure, a negative value.
2311  */
2312 static int
2313 eth_igb_add_syn_filter(struct rte_eth_dev *dev,
2314                         struct rte_syn_filter *filter, uint16_t rx_queue)
2315 {
2316         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2317         uint32_t synqf, rfctl;
2318
2319         MAC_TYPE_FILTER_SUP(hw->mac.type);
2320
2321         if (rx_queue >= IGB_MAX_RX_QUEUE_NUM)
2322                 return -EINVAL;
2323
2324         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
2325         if (synqf & E1000_SYN_FILTER_ENABLE)
2326                 return -EINVAL;
2327
2328         synqf = (uint32_t)(((rx_queue << E1000_SYN_FILTER_QUEUE_SHIFT) &
2329                 E1000_SYN_FILTER_QUEUE) | E1000_SYN_FILTER_ENABLE);
2330
2331         rfctl = E1000_READ_REG(hw, E1000_RFCTL);
2332         if (filter->hig_pri)
2333                 rfctl |= E1000_RFCTL_SYNQFP;
2334         else
2335                 rfctl &= ~E1000_RFCTL_SYNQFP;
2336
2337         E1000_WRITE_REG(hw, E1000_SYNQF(0), synqf);
2338         E1000_WRITE_REG(hw, E1000_RFCTL, rfctl);
2339         return 0;
2340 }
2341
2342 /*
2343  * remove the syn filter
2344  *
2345  * @param
2346  * dev: Pointer to struct rte_eth_dev.
2347  *
2348  * @return
2349  *    - On success, zero.
2350  *    - On failure, a negative value.
2351  */
2352 static int
2353 eth_igb_remove_syn_filter(struct rte_eth_dev *dev)
2354 {
2355         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2356
2357         MAC_TYPE_FILTER_SUP(hw->mac.type);
2358
2359         E1000_WRITE_REG(hw, E1000_SYNQF(0), 0);
2360         return 0;
2361 }
2362
2363 /*
2364  * get the syn filter's info
2365  *
2366  * @param
2367  * dev: Pointer to struct rte_eth_dev.
2368  * filter: ponter to the filter that returns.
2369  * *rx_queue: pointer to the queue id the filter assigned to.
2370  *
2371  * @return
2372  *    - On success, zero.
2373  *    - On failure, a negative value.
2374  */
2375 static int
2376 eth_igb_get_syn_filter(struct rte_eth_dev *dev,
2377                         struct rte_syn_filter *filter, uint16_t *rx_queue)
2378 {
2379         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2380         uint32_t synqf, rfctl;
2381
2382         MAC_TYPE_FILTER_SUP(hw->mac.type);
2383         synqf = E1000_READ_REG(hw, E1000_SYNQF(0));
2384         if (synqf & E1000_SYN_FILTER_ENABLE) {
2385                 rfctl = E1000_READ_REG(hw, E1000_RFCTL);
2386                 filter->hig_pri = (rfctl & E1000_RFCTL_SYNQFP) ? 1 : 0;
2387                 *rx_queue = (uint8_t)((synqf & E1000_SYN_FILTER_QUEUE) >>
2388                                 E1000_SYN_FILTER_QUEUE_SHIFT);
2389                 return 0;
2390         }
2391         return -ENOENT;
2392 }
2393
2394 /*
2395  * add an ethertype filter
2396  *
2397  * @param
2398  * dev: Pointer to struct rte_eth_dev.
2399  * index: the index the filter allocates.
2400  * filter: ponter to the filter that will be added.
2401  * rx_queue: the queue id the filter assigned to.
2402  *
2403  * @return
2404  *    - On success, zero.
2405  *    - On failure, a negative value.
2406  */
2407 static int
2408 eth_igb_add_ethertype_filter(struct rte_eth_dev *dev, uint16_t index,
2409                         struct rte_ethertype_filter *filter, uint16_t rx_queue)
2410 {
2411         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2412         uint32_t etqf;
2413
2414         MAC_TYPE_FILTER_SUP(hw->mac.type);
2415
2416         if (index >= E1000_MAX_ETQF_FILTERS || rx_queue >= IGB_MAX_RX_QUEUE_NUM)
2417                 return -EINVAL;
2418
2419         etqf = E1000_READ_REG(hw, E1000_ETQF(index));
2420         if (etqf & E1000_ETQF_FILTER_ENABLE)
2421                 return -EINVAL;  /* filter index is in use. */
2422         else
2423                 etqf = 0;
2424
2425         etqf |= E1000_ETQF_FILTER_ENABLE | E1000_ETQF_QUEUE_ENABLE;
2426         etqf |= (uint32_t)(filter->ethertype & E1000_ETQF_ETHERTYPE);
2427         etqf |= rx_queue << E1000_ETQF_QUEUE_SHIFT;
2428
2429         if (filter->priority_en) {
2430                 PMD_INIT_LOG(ERR, "vlan and priority (%d) is not supported"
2431                         " in E1000.", filter->priority);
2432                 return -EINVAL;
2433         }
2434
2435         E1000_WRITE_REG(hw, E1000_ETQF(index), etqf);
2436         return 0;
2437 }
2438
2439 /*
2440  * remove an ethertype filter
2441  *
2442  * @param
2443  * dev: Pointer to struct rte_eth_dev.
2444  * index: the index the filter allocates.
2445  *
2446  * @return
2447  *    - On success, zero.
2448  *    - On failure, a negative value.
2449  */
2450 static int
2451 eth_igb_remove_ethertype_filter(struct rte_eth_dev *dev, uint16_t index)
2452 {
2453         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2454
2455         MAC_TYPE_FILTER_SUP(hw->mac.type);
2456
2457         if (index >= E1000_MAX_ETQF_FILTERS)
2458                 return -EINVAL;
2459
2460         E1000_WRITE_REG(hw, E1000_ETQF(index), 0);
2461         return 0;
2462 }
2463
2464 /*
2465  * get an ethertype filter
2466  *
2467  * @param
2468  * dev: Pointer to struct rte_eth_dev.
2469  * index: the index the filter allocates.
2470  * filter: ponter to the filter that will be gotten.
2471  * *rx_queue: the ponited of the queue id the filter assigned to.
2472  *
2473  * @return
2474  *    - On success, zero.
2475  *    - On failure, a negative value.
2476  */
2477 static int
2478 eth_igb_get_ethertype_filter(struct rte_eth_dev *dev, uint16_t index,
2479                         struct rte_ethertype_filter *filter, uint16_t *rx_queue)
2480 {
2481         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2482         uint32_t etqf;
2483
2484         MAC_TYPE_FILTER_SUP(hw->mac.type);
2485
2486         if (index >= E1000_MAX_ETQF_FILTERS)
2487                 return -EINVAL;
2488
2489         etqf = E1000_READ_REG(hw, E1000_ETQF(index));
2490         if (etqf & E1000_ETQF_FILTER_ENABLE) {
2491                 filter->ethertype = etqf & E1000_ETQF_ETHERTYPE;
2492                 filter->priority_en = 0;
2493                 *rx_queue = (etqf & E1000_ETQF_QUEUE) >> E1000_ETQF_QUEUE_SHIFT;
2494                 return 0;
2495         }
2496         return -ENOENT;
2497 }
2498
2499 #define MAC_TYPE_FILTER_SUP_EXT(type)    do {\
2500         if ((type) != e1000_82580 && (type) != e1000_i350)\
2501                 return -ENOSYS; \
2502 } while (0)
2503
2504 /*
2505  * add a 2tuple filter
2506  *
2507  * @param
2508  * dev: Pointer to struct rte_eth_dev.
2509  * index: the index the filter allocates.
2510  * filter: ponter to the filter that will be added.
2511  * rx_queue: the queue id the filter assigned to.
2512  *
2513  * @return
2514  *    - On success, zero.
2515  *    - On failure, a negative value.
2516  */
2517 static int
2518 eth_igb_add_2tuple_filter(struct rte_eth_dev *dev, uint16_t index,
2519                         struct rte_2tuple_filter *filter, uint16_t rx_queue)
2520 {
2521         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2522         uint32_t ttqf, imir = 0;
2523         uint32_t imir_ext = 0;
2524
2525         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
2526
2527         if (index >= E1000_MAX_TTQF_FILTERS ||
2528                 rx_queue >= IGB_MAX_RX_QUEUE_NUM ||
2529                 filter->priority > E1000_2TUPLE_MAX_PRI)
2530                 return -EINVAL;  /* filter index is out of range. */
2531         if  (filter->tcp_flags > TCP_FLAG_ALL)
2532                 return -EINVAL;  /* flags is invalid. */
2533
2534         ttqf = E1000_READ_REG(hw, E1000_TTQF(index));
2535         if (ttqf & E1000_TTQF_QUEUE_ENABLE)
2536                 return -EINVAL;  /* filter index is in use. */
2537
2538         imir = (uint32_t)(filter->dst_port & E1000_IMIR_DSTPORT);
2539         if (filter->dst_port_mask == 1) /* 1b means not compare. */
2540                 imir |= E1000_IMIR_PORT_BP;
2541         else
2542                 imir &= ~E1000_IMIR_PORT_BP;
2543
2544         imir |= filter->priority << E1000_IMIR_PRIORITY_SHIFT;
2545
2546         ttqf = 0;
2547         ttqf |= E1000_TTQF_QUEUE_ENABLE;
2548         ttqf |= (uint32_t)(rx_queue << E1000_TTQF_QUEUE_SHIFT);
2549         ttqf |= (uint32_t)(filter->protocol & E1000_TTQF_PROTOCOL_MASK);
2550         if (filter->protocol_mask == 1)
2551                 ttqf |= E1000_TTQF_MASK_ENABLE;
2552         else
2553                 ttqf &= ~E1000_TTQF_MASK_ENABLE;
2554
2555         imir_ext |= E1000_IMIR_EXT_SIZE_BP;
2556         /* tcp flags bits setting. */
2557         if (filter->tcp_flags & TCP_FLAG_ALL) {
2558                 if (filter->tcp_flags & TCP_UGR_FLAG)
2559                         imir_ext |= E1000_IMIR_EXT_CTRL_UGR;
2560                 if (filter->tcp_flags & TCP_ACK_FLAG)
2561                         imir_ext |= E1000_IMIR_EXT_CTRL_ACK;
2562                 if (filter->tcp_flags & TCP_PSH_FLAG)
2563                         imir_ext |= E1000_IMIR_EXT_CTRL_PSH;
2564                 if (filter->tcp_flags & TCP_RST_FLAG)
2565                         imir_ext |= E1000_IMIR_EXT_CTRL_RST;
2566                 if (filter->tcp_flags & TCP_SYN_FLAG)
2567                         imir_ext |= E1000_IMIR_EXT_CTRL_SYN;
2568                 if (filter->tcp_flags & TCP_FIN_FLAG)
2569                         imir_ext |= E1000_IMIR_EXT_CTRL_FIN;
2570                 imir_ext &= ~E1000_IMIR_EXT_CTRL_BP;
2571         } else
2572                 imir_ext |= E1000_IMIR_EXT_CTRL_BP;
2573         E1000_WRITE_REG(hw, E1000_IMIR(index), imir);
2574         E1000_WRITE_REG(hw, E1000_TTQF(index), ttqf);
2575         E1000_WRITE_REG(hw, E1000_IMIREXT(index), imir_ext);
2576         return 0;
2577 }
2578
2579 /*
2580  * remove a 2tuple filter
2581  *
2582  * @param
2583  * dev: Pointer to struct rte_eth_dev.
2584  * index: the index the filter allocates.
2585  *
2586  * @return
2587  *    - On success, zero.
2588  *    - On failure, a negative value.
2589  */
2590 static int
2591 eth_igb_remove_2tuple_filter(struct rte_eth_dev *dev,
2592                         uint16_t index)
2593 {
2594         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2595
2596         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
2597
2598         if (index >= E1000_MAX_TTQF_FILTERS)
2599                 return -EINVAL;  /* filter index is out of range */
2600
2601         E1000_WRITE_REG(hw, E1000_TTQF(index), 0);
2602         E1000_WRITE_REG(hw, E1000_IMIR(index), 0);
2603         E1000_WRITE_REG(hw, E1000_IMIREXT(index), 0);
2604         return 0;
2605 }
2606
2607 /*
2608  * get a 2tuple filter
2609  *
2610  * @param
2611  * dev: Pointer to struct rte_eth_dev.
2612  * index: the index the filter allocates.
2613  * filter: ponter to the filter that returns.
2614  * *rx_queue: pointer of the queue id the filter assigned to.
2615  *
2616  * @return
2617  *    - On success, zero.
2618  *    - On failure, a negative value.
2619  */
2620 static int
2621 eth_igb_get_2tuple_filter(struct rte_eth_dev *dev, uint16_t index,
2622                         struct rte_2tuple_filter *filter, uint16_t *rx_queue)
2623 {
2624         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2625         uint32_t imir, ttqf, imir_ext;
2626
2627         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
2628
2629         if (index >= E1000_MAX_TTQF_FILTERS)
2630                 return -EINVAL;  /* filter index is out of range. */
2631
2632         ttqf = E1000_READ_REG(hw, E1000_TTQF(index));
2633         if (ttqf & E1000_TTQF_QUEUE_ENABLE) {
2634                 imir = E1000_READ_REG(hw, E1000_IMIR(index));
2635                 filter->protocol = ttqf & E1000_TTQF_PROTOCOL_MASK;
2636                 filter->protocol_mask = (ttqf & E1000_TTQF_MASK_ENABLE) ? 1 : 0;
2637                 *rx_queue = (ttqf & E1000_TTQF_RX_QUEUE_MASK) >>
2638                                 E1000_TTQF_QUEUE_SHIFT;
2639                 filter->dst_port = (uint16_t)(imir & E1000_IMIR_DSTPORT);
2640                 filter->dst_port_mask = (imir & E1000_IMIR_PORT_BP) ? 1 : 0;
2641                 filter->priority = (imir & E1000_IMIR_PRIORITY) >>
2642                         E1000_IMIR_PRIORITY_SHIFT;
2643
2644                 imir_ext = E1000_READ_REG(hw, E1000_IMIREXT(index));
2645                 if (!(imir_ext & E1000_IMIR_EXT_CTRL_BP)) {
2646                         if (imir_ext & E1000_IMIR_EXT_CTRL_UGR)
2647                                 filter->tcp_flags |= TCP_UGR_FLAG;
2648                         if (imir_ext & E1000_IMIR_EXT_CTRL_ACK)
2649                                 filter->tcp_flags |= TCP_ACK_FLAG;
2650                         if (imir_ext & E1000_IMIR_EXT_CTRL_PSH)
2651                                 filter->tcp_flags |= TCP_PSH_FLAG;
2652                         if (imir_ext & E1000_IMIR_EXT_CTRL_RST)
2653                                 filter->tcp_flags |= TCP_RST_FLAG;
2654                         if (imir_ext & E1000_IMIR_EXT_CTRL_SYN)
2655                                 filter->tcp_flags |= TCP_SYN_FLAG;
2656                         if (imir_ext & E1000_IMIR_EXT_CTRL_FIN)
2657                                 filter->tcp_flags |= TCP_FIN_FLAG;
2658                 } else
2659                         filter->tcp_flags = 0;
2660                 return 0;
2661         }
2662         return -ENOENT;
2663 }
2664
2665 /*
2666  * add a flex filter
2667  *
2668  * @param
2669  * dev: Pointer to struct rte_eth_dev.
2670  * index: the index the filter allocates.
2671  * filter: ponter to the filter that will be added.
2672  * rx_queue: the queue id the filter assigned to.
2673  *
2674  * @return
2675  *    - On success, zero.
2676  *     - On failure, a negative value.
2677  */
2678 static int
2679 eth_igb_add_flex_filter(struct rte_eth_dev *dev, uint16_t index,
2680                         struct rte_flex_filter *filter, uint16_t rx_queue)
2681 {
2682         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2683         uint32_t wufc, en_bits = 0;
2684         uint32_t queueing = 0;
2685         uint32_t reg_off = 0;
2686         uint8_t i, j = 0;
2687
2688         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
2689
2690         if (index >= E1000_MAX_FLEXIBLE_FILTERS)
2691                 return -EINVAL;  /* filter index is out of range. */
2692
2693         if (filter->len == 0 || filter->len > E1000_MAX_FLEX_FILTER_LEN ||
2694                 filter->len % 8 != 0 ||
2695                 filter->priority > E1000_MAX_FLEX_FILTER_PRI)
2696                 return -EINVAL;
2697
2698         wufc = E1000_READ_REG(hw, E1000_WUFC);
2699         en_bits = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << index);
2700         if ((wufc & en_bits) == en_bits)
2701                 return -EINVAL;  /* the filter is in use. */
2702
2703         E1000_WRITE_REG(hw, E1000_WUFC,
2704                 wufc | E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << index));
2705
2706         j = 0;
2707         if (index < E1000_MAX_FHFT)
2708                 reg_off = E1000_FHFT(index);
2709         else
2710                 reg_off = E1000_FHFT_EXT(index - E1000_MAX_FHFT);
2711
2712         for (i = 0; i < 16; i++) {
2713                 E1000_WRITE_REG(hw, reg_off + i*4*4, filter->dwords[j]);
2714                 E1000_WRITE_REG(hw, reg_off + (i*4+1)*4, filter->dwords[++j]);
2715                 E1000_WRITE_REG(hw, reg_off + (i*4+2)*4,
2716                                 (uint32_t)filter->mask[i]);
2717                 ++j;
2718         }
2719         queueing |= filter->len |
2720                 (rx_queue << E1000_FHFT_QUEUEING_QUEUE_SHIFT) |
2721                 (filter->priority << E1000_FHFT_QUEUEING_PRIO_SHIFT);
2722         E1000_WRITE_REG(hw, reg_off + E1000_FHFT_QUEUEING_OFFSET, queueing);
2723         return 0;
2724 }
2725
2726 /*
2727  * remove a flex filter
2728  *
2729  * @param
2730  * dev: Pointer to struct rte_eth_dev.
2731  * index: the index the filter allocates.
2732  *
2733  * @return
2734  *    - On success, zero.
2735  *    - On failure, a negative value.
2736  */
2737 static int
2738 eth_igb_remove_flex_filter(struct rte_eth_dev *dev,
2739                                 uint16_t index)
2740 {
2741         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2742         uint32_t wufc, reg_off = 0;
2743         uint8_t i;
2744
2745         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
2746
2747         if (index >= E1000_MAX_FLEXIBLE_FILTERS)
2748                 return -EINVAL;  /* filter index is out of range. */
2749
2750         wufc = E1000_READ_REG(hw, E1000_WUFC);
2751         E1000_WRITE_REG(hw, E1000_WUFC, wufc & (~(E1000_WUFC_FLX0 << index)));
2752
2753         if (index < E1000_MAX_FHFT)
2754                 reg_off = E1000_FHFT(index);
2755         else
2756                 reg_off = E1000_FHFT_EXT(index - E1000_MAX_FHFT);
2757
2758         for (i = 0; i < 64; i++)
2759                 E1000_WRITE_REG(hw, reg_off + i*4, 0);
2760         return 0;
2761 }
2762
2763 /*
2764  * get a flex filter
2765  *
2766  * @param
2767  * dev: Pointer to struct rte_eth_dev.
2768  * index: the index the filter allocates.
2769  * filter: ponter to the filter that returns.
2770  * *rx_queue: the pointer of the queue id the filter assigned to.
2771  *
2772  * @return
2773  *    - On success, zero.
2774  *    - On failure, a negative value.
2775  */
2776 static int
2777 eth_igb_get_flex_filter(struct rte_eth_dev *dev, uint16_t index,
2778                         struct rte_flex_filter *filter, uint16_t *rx_queue)
2779 {
2780         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2781         uint32_t wufc, queueing, wufc_en = 0;
2782         uint8_t i, j;
2783
2784         MAC_TYPE_FILTER_SUP_EXT(hw->mac.type);
2785
2786         if (index >= E1000_MAX_FLEXIBLE_FILTERS)
2787                 return -EINVAL;  /* filter index is out of range. */
2788
2789         wufc = E1000_READ_REG(hw, E1000_WUFC);
2790         wufc_en = E1000_WUFC_FLEX_HQ | (E1000_WUFC_FLX0 << index);
2791
2792         if ((wufc & wufc_en) == wufc_en) {
2793                 uint32_t reg_off = 0;
2794                 j = 0;
2795                 if (index < E1000_MAX_FHFT)
2796                         reg_off = E1000_FHFT(index);
2797                 else
2798                         reg_off = E1000_FHFT_EXT(index - E1000_MAX_FHFT);
2799
2800                 for (i = 0; i < 16; i++, j = i * 2) {
2801                         filter->dwords[j] =
2802                                 E1000_READ_REG(hw, reg_off + i*4*4);
2803                         filter->dwords[j+1] =
2804                                 E1000_READ_REG(hw, reg_off + (i*4+1)*4);
2805                         filter->mask[i] =
2806                                 E1000_READ_REG(hw, reg_off + (i*4+2)*4);
2807                 }
2808                 queueing = E1000_READ_REG(hw,
2809                                 reg_off + E1000_FHFT_QUEUEING_OFFSET);
2810                 filter->len = queueing & E1000_FHFT_QUEUEING_LEN;
2811                 filter->priority = (queueing & E1000_FHFT_QUEUEING_PRIO) >>
2812                         E1000_FHFT_QUEUEING_PRIO_SHIFT;
2813                 *rx_queue = (queueing & E1000_FHFT_QUEUEING_QUEUE) >>
2814                         E1000_FHFT_QUEUEING_QUEUE_SHIFT;
2815                 return 0;
2816         }
2817         return -ENOENT;
2818 }
2819
2820 /*
2821  * add a 5tuple filter
2822  *
2823  * @param
2824  * dev: Pointer to struct rte_eth_dev.
2825  * index: the index the filter allocates.
2826  * filter: ponter to the filter that will be added.
2827  * rx_queue: the queue id the filter assigned to.
2828  *
2829  * @return
2830  *    - On success, zero.
2831  *    - On failure, a negative value.
2832  */
2833 static int
2834 eth_igb_add_5tuple_filter(struct rte_eth_dev *dev, uint16_t index,
2835                         struct rte_5tuple_filter *filter, uint16_t rx_queue)
2836 {
2837         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2838         uint32_t ftqf, spqf = 0;
2839         uint32_t imir = 0;
2840         uint32_t imir_ext = 0;
2841
2842         if (hw->mac.type != e1000_82576)
2843                 return -ENOSYS;
2844
2845         if (index >= E1000_MAX_FTQF_FILTERS ||
2846                 rx_queue >= IGB_MAX_RX_QUEUE_NUM_82576)
2847                 return -EINVAL;  /* filter index is out of range. */
2848
2849         ftqf = E1000_READ_REG(hw, E1000_FTQF(index));
2850         if (ftqf & E1000_FTQF_QUEUE_ENABLE)
2851                 return -EINVAL;  /* filter index is in use. */
2852
2853         ftqf = 0;
2854         ftqf |= filter->protocol & E1000_FTQF_PROTOCOL_MASK;
2855         if (filter->src_ip_mask == 1) /* 1b means not compare. */
2856                 ftqf |= E1000_FTQF_SOURCE_ADDR_MASK;
2857         if (filter->dst_ip_mask == 1)
2858                 ftqf |= E1000_FTQF_DEST_ADDR_MASK;
2859         if (filter->src_port_mask == 1)
2860                 ftqf |= E1000_FTQF_SOURCE_PORT_MASK;
2861         if (filter->protocol_mask == 1)
2862                 ftqf |= E1000_FTQF_PROTOCOL_COMP_MASK;
2863         ftqf |= (rx_queue << E1000_FTQF_QUEUE_SHIFT) & E1000_FTQF_QUEUE_MASK;
2864         ftqf |= E1000_FTQF_VF_MASK_EN;
2865         ftqf |= E1000_FTQF_QUEUE_ENABLE;
2866         E1000_WRITE_REG(hw, E1000_FTQF(index), ftqf);
2867         E1000_WRITE_REG(hw, E1000_DAQF(index), filter->dst_ip);
2868         E1000_WRITE_REG(hw, E1000_SAQF(index), filter->src_ip);
2869
2870         spqf |= filter->src_port & E1000_SPQF_SRCPORT;
2871         E1000_WRITE_REG(hw, E1000_SPQF(index), spqf);
2872
2873         imir |= (uint32_t)(filter->dst_port & E1000_IMIR_DSTPORT);
2874         if (filter->dst_port_mask == 1) /* 1b means not compare. */
2875                 imir |= E1000_IMIR_PORT_BP;
2876         else
2877                 imir &= ~E1000_IMIR_PORT_BP;
2878         imir |= filter->priority << E1000_IMIR_PRIORITY_SHIFT;
2879
2880         imir_ext |= E1000_IMIR_EXT_SIZE_BP;
2881         /* tcp flags bits setting. */
2882         if (filter->tcp_flags & TCP_FLAG_ALL) {
2883                 if (filter->tcp_flags & TCP_UGR_FLAG)
2884                         imir_ext |= E1000_IMIR_EXT_CTRL_UGR;
2885                 if (filter->tcp_flags & TCP_ACK_FLAG)
2886                         imir_ext |= E1000_IMIR_EXT_CTRL_ACK;
2887                 if (filter->tcp_flags & TCP_PSH_FLAG)
2888                         imir_ext |= E1000_IMIR_EXT_CTRL_PSH;
2889                 if (filter->tcp_flags & TCP_RST_FLAG)
2890                         imir_ext |= E1000_IMIR_EXT_CTRL_RST;
2891                 if (filter->tcp_flags & TCP_SYN_FLAG)
2892                         imir_ext |= E1000_IMIR_EXT_CTRL_SYN;
2893                 if (filter->tcp_flags & TCP_FIN_FLAG)
2894                         imir_ext |= E1000_IMIR_EXT_CTRL_FIN;
2895         } else
2896                 imir_ext |= E1000_IMIR_EXT_CTRL_BP;
2897         E1000_WRITE_REG(hw, E1000_IMIR(index), imir);
2898         E1000_WRITE_REG(hw, E1000_IMIREXT(index), imir_ext);
2899         return 0;
2900 }
2901
2902 /*
2903  * remove a 5tuple filter
2904  *
2905  * @param
2906  * dev: Pointer to struct rte_eth_dev.
2907  * index: the index the filter allocates
2908  *
2909  * @return
2910  *    - On success, zero.
2911  *    - On failure, a negative value.
2912  */
2913 static int
2914 eth_igb_remove_5tuple_filter(struct rte_eth_dev *dev,
2915                                 uint16_t index)
2916 {
2917         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2918
2919         if (hw->mac.type != e1000_82576)
2920                 return -ENOSYS;
2921
2922         if (index >= E1000_MAX_FTQF_FILTERS)
2923                 return -EINVAL;  /* filter index is out of range. */
2924
2925         E1000_WRITE_REG(hw, E1000_FTQF(index), 0);
2926         E1000_WRITE_REG(hw, E1000_DAQF(index), 0);
2927         E1000_WRITE_REG(hw, E1000_SAQF(index), 0);
2928         E1000_WRITE_REG(hw, E1000_SPQF(index), 0);
2929         E1000_WRITE_REG(hw, E1000_IMIR(index), 0);
2930         E1000_WRITE_REG(hw, E1000_IMIREXT(index), 0);
2931         return 0;
2932 }
2933
2934 /*
2935  * get a 5tuple filter
2936  *
2937  * @param
2938  * dev: Pointer to struct rte_eth_dev.
2939  * index: the index the filter allocates
2940  * filter: ponter to the filter that returns
2941  * *rx_queue: pointer of the queue id the filter assigned to
2942  *
2943  * @return
2944  *    - On success, zero.
2945  *    - On failure, a negative value.
2946  */
2947 static int
2948 eth_igb_get_5tuple_filter(struct rte_eth_dev *dev, uint16_t index,
2949                         struct rte_5tuple_filter *filter, uint16_t *rx_queue)
2950 {
2951         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2952         uint32_t spqf, ftqf, imir, imir_ext;
2953
2954         if (hw->mac.type != e1000_82576)
2955                 return -ENOSYS;
2956
2957         if (index >= E1000_MAX_FTQF_FILTERS)
2958                 return -EINVAL;  /* filter index is out of range. */
2959
2960         ftqf = E1000_READ_REG(hw, E1000_FTQF(index));
2961         if (ftqf & E1000_FTQF_QUEUE_ENABLE) {
2962                 filter->src_ip_mask =
2963                         (ftqf & E1000_FTQF_SOURCE_ADDR_MASK) ? 1 : 0;
2964                 filter->dst_ip_mask =
2965                         (ftqf & E1000_FTQF_DEST_ADDR_MASK) ? 1 : 0;
2966                 filter->src_port_mask =
2967                         (ftqf & E1000_FTQF_SOURCE_PORT_MASK) ? 1 : 0;
2968                 filter->protocol_mask =
2969                         (ftqf & E1000_FTQF_PROTOCOL_COMP_MASK) ? 1 : 0;
2970                 filter->protocol =
2971                         (uint8_t)ftqf & E1000_FTQF_PROTOCOL_MASK;
2972                 *rx_queue = (uint16_t)((ftqf & E1000_FTQF_QUEUE_MASK) >>
2973                                 E1000_FTQF_QUEUE_SHIFT);
2974
2975                 spqf = E1000_READ_REG(hw, E1000_SPQF(index));
2976                 filter->src_port = spqf & E1000_SPQF_SRCPORT;
2977
2978                 filter->dst_ip = E1000_READ_REG(hw, E1000_DAQF(index));
2979                 filter->src_ip = E1000_READ_REG(hw, E1000_SAQF(index));
2980
2981                 imir = E1000_READ_REG(hw, E1000_IMIR(index));
2982                 filter->dst_port_mask = (imir & E1000_IMIR_PORT_BP) ? 1 : 0;
2983                 filter->dst_port = (uint16_t)(imir & E1000_IMIR_DSTPORT);
2984                 filter->priority = (imir & E1000_IMIR_PRIORITY) >>
2985                         E1000_IMIR_PRIORITY_SHIFT;
2986
2987                 imir_ext = E1000_READ_REG(hw, E1000_IMIREXT(index));
2988                 if (!(imir_ext & E1000_IMIR_EXT_CTRL_BP)) {
2989                         if (imir_ext & E1000_IMIR_EXT_CTRL_UGR)
2990                                 filter->tcp_flags |= TCP_UGR_FLAG;
2991                         if (imir_ext & E1000_IMIR_EXT_CTRL_ACK)
2992                                 filter->tcp_flags |= TCP_ACK_FLAG;
2993                         if (imir_ext & E1000_IMIR_EXT_CTRL_PSH)
2994                                 filter->tcp_flags |= TCP_PSH_FLAG;
2995                         if (imir_ext & E1000_IMIR_EXT_CTRL_RST)
2996                                 filter->tcp_flags |= TCP_RST_FLAG;
2997                         if (imir_ext & E1000_IMIR_EXT_CTRL_SYN)
2998                                 filter->tcp_flags |= TCP_SYN_FLAG;
2999                         if (imir_ext & E1000_IMIR_EXT_CTRL_FIN)
3000                                 filter->tcp_flags |= TCP_FIN_FLAG;
3001                 } else
3002                         filter->tcp_flags = 0;
3003                 return 0;
3004         }
3005         return -ENOENT;
3006 }
3007
3008 static struct rte_driver pmd_igb_drv = {
3009         .type = PMD_PDEV,
3010         .init = rte_igb_pmd_init,
3011 };
3012
3013 static struct rte_driver pmd_igbvf_drv = {
3014         .type = PMD_PDEV,
3015         .init = rte_igbvf_pmd_init,
3016 };
3017
3018 PMD_REGISTER_DRIVER(pmd_igb_drv);
3019 PMD_REGISTER_DRIVER(pmd_igbvf_drv);