lib/librte_pmd_e1000/igb_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <stdarg.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_interrupts.h>
  42 #include <rte_byteorder.h>
  43 #include <rte_log.h>
  44 #include <rte_debug.h>
  45 #include <rte_pci.h>
  46 #include <rte_ether.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_memory.h>
  49 #include <rte_memzone.h>
  50 #include <rte_tailq.h>
  51 #include <rte_eal.h>
  52 #include <rte_atomic.h>
  53 #include <rte_malloc.h>
  54 #include <rte_dev.h>
  55
  56 #include "e1000_logs.h"
  57 #include "e1000/e1000_api.h"
  58 #include "e1000_ethdev.h"
  59
  60 static int  eth_igb_configure(struct rte_eth_dev *dev);
  61 static int  eth_igb_start(struct rte_eth_dev *dev);
  62 static void eth_igb_stop(struct rte_eth_dev *dev);
  63 static void eth_igb_close(struct rte_eth_dev *dev);
  64 static void eth_igb_promiscuous_enable(struct rte_eth_dev *dev);
  65 static void eth_igb_promiscuous_disable(struct rte_eth_dev *dev);
  66 static void eth_igb_allmulticast_enable(struct rte_eth_dev *dev);
  67 static void eth_igb_allmulticast_disable(struct rte_eth_dev *dev);
  68 static int  eth_igb_link_update(struct rte_eth_dev *dev,
  69                                 int wait_to_complete);
  70 static void eth_igb_stats_get(struct rte_eth_dev *dev,
  71                                 struct rte_eth_stats *rte_stats);
  72 static void eth_igb_stats_reset(struct rte_eth_dev *dev);
  73 static void eth_igb_infos_get(struct rte_eth_dev *dev,
  74                                 struct rte_eth_dev_info *dev_info);
  75 static int  eth_igb_flow_ctrl_set(struct rte_eth_dev *dev,
  76                                 struct rte_eth_fc_conf *fc_conf);
  77 static int eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev);
  78 static int eth_igb_interrupt_get_status(struct rte_eth_dev *dev);
  79 static int eth_igb_interrupt_action(struct rte_eth_dev *dev);
  80 static void eth_igb_interrupt_handler(struct rte_intr_handle *handle,
  81                                                         void *param);
  82 static int  igb_hardware_init(struct e1000_hw *hw);
  83 static void igb_hw_control_acquire(struct e1000_hw *hw);
  84 static void igb_hw_control_release(struct e1000_hw *hw);
  85 static void igb_init_manageability(struct e1000_hw *hw);
  86 static void igb_release_manageability(struct e1000_hw *hw);
  87
  88 static int eth_igb_vlan_filter_set(struct rte_eth_dev *dev,
  89                 uint16_t vlan_id, int on);
  90 static void eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid_id);
  91 static void eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask);
  92
  93 static void igb_vlan_hw_filter_enable(struct rte_eth_dev *dev);
  94 static void igb_vlan_hw_filter_disable(struct rte_eth_dev *dev);
  95 static void igb_vlan_hw_strip_enable(struct rte_eth_dev *dev);
  96 static void igb_vlan_hw_strip_disable(struct rte_eth_dev *dev);
  97 static void igb_vlan_hw_extend_enable(struct rte_eth_dev *dev);
  98 static void igb_vlan_hw_extend_disable(struct rte_eth_dev *dev);
  99
 100 static int eth_igb_led_on(struct rte_eth_dev *dev);
 101 static int eth_igb_led_off(struct rte_eth_dev *dev);
 102
 103 static void igb_intr_disable(struct e1000_hw *hw);
 104 static int  igb_get_rx_buffer_size(struct e1000_hw *hw);
 105 static void eth_igb_rar_set(struct rte_eth_dev *dev,
 106                 struct ether_addr *mac_addr,
 107                 uint32_t index, uint32_t pool);
 108 static void eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index);
 109
 110 static void igbvf_intr_disable(struct e1000_hw *hw);
 111 static int igbvf_dev_configure(struct rte_eth_dev *dev);
 112 static int igbvf_dev_start(struct rte_eth_dev *dev);
 113 static void igbvf_dev_stop(struct rte_eth_dev *dev);
 114 static void igbvf_dev_close(struct rte_eth_dev *dev);
 115 static int eth_igbvf_link_update(struct e1000_hw *hw);
 116 static void eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats);
 117 static void eth_igbvf_stats_reset(struct rte_eth_dev *dev);
 118 static int igbvf_vlan_filter_set(struct rte_eth_dev *dev,
 119                 uint16_t vlan_id, int on);
 120 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on);
 121 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on);
 122 static int eth_igb_rss_reta_update(struct rte_eth_dev *dev,
 123                  struct rte_eth_rss_reta *reta_conf);
 124 static int eth_igb_rss_reta_query(struct rte_eth_dev *dev,
 125                 struct rte_eth_rss_reta *reta_conf);
 126
 127 /*
 128  * Define VF Stats MACRO for Non "cleared on read" register
 129  */
 130 #define UPDATE_VF_STAT(reg, last, cur)            \
 131 {                                                 \
 132         u32 latest = E1000_READ_REG(hw, reg);     \
 133         cur += latest - last;                     \
 134         last = latest;                            \
 135 }
 136
 137
 138 #define IGB_FC_PAUSE_TIME 0x0680
 139 #define IGB_LINK_UPDATE_CHECK_TIMEOUT  90  /* 9s */
 140 #define IGB_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */
 141
 142 #define IGBVF_PMD_NAME "rte_igbvf_pmd"     /* PMD name */
 143
 144 static enum e1000_fc_mode igb_fc_setting = e1000_fc_full;
 145
 146 /*
 147  * The set of PCI devices this driver supports
 148  */
 149 static struct rte_pci_id pci_id_igb_map[] = {
 150
 151 #define RTE_PCI_DEV_ID_DECL_IGB(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 152 #include "rte_pci_dev_ids.h"
 153
 154 {.device_id = 0},
 155 };
 156
 157 /*
 158  * The set of PCI devices this driver supports (for 82576&I350 VF)
 159  */
 160 static struct rte_pci_id pci_id_igbvf_map[] = {
 161
 162 #define RTE_PCI_DEV_ID_DECL_IGBVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
 163 #include "rte_pci_dev_ids.h"
 164
 165 {.device_id = 0},
 166 };
 167
 168 static struct eth_dev_ops eth_igb_ops = {
 169         .dev_configure        = eth_igb_configure,
 170         .dev_start            = eth_igb_start,
 171         .dev_stop             = eth_igb_stop,
 172         .dev_close            = eth_igb_close,
 173         .promiscuous_enable   = eth_igb_promiscuous_enable,
 174         .promiscuous_disable  = eth_igb_promiscuous_disable,
 175         .allmulticast_enable  = eth_igb_allmulticast_enable,
 176         .allmulticast_disable = eth_igb_allmulticast_disable,
 177         .link_update          = eth_igb_link_update,
 178         .stats_get            = eth_igb_stats_get,
 179         .stats_reset          = eth_igb_stats_reset,
 180         .dev_infos_get        = eth_igb_infos_get,
 181         .vlan_filter_set      = eth_igb_vlan_filter_set,
 182         .vlan_tpid_set        = eth_igb_vlan_tpid_set,
 183         .vlan_offload_set     = eth_igb_vlan_offload_set,
 184         .rx_queue_setup       = eth_igb_rx_queue_setup,
 185         .rx_queue_release     = eth_igb_rx_queue_release,
 186         .rx_queue_count       = eth_igb_rx_queue_count,
 187         .rx_descriptor_done   = eth_igb_rx_descriptor_done,
 188         .tx_queue_setup       = eth_igb_tx_queue_setup,
 189         .tx_queue_release     = eth_igb_tx_queue_release,
 190         .dev_led_on           = eth_igb_led_on,
 191         .dev_led_off          = eth_igb_led_off,
 192         .flow_ctrl_set        = eth_igb_flow_ctrl_set,
 193         .mac_addr_add         = eth_igb_rar_set,
 194         .mac_addr_remove      = eth_igb_rar_clear,
 195         .reta_update          = eth_igb_rss_reta_update,
 196         .reta_query           = eth_igb_rss_reta_query,
 197         .rss_hash_update      = eth_igb_rss_hash_update,
 198         .rss_hash_conf_get    = eth_igb_rss_hash_conf_get,
 199 };
 200
 201 /*
 202  * dev_ops for virtual function, bare necessities for basic vf
 203  * operation have been implemented
 204  */
 205 static struct eth_dev_ops igbvf_eth_dev_ops = {
 206         .dev_configure        = igbvf_dev_configure,
 207         .dev_start            = igbvf_dev_start,
 208         .dev_stop             = igbvf_dev_stop,
 209         .dev_close            = igbvf_dev_close,
 210         .link_update          = eth_igb_link_update,
 211         .stats_get            = eth_igbvf_stats_get,
 212         .stats_reset          = eth_igbvf_stats_reset,
 213         .vlan_filter_set      = igbvf_vlan_filter_set,
 214         .dev_infos_get        = eth_igb_infos_get,
 215         .rx_queue_setup       = eth_igb_rx_queue_setup,
 216         .rx_queue_release     = eth_igb_rx_queue_release,
 217         .tx_queue_setup       = eth_igb_tx_queue_setup,
 218         .tx_queue_release     = eth_igb_tx_queue_release,
 219 };
 220
 221 /**
 222  * Atomically reads the link status information from global
 223  * structure rte_eth_dev.
 224  *
 225  * @param dev
 226  *   - Pointer to the structure rte_eth_dev to read from.
 227  *   - Pointer to the buffer to be saved with the link status.
 228  *
 229  * @return
 230  *   - On success, zero.
 231  *   - On failure, negative value.
 232  */
 233 static inline int
 234 rte_igb_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 235                                 struct rte_eth_link *link)
 236 {
 237         struct rte_eth_link *dst = link;
 238         struct rte_eth_link *src = &(dev->data->dev_link);
 239
 240         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 241                                         *(uint64_t *)src) == 0)
 242                 return -1;
 243
 244         return 0;
 245 }
 246
 247 /**
 248  * Atomically writes the link status information into global
 249  * structure rte_eth_dev.
 250  *
 251  * @param dev
 252  *   - Pointer to the structure rte_eth_dev to read from.
 253  *   - Pointer to the buffer to be saved with the link status.
 254  *
 255  * @return
 256  *   - On success, zero.
 257  *   - On failure, negative value.
 258  */
 259 static inline int
 260 rte_igb_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 261                                 struct rte_eth_link *link)
 262 {
 263         struct rte_eth_link *dst = &(dev->data->dev_link);
 264         struct rte_eth_link *src = link;
 265
 266         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 267                                         *(uint64_t *)src) == 0)
 268                 return -1;
 269
 270         return 0;
 271 }
 272
 273 static inline void
 274 igb_intr_enable(struct rte_eth_dev *dev)
 275 {
 276         struct e1000_interrupt *intr =
 277                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 278         struct e1000_hw *hw =
 279                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 280
 281         E1000_WRITE_REG(hw, E1000_IMS, intr->mask);
 282         E1000_WRITE_FLUSH(hw);
 283 }
 284
 285 static void
 286 igb_intr_disable(struct e1000_hw *hw)
 287 {
 288         E1000_WRITE_REG(hw, E1000_IMC, ~0);
 289         E1000_WRITE_FLUSH(hw);
 290 }
 291
 292 static inline int32_t
 293 igb_pf_reset_hw(struct e1000_hw *hw)
 294 {
 295         uint32_t ctrl_ext;
 296         int32_t status;
 297
 298         status = e1000_reset_hw(hw);
 299
 300         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 301         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 302         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 303         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 304         E1000_WRITE_FLUSH(hw);
 305
 306         return status;
 307 }
 308
 309 static void
 310 igb_identify_hardware(struct rte_eth_dev *dev)
 311 {
 312         struct e1000_hw *hw =
 313                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 314
 315         hw->vendor_id = dev->pci_dev->id.vendor_id;
 316         hw->device_id = dev->pci_dev->id.device_id;
 317         hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
 318         hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
 319
 320         e1000_set_mac_type(hw);
 321
 322         /* need to check if it is a vf device below */
 323 }
 324
 325 static int
 326 igb_reset_swfw_lock(struct e1000_hw *hw)
 327 {
 328         int ret_val;
 329
 330         /*
 331          * Do mac ops initialization manually here, since we will need
 332          * some function pointers set by this call.
 333          */
 334         ret_val = e1000_init_mac_params(hw);
 335         if (ret_val)
 336                 return ret_val;
 337
 338         /*
 339          * SMBI lock should not fail in this early stage. If this is the case,
 340          * it is due to an improper exit of the application.
 341          * So force the release of the faulty lock.
 342          */
 343         if (e1000_get_hw_semaphore_generic(hw) < 0) {
 344                 DEBUGOUT("SMBI lock released");
 345         }
 346         e1000_put_hw_semaphore_generic(hw);
 347
 348         if (hw->mac.ops.acquire_swfw_sync != NULL) {
 349                 uint16_t mask;
 350
 351                 /*
 352                  * Phy lock should not fail in this early stage. If this is the case,
 353                  * it is due to an improper exit of the application.
 354                  * So force the release of the faulty lock.
 355                  */
 356                 mask = E1000_SWFW_PHY0_SM << hw->bus.func;
 357                 if (hw->bus.func > E1000_FUNC_1)
 358                         mask <<= 2;
 359                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 360                         DEBUGOUT1("SWFW phy%d lock released", hw->bus.func);
 361                 }
 362                 hw->mac.ops.release_swfw_sync(hw, mask);
 363
 364                 /*
 365                  * This one is more tricky since it is common to all ports; but
 366                  * swfw_sync retries last long enough (1s) to be almost sure that if
 367                  * lock can not be taken it is due to an improper lock of the
 368                  * semaphore.
 369                  */
 370                 mask = E1000_SWFW_EEP_SM;
 371                 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
 372                         DEBUGOUT("SWFW common locks released");
 373                 }
 374                 hw->mac.ops.release_swfw_sync(hw, mask);
 375         }
 376
 377         return E1000_SUCCESS;
 378 }
 379
 380 static int
 381 eth_igb_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 382                    struct rte_eth_dev *eth_dev)
 383 {
 384         int error = 0;
 385         struct rte_pci_device *pci_dev;
 386         struct e1000_hw *hw =
 387                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 388         struct e1000_vfta * shadow_vfta =
 389                         E1000_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
 390         uint32_t ctrl_ext;
 391
 392         pci_dev = eth_dev->pci_dev;
 393         eth_dev->dev_ops = &eth_igb_ops;
 394         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 395         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 396
 397         /* for secondary processes, we don't initialise any further as primary
 398          * has already done this work. Only check we don't need a different
 399          * RX function */
 400         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 401                 if (eth_dev->data->scattered_rx)
 402                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 403                 return 0;
 404         }
 405
 406         hw->hw_addr= (void *)pci_dev->mem_resource[0].addr;
 407
 408         igb_identify_hardware(eth_dev);
 409         if (e1000_setup_init_funcs(hw, FALSE) != E1000_SUCCESS) {
 410                 error = -EIO;
 411                 goto err_late;
 412         }
 413
 414         e1000_get_bus_info(hw);
 415
 416         /* Reset any pending lock */
 417         if (igb_reset_swfw_lock(hw) != E1000_SUCCESS) {
 418                 error = -EIO;
 419                 goto err_late;
 420         }
 421
 422         /* Finish initialization */
 423         if (e1000_setup_init_funcs(hw, TRUE) != E1000_SUCCESS) {
 424                 error = -EIO;
 425                 goto err_late;
 426         }
 427
 428         hw->mac.autoneg = 1;
 429         hw->phy.autoneg_wait_to_complete = 0;
 430         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 431
 432         /* Copper options */
 433         if (hw->phy.media_type == e1000_media_type_copper) {
 434                 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
 435                 hw->phy.disable_polarity_correction = 0;
 436                 hw->phy.ms_type = e1000_ms_hw_default;
 437         }
 438
 439         /*
 440          * Start from a known state, this is important in reading the nvm
 441          * and mac from that.
 442          */
 443         igb_pf_reset_hw(hw);
 444
 445         /* Make sure we have a good EEPROM before we read from it */
 446         if (e1000_validate_nvm_checksum(hw) < 0) {
 447                 /*
 448                  * Some PCI-E parts fail the first check due to
 449                  * the link being in sleep state, call it again,
 450                  * if it fails a second time its a real issue.
 451                  */
 452                 if (e1000_validate_nvm_checksum(hw) < 0) {
 453                         PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
 454                         error = -EIO;
 455                         goto err_late;
 456                 }
 457         }
 458
 459         /* Read the permanent MAC address out of the EEPROM */
 460         if (e1000_read_mac_addr(hw) != 0) {
 461                 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
 462                 error = -EIO;
 463                 goto err_late;
 464         }
 465
 466         /* Allocate memory for storing MAC addresses */
 467         eth_dev->data->mac_addrs = rte_zmalloc("e1000",
 468                 ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
 469         if (eth_dev->data->mac_addrs == NULL) {
 470                 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to "
 471                                                 "store MAC addresses",
 472                                 ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 473                 error = -ENOMEM;
 474                 goto err_late;
 475         }
 476
 477         /* Copy the permanent MAC address */
 478         ether_addr_copy((struct ether_addr *)hw->mac.addr, &eth_dev->data->mac_addrs[0]);
 479
 480         /* initialize the vfta */
 481         memset(shadow_vfta, 0, sizeof(*shadow_vfta));
 482
 483         /* Now initialize the hardware */
 484         if (igb_hardware_init(hw) != 0) {
 485                 PMD_INIT_LOG(ERR, "Hardware initialization failed");
 486                 rte_free(eth_dev->data->mac_addrs);
 487                 eth_dev->data->mac_addrs = NULL;
 488                 error = -ENODEV;
 489                 goto err_late;
 490         }
 491         hw->mac.get_link_status = 1;
 492
 493         /* Indicate SOL/IDER usage */
 494         if (e1000_check_reset_block(hw) < 0) {
 495                 PMD_INIT_LOG(ERR, "PHY reset is blocked due to"
 496                                         "SOL/IDER session");
 497         }
 498
 499         /* initialize PF if max_vfs not zero */
 500         igb_pf_host_init(eth_dev);
 501
 502         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 503         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 504         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 505         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 506         E1000_WRITE_FLUSH(hw);
 507
 508         PMD_INIT_LOG(INFO, "port_id %d vendorID=0x%x deviceID=0x%x\n",
 509                      eth_dev->data->port_id, pci_dev->id.vendor_id,
 510                      pci_dev->id.device_id);
 511
 512         rte_intr_callback_register(&(pci_dev->intr_handle),
 513                 eth_igb_interrupt_handler, (void *)eth_dev);
 514
 515         /* enable uio intr after callback register */
 516         rte_intr_enable(&(pci_dev->intr_handle));
 517
 518         /* enable support intr */
 519         igb_intr_enable(eth_dev);
 520
 521         return 0;
 522
 523 err_late:
 524         igb_hw_control_release(hw);
 525
 526         return (error);
 527 }
 528
 529 /*
 530  * Virtual Function device init
 531  */
 532 static int
 533 eth_igbvf_dev_init(__attribute__((unused)) struct eth_driver *eth_drv,
 534                 struct rte_eth_dev *eth_dev)
 535 {
 536         struct rte_pci_device *pci_dev;
 537         struct e1000_hw *hw =
 538                 E1000_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 539         int diag;
 540
 541         PMD_INIT_LOG(DEBUG, "eth_igbvf_dev_init");
 542
 543         eth_dev->dev_ops = &igbvf_eth_dev_ops;
 544         eth_dev->rx_pkt_burst = &eth_igb_recv_pkts;
 545         eth_dev->tx_pkt_burst = &eth_igb_xmit_pkts;
 546
 547         /* for secondary processes, we don't initialise any further as primary
 548          * has already done this work. Only check we don't need a different
 549          * RX function */
 550         if (rte_eal_process_type() != RTE_PROC_PRIMARY){
 551                 if (eth_dev->data->scattered_rx)
 552                         eth_dev->rx_pkt_burst = &eth_igb_recv_scattered_pkts;
 553                 return 0;
 554         }
 555
 556         pci_dev = eth_dev->pci_dev;
 557
 558         hw->device_id = pci_dev->id.device_id;
 559         hw->vendor_id = pci_dev->id.vendor_id;
 560         hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
 561
 562         /* Initialize the shared code */
 563         diag = e1000_setup_init_funcs(hw, TRUE);
 564         if (diag != 0) {
 565                 PMD_INIT_LOG(ERR, "Shared code init failed for igbvf: %d",
 566                         diag);
 567                 return -EIO;
 568         }
 569
 570         /* init_mailbox_params */
 571         hw->mbx.ops.init_params(hw);
 572
 573         /* Disable the interrupts for VF */
 574         igbvf_intr_disable(hw);
 575
 576         diag = hw->mac.ops.reset_hw(hw);
 577
 578         /* Allocate memory for storing MAC addresses */
 579         eth_dev->data->mac_addrs = rte_zmalloc("igbvf", ETHER_ADDR_LEN *
 580                 hw->mac.rar_entry_count, 0);
 581         if (eth_dev->data->mac_addrs == NULL) {
 582                 PMD_INIT_LOG(ERR,
 583                         "Failed to allocate %d bytes needed to store MAC "
 584                         "addresses",
 585                         ETHER_ADDR_LEN * hw->mac.rar_entry_count);
 586                 return -ENOMEM;
 587         }
 588
 589         /* Copy the permanent MAC address */
 590         ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
 591                         &eth_dev->data->mac_addrs[0]);
 592
 593         PMD_INIT_LOG(DEBUG, "\nport %d vendorID=0x%x deviceID=0x%x "
 594                         "mac.type=%s\n",
 595                         eth_dev->data->port_id, pci_dev->id.vendor_id,
 596                         pci_dev->id.device_id,
 597                         "igb_mac_82576_vf");
 598
 599         return 0;
 600 }
 601
 602 static struct eth_driver rte_igb_pmd = {
 603         {
 604                 .name = "rte_igb_pmd",
 605                 .id_table = pci_id_igb_map,
 606                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 607         },
 608         .eth_dev_init = eth_igb_dev_init,
 609         .dev_private_size = sizeof(struct e1000_adapter),
 610 };
 611
 612 /*
 613  * virtual function driver struct
 614  */
 615 static struct eth_driver rte_igbvf_pmd = {
 616         {
 617                 .name = "rte_igbvf_pmd",
 618                 .id_table = pci_id_igbvf_map,
 619                 .drv_flags = RTE_PCI_DRV_NEED_IGB_UIO,
 620         },
 621         .eth_dev_init = eth_igbvf_dev_init,
 622         .dev_private_size = sizeof(struct e1000_adapter),
 623 };
 624
 625 static int
 626 rte_igb_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 627 {
 628         rte_eth_driver_register(&rte_igb_pmd);
 629         return 0;
 630 }
 631
 632 static void
 633 igb_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
 634 {
 635         struct e1000_hw *hw =
 636                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 637         /* RCTL: enable VLAN filter since VMDq always use VLAN filter */
 638         uint32_t rctl = E1000_READ_REG(hw, E1000_RCTL);
 639         rctl |= E1000_RCTL_VFE;
 640         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
 641 }
 642
 643 /*
 644  * VF Driver initialization routine.
 645  * Invoked one at EAL init time.
 646  * Register itself as the [Virtual Poll Mode] Driver of PCI IGB devices.
 647  */
 648 static int
 649 rte_igbvf_pmd_init(const char *name __rte_unused, const char *params __rte_unused)
 650 {
 651         DEBUGFUNC("rte_igbvf_pmd_init");
 652
 653         rte_eth_driver_register(&rte_igbvf_pmd);
 654         return (0);
 655 }
 656
 657 static int
 658 eth_igb_configure(struct rte_eth_dev *dev)
 659 {
 660         struct e1000_interrupt *intr =
 661                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
 662
 663         PMD_INIT_LOG(DEBUG, ">>");
 664
 665         intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
 666
 667         PMD_INIT_LOG(DEBUG, "<<");
 668
 669         return (0);
 670 }
 671
 672 static int
 673 eth_igb_start(struct rte_eth_dev *dev)
 674 {
 675         struct e1000_hw *hw =
 676                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 677         int ret, i, mask;
 678         uint32_t ctrl_ext;
 679
 680         PMD_INIT_LOG(DEBUG, ">>");
 681
 682         /* Power up the phy. Needed to make the link go Up */
 683         e1000_power_up_phy(hw);
 684
 685         /*
 686          * Packet Buffer Allocation (PBA)
 687          * Writing PBA sets the receive portion of the buffer
 688          * the remainder is used for the transmit buffer.
 689          */
 690         if (hw->mac.type == e1000_82575) {
 691                 uint32_t pba;
 692
 693                 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
 694                 E1000_WRITE_REG(hw, E1000_PBA, pba);
 695         }
 696
 697         /* Put the address into the Receive Address Array */
 698         e1000_rar_set(hw, hw->mac.addr, 0);
 699
 700         /* Initialize the hardware */
 701         if (igb_hardware_init(hw)) {
 702                 PMD_INIT_LOG(ERR, "Unable to initialize the hardware");
 703                 return (-EIO);
 704         }
 705
 706         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
 707
 708         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
 709         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
 710         ctrl_ext |= E1000_CTRL_EXT_PFRSTD;
 711         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
 712         E1000_WRITE_FLUSH(hw);
 713
 714         /* configure PF module if SRIOV enabled */
 715         igb_pf_host_configure(dev);
 716
 717         /* Configure for OS presence */
 718         igb_init_manageability(hw);
 719
 720         eth_igb_tx_init(dev);
 721
 722         /* This can fail when allocating mbufs for descriptor rings */
 723         ret = eth_igb_rx_init(dev);
 724         if (ret) {
 725                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
 726                 igb_dev_clear_queues(dev);
 727                 return ret;
 728         }
 729
 730         e1000_clear_hw_cntrs_base_generic(hw);
 731
 732         /*
 733          * VLAN Offload Settings
 734          */
 735         mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | \
 736                         ETH_VLAN_EXTEND_MASK;
 737         eth_igb_vlan_offload_set(dev, mask);
 738
 739         if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
 740                 /* Enable VLAN filter since VMDq always use VLAN filter */
 741                 igb_vmdq_vlan_hw_filter_enable(dev);
 742         }
 743
 744         /*
 745          * Configure the Interrupt Moderation register (EITR) with the maximum
 746          * possible value (0xFFFF) to minimize "System Partial Write" issued by
 747          * spurious [DMA] memory updates of RX and TX ring descriptors.
 748          *
 749          * With a EITR granularity of 2 microseconds in the 82576, only 7/8
 750          * spurious memory updates per second should be expected.
 751          * ((65535 * 2) / 1000.1000 ~= 0.131 second).
 752          *
 753          * Because interrupts are not used at all, the MSI-X is not activated
 754          * and interrupt moderation is controlled by EITR[0].
 755          *
 756          * Note that having [almost] disabled memory updates of RX and TX ring
 757          * descriptors through the Interrupt Moderation mechanism, memory
 758          * updates of ring descriptors are now moderated by the configurable
 759          * value of Write-Back Threshold registers.
 760          */
 761         if ((hw->mac.type == e1000_82576) || (hw->mac.type == e1000_82580) ||
 762                 (hw->mac.type == e1000_i350) || (hw->mac.type == e1000_i210)) {
 763                 uint32_t ivar;
 764
 765                 /* Enable all RX & TX queues in the IVAR registers */
 766                 ivar = (uint32_t) ((E1000_IVAR_VALID << 16) | E1000_IVAR_VALID);
 767                 for (i = 0; i < 8; i++)
 768                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, i, ivar);
 769
 770                 /* Configure EITR with the maximum possible value (0xFFFF) */
 771                 E1000_WRITE_REG(hw, E1000_EITR(0), 0xFFFF);
 772         }
 773
 774         /* Setup link speed and duplex */
 775         switch (dev->data->dev_conf.link_speed) {
 776         case ETH_LINK_SPEED_AUTONEG:
 777                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
 778                         hw->phy.autoneg_advertised = E1000_ALL_SPEED_DUPLEX;
 779                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
 780                         hw->phy.autoneg_advertised = E1000_ALL_HALF_DUPLEX;
 781                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
 782                         hw->phy.autoneg_advertised = E1000_ALL_FULL_DUPLEX;
 783                 else
 784                         goto error_invalid_config;
 785                 break;
 786         case ETH_LINK_SPEED_10:
 787                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
 788                         hw->phy.autoneg_advertised = E1000_ALL_10_SPEED;
 789                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
 790                         hw->phy.autoneg_advertised = ADVERTISE_10_HALF;
 791                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
 792                         hw->phy.autoneg_advertised = ADVERTISE_10_FULL;
 793                 else
 794                         goto error_invalid_config;
 795                 break;
 796         case ETH_LINK_SPEED_100:
 797                 if (dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX)
 798                         hw->phy.autoneg_advertised = E1000_ALL_100_SPEED;
 799                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_HALF_DUPLEX)
 800                         hw->phy.autoneg_advertised = ADVERTISE_100_HALF;
 801                 else if (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX)
 802                         hw->phy.autoneg_advertised = ADVERTISE_100_FULL;
 803                 else
 804                         goto error_invalid_config;
 805                 break;
 806         case ETH_LINK_SPEED_1000:
 807                 if ((dev->data->dev_conf.link_duplex == ETH_LINK_AUTONEG_DUPLEX) ||
 808                                 (dev->data->dev_conf.link_duplex == ETH_LINK_FULL_DUPLEX))
 809                         hw->phy.autoneg_advertised = ADVERTISE_1000_FULL;
 810                 else
 811                         goto error_invalid_config;
 812                 break;
 813         case ETH_LINK_SPEED_10000:
 814         default:
 815                 goto error_invalid_config;
 816         }
 817         e1000_setup_link(hw);
 818
 819         /* check if lsc interrupt feature is enabled */
 820         if (dev->data->dev_conf.intr_conf.lsc != 0)
 821                 ret = eth_igb_lsc_interrupt_setup(dev);
 822
 823         /* resume enabled intr since hw reset */
 824         igb_intr_enable(dev);
 825
 826         PMD_INIT_LOG(DEBUG, "<<");
 827
 828         return (0);
 829
 830 error_invalid_config:
 831         PMD_INIT_LOG(ERR, "Invalid link_speed/link_duplex (%u/%u) for port %u\n",
 832                         dev->data->dev_conf.link_speed,
 833                         dev->data->dev_conf.link_duplex, dev->data->port_id);
 834         igb_dev_clear_queues(dev);
 835         return (-EINVAL);
 836 }
 837
 838 /*********************************************************************
 839  *
 840  *  This routine disables all traffic on the adapter by issuing a
 841  *  global reset on the MAC.
 842  *
 843  **********************************************************************/
 844 static void
 845 eth_igb_stop(struct rte_eth_dev *dev)
 846 {
 847         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 848         struct rte_eth_link link;
 849
 850         igb_intr_disable(hw);
 851         igb_pf_reset_hw(hw);
 852         E1000_WRITE_REG(hw, E1000_WUC, 0);
 853
 854         /* Set bit for Go Link disconnect */
 855         if (hw->mac.type >= e1000_82580) {
 856                 uint32_t phpm_reg;
 857
 858                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
 859                 phpm_reg |= E1000_82580_PM_GO_LINKD;
 860                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
 861         }
 862
 863         /* Power down the phy. Needed to make the link go Down */
 864         e1000_power_down_phy(hw);
 865
 866         igb_dev_clear_queues(dev);
 867
 868         /* clear the recorded link status */
 869         memset(&link, 0, sizeof(link));
 870         rte_igb_dev_atomic_write_link_status(dev, &link);
 871 }
 872
 873 static void
 874 eth_igb_close(struct rte_eth_dev *dev)
 875 {
 876         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 877         struct rte_eth_link link;
 878
 879         eth_igb_stop(dev);
 880         e1000_phy_hw_reset(hw);
 881         igb_release_manageability(hw);
 882         igb_hw_control_release(hw);
 883
 884         /* Clear bit for Go Link disconnect */
 885         if (hw->mac.type >= e1000_82580) {
 886                 uint32_t phpm_reg;
 887
 888                 phpm_reg = E1000_READ_REG(hw, E1000_82580_PHY_POWER_MGMT);
 889                 phpm_reg &= ~E1000_82580_PM_GO_LINKD;
 890                 E1000_WRITE_REG(hw, E1000_82580_PHY_POWER_MGMT, phpm_reg);
 891         }
 892
 893         igb_dev_clear_queues(dev);
 894
 895         memset(&link, 0, sizeof(link));
 896         rte_igb_dev_atomic_write_link_status(dev, &link);
 897 }
 898
 899 static int
 900 igb_get_rx_buffer_size(struct e1000_hw *hw)
 901 {
 902         uint32_t rx_buf_size;
 903         if (hw->mac.type == e1000_82576) {
 904                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xffff) << 10;
 905         } else if (hw->mac.type == e1000_82580 || hw->mac.type == e1000_i350) {
 906                 /* PBS needs to be translated according to a lookup table */
 907                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0xf);
 908                 rx_buf_size = (uint32_t) e1000_rxpbs_adjust_82580(rx_buf_size);
 909                 rx_buf_size = (rx_buf_size << 10);
 910         } else if (hw->mac.type == e1000_i210) {
 911                 rx_buf_size = (E1000_READ_REG(hw, E1000_RXPBS) & 0x3f) << 10;
 912         } else {
 913                 rx_buf_size = (E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10;
 914         }
 915
 916         return rx_buf_size;
 917 }
 918
 919 /*********************************************************************
 920  *
 921  *  Initialize the hardware
 922  *
 923  **********************************************************************/
 924 static int
 925 igb_hardware_init(struct e1000_hw *hw)
 926 {
 927         uint32_t rx_buf_size;
 928         int diag;
 929
 930         /* Let the firmware know the OS is in control */
 931         igb_hw_control_acquire(hw);
 932
 933         /*
 934          * These parameters control the automatic generation (Tx) and
 935          * response (Rx) to Ethernet PAUSE frames.
 936          * - High water mark should allow for at least two standard size (1518)
 937          *   frames to be received after sending an XOFF.
 938          * - Low water mark works best when it is very near the high water mark.
 939          *   This allows the receiver to restart by sending XON when it has
 940          *   drained a bit. Here we use an arbitrary value of 1500 which will
 941          *   restart after one full frame is pulled from the buffer. There
 942          *   could be several smaller frames in the buffer and if so they will
 943          *   not trigger the XON until their total number reduces the buffer
 944          *   by 1500.
 945          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
 946          */
 947         rx_buf_size = igb_get_rx_buffer_size(hw);
 948
 949         hw->fc.high_water = rx_buf_size - (ETHER_MAX_LEN * 2);
 950         hw->fc.low_water = hw->fc.high_water - 1500;
 951         hw->fc.pause_time = IGB_FC_PAUSE_TIME;
 952         hw->fc.send_xon = 1;
 953
 954         /* Set Flow control, use the tunable location if sane */
 955         if ((igb_fc_setting != e1000_fc_none) && (igb_fc_setting < 4))
 956                 hw->fc.requested_mode = igb_fc_setting;
 957         else
 958                 hw->fc.requested_mode = e1000_fc_none;
 959
 960         /* Issue a global reset */
 961         igb_pf_reset_hw(hw);
 962         E1000_WRITE_REG(hw, E1000_WUC, 0);
 963
 964         diag = e1000_init_hw(hw);
 965         if (diag < 0)
 966                 return (diag);
 967
 968         E1000_WRITE_REG(hw, E1000_VET, ETHER_TYPE_VLAN << 16 | ETHER_TYPE_VLAN);
 969         e1000_get_phy_info(hw);
 970         e1000_check_for_link(hw);
 971
 972         return (0);
 973 }
 974
 975 /* This function is based on igb_update_stats_counters() in igb/if_igb.c */
 976 static void
 977 eth_igb_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
 978 {
 979         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 980         struct e1000_hw_stats *stats =
 981                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
 982         int pause_frames;
 983
 984         if(hw->phy.media_type == e1000_media_type_copper ||
 985             (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) {
 986                 stats->symerrs +=
 987                     E1000_READ_REG(hw,E1000_SYMERRS);
 988                 stats->sec += E1000_READ_REG(hw, E1000_SEC);
 989         }
 990
 991         stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
 992         stats->mpc += E1000_READ_REG(hw, E1000_MPC);
 993         stats->scc += E1000_READ_REG(hw, E1000_SCC);
 994         stats->ecol += E1000_READ_REG(hw, E1000_ECOL);
 995
 996         stats->mcc += E1000_READ_REG(hw, E1000_MCC);
 997         stats->latecol += E1000_READ_REG(hw, E1000_LATECOL);
 998         stats->colc += E1000_READ_REG(hw, E1000_COLC);
 999         stats->dc += E1000_READ_REG(hw, E1000_DC);
1000         stats->rlec += E1000_READ_REG(hw, E1000_RLEC);
1001         stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
1002         stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC);
1003         /*
1004         ** For watchdog management we need to know if we have been
1005         ** paused during the last interval, so capture that here.
1006         */
1007         pause_frames = E1000_READ_REG(hw, E1000_XOFFRXC);
1008         stats->xoffrxc += pause_frames;
1009         stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
1010         stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC);
1011         stats->prc64 += E1000_READ_REG(hw, E1000_PRC64);
1012         stats->prc127 += E1000_READ_REG(hw, E1000_PRC127);
1013         stats->prc255 += E1000_READ_REG(hw, E1000_PRC255);
1014         stats->prc511 += E1000_READ_REG(hw, E1000_PRC511);
1015         stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
1016         stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
1017         stats->gprc += E1000_READ_REG(hw, E1000_GPRC);
1018         stats->bprc += E1000_READ_REG(hw, E1000_BPRC);
1019         stats->mprc += E1000_READ_REG(hw, E1000_MPRC);
1020         stats->gptc += E1000_READ_REG(hw, E1000_GPTC);
1021
1022         /* For the 64-bit byte counters the low dword must be read first. */
1023         /* Both registers clear on the read of the high dword */
1024
1025         stats->gorc += E1000_READ_REG(hw, E1000_GORCL);
1026         stats->gorc += ((uint64_t)E1000_READ_REG(hw, E1000_GORCH) << 32);
1027         stats->gotc += E1000_READ_REG(hw, E1000_GOTCL);
1028         stats->gotc += ((uint64_t)E1000_READ_REG(hw, E1000_GOTCH) << 32);
1029
1030         stats->rnbc += E1000_READ_REG(hw, E1000_RNBC);
1031         stats->ruc += E1000_READ_REG(hw, E1000_RUC);
1032         stats->rfc += E1000_READ_REG(hw, E1000_RFC);
1033         stats->roc += E1000_READ_REG(hw, E1000_ROC);
1034         stats->rjc += E1000_READ_REG(hw, E1000_RJC);
1035
1036         stats->tor += E1000_READ_REG(hw, E1000_TORH);
1037         stats->tot += E1000_READ_REG(hw, E1000_TOTH);
1038
1039         stats->tpr += E1000_READ_REG(hw, E1000_TPR);
1040         stats->tpt += E1000_READ_REG(hw, E1000_TPT);
1041         stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64);
1042         stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127);
1043         stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255);
1044         stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511);
1045         stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
1046         stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
1047         stats->mptc += E1000_READ_REG(hw, E1000_MPTC);
1048         stats->bptc += E1000_READ_REG(hw, E1000_BPTC);
1049
1050         /* Interrupt Counts */
1051
1052         stats->iac += E1000_READ_REG(hw, E1000_IAC);
1053         stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
1054         stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
1055         stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
1056         stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
1057         stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
1058         stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
1059         stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
1060         stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
1061
1062         /* Host to Card Statistics */
1063
1064         stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC);
1065         stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC);
1066         stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC);
1067         stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC);
1068         stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC);
1069         stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC);
1070         stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC);
1071         stats->hgorc += E1000_READ_REG(hw, E1000_HGORCL);
1072         stats->hgorc += ((uint64_t)E1000_READ_REG(hw, E1000_HGORCH) << 32);
1073         stats->hgotc += E1000_READ_REG(hw, E1000_HGOTCL);
1074         stats->hgotc += ((uint64_t)E1000_READ_REG(hw, E1000_HGOTCH) << 32);
1075         stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS);
1076         stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC);
1077         stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC);
1078
1079         stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
1080         stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
1081         stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS);
1082         stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR);
1083         stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC);
1084         stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
1085
1086         if (rte_stats == NULL)
1087                 return;
1088
1089         /* Rx Errors */
1090         rte_stats->ierrors = stats->rxerrc + stats->crcerrs + stats->algnerrc +
1091             stats->ruc + stats->roc + stats->mpc + stats->cexterr;
1092
1093         /* Tx Errors */
1094         rte_stats->oerrors = stats->ecol + stats->latecol;
1095
1096         /* XON/XOFF pause frames */
1097         rte_stats->tx_pause_xon  = stats->xontxc;
1098         rte_stats->rx_pause_xon  = stats->xonrxc;
1099         rte_stats->tx_pause_xoff = stats->xofftxc;
1100         rte_stats->rx_pause_xoff = stats->xoffrxc;
1101
1102         rte_stats->ipackets = stats->gprc;
1103         rte_stats->opackets = stats->gptc;
1104         rte_stats->ibytes   = stats->gorc;
1105         rte_stats->obytes   = stats->gotc;
1106 }
1107
1108 static void
1109 eth_igb_stats_reset(struct rte_eth_dev *dev)
1110 {
1111         struct e1000_hw_stats *hw_stats =
1112                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1113
1114         /* HW registers are cleared on read */
1115         eth_igb_stats_get(dev, NULL);
1116
1117         /* Reset software totals */
1118         memset(hw_stats, 0, sizeof(*hw_stats));
1119 }
1120
1121 static void
1122 eth_igbvf_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1123 {
1124         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1125         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1126                           E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1127
1128         /* Good Rx packets, include VF loopback */
1129         UPDATE_VF_STAT(E1000_VFGPRC,
1130             hw_stats->last_gprc, hw_stats->gprc);
1131
1132         /* Good Rx octets, include VF loopback */
1133         UPDATE_VF_STAT(E1000_VFGORC,
1134             hw_stats->last_gorc, hw_stats->gorc);
1135
1136         /* Good Tx packets, include VF loopback */
1137         UPDATE_VF_STAT(E1000_VFGPTC,
1138             hw_stats->last_gptc, hw_stats->gptc);
1139
1140         /* Good Tx octets, include VF loopback */
1141         UPDATE_VF_STAT(E1000_VFGOTC,
1142             hw_stats->last_gotc, hw_stats->gotc);
1143
1144         /* Rx Multicst packets */
1145         UPDATE_VF_STAT(E1000_VFMPRC,
1146             hw_stats->last_mprc, hw_stats->mprc);
1147
1148         /* Good Rx loopback packets */
1149         UPDATE_VF_STAT(E1000_VFGPRLBC,
1150             hw_stats->last_gprlbc, hw_stats->gprlbc);
1151
1152         /* Good Rx loopback octets */
1153         UPDATE_VF_STAT(E1000_VFGORLBC,
1154             hw_stats->last_gorlbc, hw_stats->gorlbc);
1155
1156         /* Good Tx loopback packets */
1157         UPDATE_VF_STAT(E1000_VFGPTLBC,
1158             hw_stats->last_gptlbc, hw_stats->gptlbc);
1159
1160         /* Good Tx loopback octets */
1161         UPDATE_VF_STAT(E1000_VFGOTLBC,
1162             hw_stats->last_gotlbc, hw_stats->gotlbc);
1163
1164         if (rte_stats == NULL)
1165                 return;
1166
1167         memset(rte_stats, 0, sizeof(*rte_stats));
1168         rte_stats->ipackets = hw_stats->gprc;
1169         rte_stats->ibytes = hw_stats->gorc;
1170         rte_stats->opackets = hw_stats->gptc;
1171         rte_stats->obytes = hw_stats->gotc;
1172         rte_stats->imcasts = hw_stats->mprc;
1173         rte_stats->ilbpackets = hw_stats->gprlbc;
1174         rte_stats->ilbbytes = hw_stats->gorlbc;
1175         rte_stats->olbpackets = hw_stats->gptlbc;
1176         rte_stats->olbbytes = hw_stats->gotlbc;
1177
1178 }
1179
1180 static void
1181 eth_igbvf_stats_reset(struct rte_eth_dev *dev)
1182 {
1183         struct e1000_vf_stats *hw_stats = (struct e1000_vf_stats*)
1184                         E1000_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1185
1186         /* Sync HW register to the last stats */
1187         eth_igbvf_stats_get(dev, NULL);
1188
1189         /* reset HW current stats*/
1190         memset(&hw_stats->gprc, 0, sizeof(*hw_stats) -
1191                offsetof(struct e1000_vf_stats, gprc));
1192
1193 }
1194
1195 static void
1196 eth_igb_infos_get(struct rte_eth_dev *dev,
1197                     struct rte_eth_dev_info *dev_info)
1198 {
1199         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1200
1201         dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1202         dev_info->max_rx_pktlen  = 0x3FFF; /* See RLPML register. */
1203         dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1204         dev_info->rx_offload_capa =
1205                 DEV_RX_OFFLOAD_VLAN_STRIP |
1206                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1207                 DEV_RX_OFFLOAD_UDP_CKSUM  |
1208                 DEV_RX_OFFLOAD_TCP_CKSUM;
1209         dev_info->tx_offload_capa =
1210                 DEV_TX_OFFLOAD_VLAN_INSERT |
1211                 DEV_TX_OFFLOAD_IPV4_CKSUM  |
1212                 DEV_TX_OFFLOAD_UDP_CKSUM   |
1213                 DEV_TX_OFFLOAD_TCP_CKSUM   |
1214                 DEV_TX_OFFLOAD_SCTP_CKSUM;
1215
1216         switch (hw->mac.type) {
1217         case e1000_82575:
1218                 dev_info->max_rx_queues = 4;
1219                 dev_info->max_tx_queues = 4;
1220                 dev_info->max_vmdq_pools = 0;
1221                 break;
1222
1223         case e1000_82576:
1224                 dev_info->max_rx_queues = 16;
1225                 dev_info->max_tx_queues = 16;
1226                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1227                 break;
1228
1229         case e1000_82580:
1230                 dev_info->max_rx_queues = 8;
1231                 dev_info->max_tx_queues = 8;
1232                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1233                 break;
1234
1235         case e1000_i350:
1236                 dev_info->max_rx_queues = 8;
1237                 dev_info->max_tx_queues = 8;
1238                 dev_info->max_vmdq_pools = ETH_8_POOLS;
1239                 break;
1240
1241         case e1000_i354:
1242                 dev_info->max_rx_queues = 8;
1243                 dev_info->max_tx_queues = 8;
1244                 break;
1245
1246         case e1000_i210:
1247                 dev_info->max_rx_queues = 4;
1248                 dev_info->max_tx_queues = 4;
1249                 dev_info->max_vmdq_pools = 0;
1250                 break;
1251
1252         case e1000_vfadapt:
1253                 dev_info->max_rx_queues = 2;
1254                 dev_info->max_tx_queues = 2;
1255                 dev_info->max_vmdq_pools = 0;
1256                 break;
1257
1258         case e1000_vfadapt_i350:
1259                 dev_info->max_rx_queues = 1;
1260                 dev_info->max_tx_queues = 1;
1261                 dev_info->max_vmdq_pools = 0;
1262                 break;
1263
1264         default:
1265                 /* Should not happen */
1266                 dev_info->max_rx_queues = 0;
1267                 dev_info->max_tx_queues = 0;
1268                 dev_info->max_vmdq_pools = 0;
1269         }
1270 }
1271
1272 /* return 0 means link status changed, -1 means not changed */
1273 static int
1274 eth_igb_link_update(struct rte_eth_dev *dev, int wait_to_complete)
1275 {
1276         struct e1000_hw *hw =
1277                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1278         struct rte_eth_link link, old;
1279         int link_check, count;
1280
1281         link_check = 0;
1282         hw->mac.get_link_status = 1;
1283
1284         /* possible wait-to-complete in up to 9 seconds */
1285         for (count = 0; count < IGB_LINK_UPDATE_CHECK_TIMEOUT; count ++) {
1286                 /* Read the real link status */
1287                 switch (hw->phy.media_type) {
1288                 case e1000_media_type_copper:
1289                         /* Do the work to read phy */
1290                         e1000_check_for_link(hw);
1291                         link_check = !hw->mac.get_link_status;
1292                         break;
1293
1294                 case e1000_media_type_fiber:
1295                         e1000_check_for_link(hw);
1296                         link_check = (E1000_READ_REG(hw, E1000_STATUS) &
1297                                       E1000_STATUS_LU);
1298                         break;
1299
1300                 case e1000_media_type_internal_serdes:
1301                         e1000_check_for_link(hw);
1302                         link_check = hw->mac.serdes_has_link;
1303                         break;
1304
1305                 /* VF device is type_unknown */
1306                 case e1000_media_type_unknown:
1307                         eth_igbvf_link_update(hw);
1308                         link_check = !hw->mac.get_link_status;
1309                         break;
1310
1311                 default:
1312                         break;
1313                 }
1314                 if (link_check || wait_to_complete == 0)
1315                         break;
1316                 rte_delay_ms(IGB_LINK_UPDATE_CHECK_INTERVAL);
1317         }
1318         memset(&link, 0, sizeof(link));
1319         rte_igb_dev_atomic_read_link_status(dev, &link);
1320         old = link;
1321
1322         /* Now we check if a transition has happened */
1323         if (link_check) {
1324                 hw->mac.ops.get_link_up_info(hw, &link.link_speed,
1325                                           &link.link_duplex);
1326                 link.link_status = 1;
1327         } else if (!link_check) {
1328                 link.link_speed = 0;
1329                 link.link_duplex = 0;
1330                 link.link_status = 0;
1331         }
1332         rte_igb_dev_atomic_write_link_status(dev, &link);
1333
1334         /* not changed */
1335         if (old.link_status == link.link_status)
1336                 return -1;
1337
1338         /* changed */
1339         return 0;
1340 }
1341
1342 /*
1343  * igb_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
1344  * For ASF and Pass Through versions of f/w this means
1345  * that the driver is loaded.
1346  */
1347 static void
1348 igb_hw_control_acquire(struct e1000_hw *hw)
1349 {
1350         uint32_t ctrl_ext;
1351
1352         /* Let firmware know the driver has taken over */
1353         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1354         E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1355 }
1356
1357 /*
1358  * igb_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
1359  * For ASF and Pass Through versions of f/w this means that the
1360  * driver is no longer loaded.
1361  */
1362 static void
1363 igb_hw_control_release(struct e1000_hw *hw)
1364 {
1365         uint32_t ctrl_ext;
1366
1367         /* Let firmware taken over control of h/w */
1368         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1369         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1370                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1371 }
1372
1373 /*
1374  * Bit of a misnomer, what this really means is
1375  * to enable OS management of the system... aka
1376  * to disable special hardware management features.
1377  */
1378 static void
1379 igb_init_manageability(struct e1000_hw *hw)
1380 {
1381         if (e1000_enable_mng_pass_thru(hw)) {
1382                 uint32_t manc2h = E1000_READ_REG(hw, E1000_MANC2H);
1383                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
1384
1385                 /* disable hardware interception of ARP */
1386                 manc &= ~(E1000_MANC_ARP_EN);
1387
1388                 /* enable receiving management packets to the host */
1389                 manc |= E1000_MANC_EN_MNG2HOST;
1390                 manc2h |= 1 << 5;  /* Mng Port 623 */
1391                 manc2h |= 1 << 6;  /* Mng Port 664 */
1392                 E1000_WRITE_REG(hw, E1000_MANC2H, manc2h);
1393                 E1000_WRITE_REG(hw, E1000_MANC, manc);
1394         }
1395 }
1396
1397 static void
1398 igb_release_manageability(struct e1000_hw *hw)
1399 {
1400         if (e1000_enable_mng_pass_thru(hw)) {
1401                 uint32_t manc = E1000_READ_REG(hw, E1000_MANC);
1402
1403                 manc |= E1000_MANC_ARP_EN;
1404                 manc &= ~E1000_MANC_EN_MNG2HOST;
1405
1406                 E1000_WRITE_REG(hw, E1000_MANC, manc);
1407         }
1408 }
1409
1410 static void
1411 eth_igb_promiscuous_enable(struct rte_eth_dev *dev)
1412 {
1413         struct e1000_hw *hw =
1414                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1415         uint32_t rctl;
1416
1417         rctl = E1000_READ_REG(hw, E1000_RCTL);
1418         rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1419         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1420 }
1421
1422 static void
1423 eth_igb_promiscuous_disable(struct rte_eth_dev *dev)
1424 {
1425         struct e1000_hw *hw =
1426                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1427         uint32_t rctl;
1428
1429         rctl = E1000_READ_REG(hw, E1000_RCTL);
1430         rctl &= (~E1000_RCTL_UPE);
1431         if (dev->data->all_multicast == 1)
1432                 rctl |= E1000_RCTL_MPE;
1433         else
1434                 rctl &= (~E1000_RCTL_MPE);
1435         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1436 }
1437
1438 static void
1439 eth_igb_allmulticast_enable(struct rte_eth_dev *dev)
1440 {
1441         struct e1000_hw *hw =
1442                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1443         uint32_t rctl;
1444
1445         rctl = E1000_READ_REG(hw, E1000_RCTL);
1446         rctl |= E1000_RCTL_MPE;
1447         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1448 }
1449
1450 static void
1451 eth_igb_allmulticast_disable(struct rte_eth_dev *dev)
1452 {
1453         struct e1000_hw *hw =
1454                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1455         uint32_t rctl;
1456
1457         if (dev->data->promiscuous == 1)
1458                 return; /* must remain in all_multicast mode */
1459         rctl = E1000_READ_REG(hw, E1000_RCTL);
1460         rctl &= (~E1000_RCTL_MPE);
1461         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1462 }
1463
1464 static int
1465 eth_igb_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1466 {
1467         struct e1000_hw *hw =
1468                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1469         struct e1000_vfta * shadow_vfta =
1470                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1471         uint32_t vfta;
1472         uint32_t vid_idx;
1473         uint32_t vid_bit;
1474
1475         vid_idx = (uint32_t) ((vlan_id >> E1000_VFTA_ENTRY_SHIFT) &
1476                               E1000_VFTA_ENTRY_MASK);
1477         vid_bit = (uint32_t) (1 << (vlan_id & E1000_VFTA_ENTRY_BIT_SHIFT_MASK));
1478         vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, vid_idx);
1479         if (on)
1480                 vfta |= vid_bit;
1481         else
1482                 vfta &= ~vid_bit;
1483         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, vid_idx, vfta);
1484
1485         /* update local VFTA copy */
1486         shadow_vfta->vfta[vid_idx] = vfta;
1487
1488         return 0;
1489 }
1490
1491 static void
1492 eth_igb_vlan_tpid_set(struct rte_eth_dev *dev, uint16_t tpid)
1493 {
1494         struct e1000_hw *hw =
1495                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1496         uint32_t reg = ETHER_TYPE_VLAN ;
1497
1498         reg |= (tpid << 16);
1499         E1000_WRITE_REG(hw, E1000_VET, reg);
1500 }
1501
1502 static void
1503 igb_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1504 {
1505         struct e1000_hw *hw =
1506                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1507         uint32_t reg;
1508
1509         /* Filter Table Disable */
1510         reg = E1000_READ_REG(hw, E1000_RCTL);
1511         reg &= ~E1000_RCTL_CFIEN;
1512         reg &= ~E1000_RCTL_VFE;
1513         E1000_WRITE_REG(hw, E1000_RCTL, reg);
1514 }
1515
1516 static void
1517 igb_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1518 {
1519         struct e1000_hw *hw =
1520                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1521         struct e1000_vfta * shadow_vfta =
1522                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1523         uint32_t reg;
1524         int i;
1525
1526         /* Filter Table Enable, CFI not used for packet acceptance */
1527         reg = E1000_READ_REG(hw, E1000_RCTL);
1528         reg &= ~E1000_RCTL_CFIEN;
1529         reg |= E1000_RCTL_VFE;
1530         E1000_WRITE_REG(hw, E1000_RCTL, reg);
1531
1532         /* restore VFTA table */
1533         for (i = 0; i < IGB_VFTA_SIZE; i++)
1534                 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, i, shadow_vfta->vfta[i]);
1535 }
1536
1537 static void
1538 igb_vlan_hw_strip_disable(struct rte_eth_dev *dev)
1539 {
1540         struct e1000_hw *hw =
1541                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1542         uint32_t reg;
1543
1544         /* VLAN Mode Disable */
1545         reg = E1000_READ_REG(hw, E1000_CTRL);
1546         reg &= ~E1000_CTRL_VME;
1547         E1000_WRITE_REG(hw, E1000_CTRL, reg);
1548 }
1549
1550 static void
1551 igb_vlan_hw_strip_enable(struct rte_eth_dev *dev)
1552 {
1553         struct e1000_hw *hw =
1554                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1555         uint32_t reg;
1556
1557         /* VLAN Mode Enable */
1558         reg = E1000_READ_REG(hw, E1000_CTRL);
1559         reg |= E1000_CTRL_VME;
1560         E1000_WRITE_REG(hw, E1000_CTRL, reg);
1561 }
1562
1563 static void
1564 igb_vlan_hw_extend_disable(struct rte_eth_dev *dev)
1565 {
1566         struct e1000_hw *hw =
1567                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1568         uint32_t reg;
1569
1570         /* CTRL_EXT: Extended VLAN */
1571         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
1572         reg &= ~E1000_CTRL_EXT_EXTEND_VLAN;
1573         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
1574
1575         /* Update maximum packet length */
1576         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
1577                 E1000_WRITE_REG(hw, E1000_RLPML,
1578                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
1579                                                 VLAN_TAG_SIZE);
1580 }
1581
1582 static void
1583 igb_vlan_hw_extend_enable(struct rte_eth_dev *dev)
1584 {
1585         struct e1000_hw *hw =
1586                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1587         uint32_t reg;
1588
1589         /* CTRL_EXT: Extended VLAN */
1590         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
1591         reg |= E1000_CTRL_EXT_EXTEND_VLAN;
1592         E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg);
1593
1594         /* Update maximum packet length */
1595         if (dev->data->dev_conf.rxmode.jumbo_frame == 1)
1596                 E1000_WRITE_REG(hw, E1000_RLPML,
1597                         dev->data->dev_conf.rxmode.max_rx_pkt_len +
1598                                                 2 * VLAN_TAG_SIZE);
1599 }
1600
1601 static void
1602 eth_igb_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1603 {
1604         if(mask & ETH_VLAN_STRIP_MASK){
1605                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1606                         igb_vlan_hw_strip_enable(dev);
1607                 else
1608                         igb_vlan_hw_strip_disable(dev);
1609         }
1610
1611         if(mask & ETH_VLAN_FILTER_MASK){
1612                 if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1613                         igb_vlan_hw_filter_enable(dev);
1614                 else
1615                         igb_vlan_hw_filter_disable(dev);
1616         }
1617
1618         if(mask & ETH_VLAN_EXTEND_MASK){
1619                 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1620                         igb_vlan_hw_extend_enable(dev);
1621                 else
1622                         igb_vlan_hw_extend_disable(dev);
1623         }
1624 }
1625
1626
1627 /**
1628  * It enables the interrupt mask and then enable the interrupt.
1629  *
1630  * @param dev
1631  *  Pointer to struct rte_eth_dev.
1632  *
1633  * @return
1634  *  - On success, zero.
1635  *  - On failure, a negative value.
1636  */
1637 static int
1638 eth_igb_lsc_interrupt_setup(struct rte_eth_dev *dev)
1639 {
1640         struct e1000_interrupt *intr =
1641                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1642
1643         intr->mask |= E1000_ICR_LSC;
1644
1645         return 0;
1646 }
1647
1648 /*
1649  * It reads ICR and gets interrupt causes, check it and set a bit flag
1650  * to update link status.
1651  *
1652  * @param dev
1653  *  Pointer to struct rte_eth_dev.
1654  *
1655  * @return
1656  *  - On success, zero.
1657  *  - On failure, a negative value.
1658  */
1659 static int
1660 eth_igb_interrupt_get_status(struct rte_eth_dev *dev)
1661 {
1662         uint32_t icr;
1663         struct e1000_hw *hw =
1664                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1665         struct e1000_interrupt *intr =
1666                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1667
1668         igb_intr_disable(hw);
1669
1670         /* read-on-clear nic registers here */
1671         icr = E1000_READ_REG(hw, E1000_ICR);
1672
1673         intr->flags = 0;
1674         if (icr & E1000_ICR_LSC) {
1675                 intr->flags |= E1000_FLAG_NEED_LINK_UPDATE;
1676         }
1677
1678         if (icr & E1000_ICR_VMMB)
1679                 intr->flags |= E1000_FLAG_MAILBOX;
1680
1681         return 0;
1682 }
1683
1684 /*
1685  * It executes link_update after knowing an interrupt is prsent.
1686  *
1687  * @param dev
1688  *  Pointer to struct rte_eth_dev.
1689  *
1690  * @return
1691  *  - On success, zero.
1692  *  - On failure, a negative value.
1693  */
1694 static int
1695 eth_igb_interrupt_action(struct rte_eth_dev *dev)
1696 {
1697         struct e1000_hw *hw =
1698                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1699         struct e1000_interrupt *intr =
1700                 E1000_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
1701         uint32_t tctl, rctl;
1702         struct rte_eth_link link;
1703         int ret;
1704
1705         if (intr->flags & E1000_FLAG_MAILBOX) {
1706                 igb_pf_mbx_process(dev);
1707                 intr->flags &= ~E1000_FLAG_MAILBOX;
1708         }
1709
1710         igb_intr_enable(dev);
1711         rte_intr_enable(&(dev->pci_dev->intr_handle));
1712
1713         if (intr->flags & E1000_FLAG_NEED_LINK_UPDATE) {
1714                 intr->flags &= ~E1000_FLAG_NEED_LINK_UPDATE;
1715
1716                 /* set get_link_status to check register later */
1717                 hw->mac.get_link_status = 1;
1718                 ret = eth_igb_link_update(dev, 0);
1719
1720                 /* check if link has changed */
1721                 if (ret < 0)
1722                         return 0;
1723
1724                 memset(&link, 0, sizeof(link));
1725                 rte_igb_dev_atomic_read_link_status(dev, &link);
1726                 if (link.link_status) {
1727                         PMD_INIT_LOG(INFO,
1728                                 " Port %d: Link Up - speed %u Mbps - %s\n",
1729                                 dev->data->port_id, (unsigned)link.link_speed,
1730                                 link.link_duplex == ETH_LINK_FULL_DUPLEX ?
1731                                         "full-duplex" : "half-duplex");
1732                 } else {
1733                         PMD_INIT_LOG(INFO, " Port %d: Link Down\n",
1734                                                 dev->data->port_id);
1735                 }
1736                 PMD_INIT_LOG(INFO, "PCI Address: %04d:%02d:%02d:%d",
1737                                         dev->pci_dev->addr.domain,
1738                                         dev->pci_dev->addr.bus,
1739                                         dev->pci_dev->addr.devid,
1740                                         dev->pci_dev->addr.function);
1741                 tctl = E1000_READ_REG(hw, E1000_TCTL);
1742                 rctl = E1000_READ_REG(hw, E1000_RCTL);
1743                 if (link.link_status) {
1744                         /* enable Tx/Rx */
1745                         tctl |= E1000_TCTL_EN;
1746                         rctl |= E1000_RCTL_EN;
1747                 } else {
1748                         /* disable Tx/Rx */
1749                         tctl &= ~E1000_TCTL_EN;
1750                         rctl &= ~E1000_RCTL_EN;
1751                 }
1752                 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
1753                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1754                 E1000_WRITE_FLUSH(hw);
1755                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
1756         }
1757
1758         return 0;
1759 }
1760
1761 /**
1762  * Interrupt handler which shall be registered at first.
1763  *
1764  * @param handle
1765  *  Pointer to interrupt handle.
1766  * @param param
1767  *  The address of parameter (struct rte_eth_dev *) regsitered before.
1768  *
1769  * @return
1770  *  void
1771  */
1772 static void
1773 eth_igb_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
1774                                                         void *param)
1775 {
1776         struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1777
1778         eth_igb_interrupt_get_status(dev);
1779         eth_igb_interrupt_action(dev);
1780 }
1781
1782 static int
1783 eth_igb_led_on(struct rte_eth_dev *dev)
1784 {
1785         struct e1000_hw *hw;
1786
1787         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1788         return (e1000_led_on(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
1789 }
1790
1791 static int
1792 eth_igb_led_off(struct rte_eth_dev *dev)
1793 {
1794         struct e1000_hw *hw;
1795
1796         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1797         return (e1000_led_off(hw) == E1000_SUCCESS ? 0 : -ENOTSUP);
1798 }
1799
1800 static int
1801 eth_igb_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1802 {
1803         struct e1000_hw *hw;
1804         int err;
1805         enum e1000_fc_mode rte_fcmode_2_e1000_fcmode[] = {
1806                 e1000_fc_none,
1807                 e1000_fc_rx_pause,
1808                 e1000_fc_tx_pause,
1809                 e1000_fc_full
1810         };
1811         uint32_t rx_buf_size;
1812         uint32_t max_high_water;
1813         uint32_t rctl;
1814
1815         hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1816         rx_buf_size = igb_get_rx_buffer_size(hw);
1817         PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x \n", rx_buf_size);
1818
1819         /* At least reserve one Ethernet frame for watermark */
1820         max_high_water = rx_buf_size - ETHER_MAX_LEN;
1821         if ((fc_conf->high_water > max_high_water) ||
1822                 (fc_conf->high_water < fc_conf->low_water)) {
1823                 PMD_INIT_LOG(ERR, "e1000 incorrect high/low water value \n");
1824                 PMD_INIT_LOG(ERR, "high water must <=  0x%x \n", max_high_water);
1825                 return (-EINVAL);
1826         }
1827
1828         hw->fc.requested_mode = rte_fcmode_2_e1000_fcmode[fc_conf->mode];
1829         hw->fc.pause_time     = fc_conf->pause_time;
1830         hw->fc.high_water     = fc_conf->high_water;
1831         hw->fc.low_water      = fc_conf->low_water;
1832         hw->fc.send_xon       = fc_conf->send_xon;
1833
1834         err = e1000_setup_link_generic(hw);
1835         if (err == E1000_SUCCESS) {
1836
1837                 /* check if we want to forward MAC frames - driver doesn't have native
1838                  * capability to do that, so we'll write the registers ourselves */
1839
1840                 rctl = E1000_READ_REG(hw, E1000_RCTL);
1841
1842                 /* set or clear MFLCN.PMCF bit depending on configuration */
1843                 if (fc_conf->mac_ctrl_frame_fwd != 0)
1844                         rctl |= E1000_RCTL_PMCF;
1845                 else
1846                         rctl &= ~E1000_RCTL_PMCF;
1847
1848                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
1849                 E1000_WRITE_FLUSH(hw);
1850
1851                 return 0;
1852         }
1853
1854         PMD_INIT_LOG(ERR, "e1000_setup_link_generic = 0x%x \n", err);
1855         return (-EIO);
1856 }
1857
1858 #define E1000_RAH_POOLSEL_SHIFT      (18)
1859 static void
1860 eth_igb_rar_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
1861                 uint32_t index, __rte_unused uint32_t pool)
1862 {
1863         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1864         uint32_t rah;
1865
1866         e1000_rar_set(hw, mac_addr->addr_bytes, index);
1867         rah = E1000_READ_REG(hw, E1000_RAH(index));
1868         rah |= (0x1 << (E1000_RAH_POOLSEL_SHIFT + pool));
1869         E1000_WRITE_REG(hw, E1000_RAH(index), rah);
1870 }
1871
1872 static void
1873 eth_igb_rar_clear(struct rte_eth_dev *dev, uint32_t index)
1874 {
1875         uint8_t addr[ETHER_ADDR_LEN];
1876         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1877
1878         memset(addr, 0, sizeof(addr));
1879
1880         e1000_rar_set(hw, addr, index);
1881 }
1882
1883 /*
1884  * Virtual Function operations
1885  */
1886 static void
1887 igbvf_intr_disable(struct e1000_hw *hw)
1888 {
1889         PMD_INIT_LOG(DEBUG, "igbvf_intr_disable");
1890
1891         /* Clear interrupt mask to stop from interrupts being generated */
1892         E1000_WRITE_REG(hw, E1000_EIMC, 0xFFFF);
1893
1894         E1000_WRITE_FLUSH(hw);
1895 }
1896
1897 static void
1898 igbvf_stop_adapter(struct rte_eth_dev *dev)
1899 {
1900         u32 reg_val;
1901         u16 i;
1902         struct rte_eth_dev_info dev_info;
1903         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1904
1905         memset(&dev_info, 0, sizeof(dev_info));
1906         eth_igb_infos_get(dev, &dev_info);
1907
1908         /* Clear interrupt mask to stop from interrupts being generated */
1909         igbvf_intr_disable(hw);
1910
1911         /* Clear any pending interrupts, flush previous writes */
1912         E1000_READ_REG(hw, E1000_EICR);
1913
1914         /* Disable the transmit unit.  Each queue must be disabled. */
1915         for (i = 0; i < dev_info.max_tx_queues; i++)
1916                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), E1000_TXDCTL_SWFLSH);
1917
1918         /* Disable the receive unit by stopping each queue */
1919         for (i = 0; i < dev_info.max_rx_queues; i++) {
1920                 reg_val = E1000_READ_REG(hw, E1000_RXDCTL(i));
1921                 reg_val &= ~E1000_RXDCTL_QUEUE_ENABLE;
1922                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), reg_val);
1923                 while (E1000_READ_REG(hw, E1000_RXDCTL(i)) & E1000_RXDCTL_QUEUE_ENABLE)
1924                         ;
1925         }
1926
1927         /* flush all queues disables */
1928         E1000_WRITE_FLUSH(hw);
1929         msec_delay(2);
1930 }
1931
1932 static int eth_igbvf_link_update(struct e1000_hw *hw)
1933 {
1934         struct e1000_mbx_info *mbx = &hw->mbx;
1935         struct e1000_mac_info *mac = &hw->mac;
1936         int ret_val = E1000_SUCCESS;
1937
1938         PMD_INIT_LOG(DEBUG, "e1000_check_for_link_vf");
1939
1940         /*
1941          * We only want to run this if there has been a rst asserted.
1942          * in this case that could mean a link change, device reset,
1943          * or a virtual function reset
1944          */
1945
1946         /* If we were hit with a reset or timeout drop the link */
1947         if (!e1000_check_for_rst(hw, 0) || !mbx->timeout)
1948                 mac->get_link_status = TRUE;
1949
1950         if (!mac->get_link_status)
1951                 goto out;
1952
1953         /* if link status is down no point in checking to see if pf is up */
1954         if (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU))
1955                 goto out;
1956
1957         /* if we passed all the tests above then the link is up and we no
1958          * longer need to check for link */
1959         mac->get_link_status = FALSE;
1960
1961 out:
1962         return ret_val;
1963 }
1964
1965
1966 static int
1967 igbvf_dev_configure(struct rte_eth_dev *dev)
1968 {
1969         struct rte_eth_conf* conf = &dev->data->dev_conf;
1970
1971         PMD_INIT_LOG(DEBUG, "\nConfigured Virtual Function port id: %d\n",
1972                 dev->data->port_id);
1973
1974         /*
1975          * VF has no ability to enable/disable HW CRC
1976          * Keep the persistent behavior the same as Host PF
1977          */
1978 #ifndef RTE_LIBRTE_E1000_PF_DISABLE_STRIP_CRC
1979         if (!conf->rxmode.hw_strip_crc) {
1980                 PMD_INIT_LOG(INFO, "VF can't disable HW CRC Strip\n");
1981                 conf->rxmode.hw_strip_crc = 1;
1982         }
1983 #else
1984         if (conf->rxmode.hw_strip_crc) {
1985                 PMD_INIT_LOG(INFO, "VF can't enable HW CRC Strip\n");
1986                 conf->rxmode.hw_strip_crc = 0;
1987         }
1988 #endif
1989
1990         return 0;
1991 }
1992
1993 static int
1994 igbvf_dev_start(struct rte_eth_dev *dev)
1995 {
1996         struct e1000_hw *hw =
1997                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1998         int ret;
1999
2000         PMD_INIT_LOG(DEBUG, "igbvf_dev_start");
2001
2002         hw->mac.ops.reset_hw(hw);
2003
2004         /* Set all vfta */
2005         igbvf_set_vfta_all(dev,1);
2006
2007         eth_igbvf_tx_init(dev);
2008
2009         /* This can fail when allocating mbufs for descriptor rings */
2010         ret = eth_igbvf_rx_init(dev);
2011         if (ret) {
2012                 PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
2013                 igb_dev_clear_queues(dev);
2014                 return ret;
2015         }
2016
2017         return 0;
2018 }
2019
2020 static void
2021 igbvf_dev_stop(struct rte_eth_dev *dev)
2022 {
2023         PMD_INIT_LOG(DEBUG, "igbvf_dev_stop");
2024
2025         igbvf_stop_adapter(dev);
2026
2027         /*
2028           * Clear what we set, but we still keep shadow_vfta to
2029           * restore after device starts
2030           */
2031         igbvf_set_vfta_all(dev,0);
2032
2033         igb_dev_clear_queues(dev);
2034 }
2035
2036 static void
2037 igbvf_dev_close(struct rte_eth_dev *dev)
2038 {
2039         struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2040
2041         PMD_INIT_LOG(DEBUG, "igbvf_dev_close");
2042
2043         e1000_reset_hw(hw);
2044
2045         igbvf_dev_stop(dev);
2046 }
2047
2048 static int igbvf_set_vfta(struct e1000_hw *hw, uint16_t vid, bool on)
2049 {
2050         struct e1000_mbx_info *mbx = &hw->mbx;
2051         uint32_t msgbuf[2];
2052
2053         /* After set vlan, vlan strip will also be enabled in igb driver*/
2054         msgbuf[0] = E1000_VF_SET_VLAN;
2055         msgbuf[1] = vid;
2056         /* Setting the 8 bit field MSG INFO to TRUE indicates "add" */
2057         if (on)
2058                 msgbuf[0] |= E1000_VF_SET_VLAN_ADD;
2059
2060         return (mbx->ops.write_posted(hw, msgbuf, 2, 0));
2061 }
2062
2063 static void igbvf_set_vfta_all(struct rte_eth_dev *dev, bool on)
2064 {
2065         struct e1000_hw *hw =
2066                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2067         struct e1000_vfta * shadow_vfta =
2068                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2069         int i = 0, j = 0, vfta = 0, mask = 1;
2070
2071         for (i = 0; i < IGB_VFTA_SIZE; i++){
2072                 vfta = shadow_vfta->vfta[i];
2073                 if(vfta){
2074                         mask = 1;
2075                         for (j = 0; j < 32; j++){
2076                                 if(vfta & mask)
2077                                         igbvf_set_vfta(hw,
2078                                                 (uint16_t)((i<<5)+j), on);
2079                                 mask<<=1;
2080                         }
2081                 }
2082         }
2083
2084 }
2085
2086 static int
2087 igbvf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2088 {
2089         struct e1000_hw *hw =
2090                 E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2091         struct e1000_vfta * shadow_vfta =
2092                 E1000_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
2093         uint32_t vid_idx = 0;
2094         uint32_t vid_bit = 0;
2095         int ret = 0;
2096
2097         PMD_INIT_LOG(DEBUG, "igbvf_vlan_filter_set");
2098
2099         /*vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf*/
2100         ret = igbvf_set_vfta(hw, vlan_id, !!on);
2101         if(ret){
2102                 PMD_INIT_LOG(ERR, "Unable to set VF vlan");
2103                 return ret;
2104         }
2105         vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
2106         vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
2107
2108         /*Save what we set and retore it after device reset*/
2109         if (on)
2110                 shadow_vfta->vfta[vid_idx] |= vid_bit;
2111         else
2112                 shadow_vfta->vfta[vid_idx] &= ~vid_bit;
2113
2114         return 0;
2115 }
2116
2117 static int
2118 eth_igb_rss_reta_update(struct rte_eth_dev *dev,
2119                                 struct rte_eth_rss_reta *reta_conf)
2120 {
2121         uint8_t i,j,mask;
2122         uint32_t reta;
2123         struct e1000_hw *hw =
2124                         E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2125
2126         /*
2127          * Update Redirection Table RETA[n],n=0...31,The redirection table has
2128          * 128-entries in 32 registers
2129          */
2130         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2131                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2132                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2133                 else
2134                         mask = (uint8_t)((reta_conf->mask_hi >>
2135                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2136                 if (mask != 0) {
2137                         reta = 0;
2138                         /* If all 4 entries were set,don't need read RETA register */
2139                         if (mask != 0xF)
2140                                 reta = E1000_READ_REG(hw,E1000_RETA(i >> 2));
2141
2142                         for (j = 0; j < 4; j++) {
2143                                 if (mask & (0x1 << j)) {
2144                                         if (mask != 0xF)
2145                                                 reta &= ~(0xFF << 8 * j);
2146                                         reta |= reta_conf->reta[i + j] << 8 * j;
2147                                 }
2148                         }
2149                         E1000_WRITE_REG(hw, E1000_RETA(i >> 2),reta);
2150                 }
2151         }
2152
2153         return 0;
2154 }
2155
2156 static int
2157 eth_igb_rss_reta_query(struct rte_eth_dev *dev,
2158                                 struct rte_eth_rss_reta *reta_conf)
2159 {
2160         uint8_t i,j,mask;
2161         uint32_t reta;
2162         struct e1000_hw *hw =
2163                         E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2164
2165         /*
2166          * Read Redirection Table RETA[n],n=0...31,The redirection table has
2167          * 128-entries in 32 registers
2168          */
2169         for(i = 0; i < ETH_RSS_RETA_NUM_ENTRIES; i += 4) {
2170                 if (i < ETH_RSS_RETA_NUM_ENTRIES/2)
2171                         mask = (uint8_t)((reta_conf->mask_lo >> i) & 0xF);
2172                 else
2173                         mask = (uint8_t)((reta_conf->mask_hi >>
2174                                 (i - ETH_RSS_RETA_NUM_ENTRIES/2)) & 0xF);
2175
2176                 if (mask != 0) {
2177                         reta = E1000_READ_REG(hw,E1000_RETA(i >> 2));
2178                         for (j = 0; j < 4; j++) {
2179                                 if (mask & (0x1 << j))
2180                                         reta_conf->reta[i + j] =
2181                                                 (uint8_t)((reta >> 8 * j) & 0xFF);
2182                         }
2183                 }
2184         }
2185
2186         return 0;
2187 }
2188
2189 static struct rte_driver pmd_igb_drv = {
2190         .type = PMD_PDEV,
2191         .init = rte_igb_pmd_init,
2192 };
2193
2194 static struct rte_driver pmd_igbvf_drv = {
2195         .type = PMD_PDEV,
2196         .init = rte_igbvf_pmd_init,
2197 };
2198
2199 PMD_REGISTER_DRIVER(pmd_igb_drv);
2200 PMD_REGISTER_DRIVER(pmd_igbvf_drv);