lib/librte_eal/linuxapp/kni/ethtool/igb/igb_main.c

   1 /*******************************************************************************
   2
   3   Intel(R) Gigabit Ethernet Linux driver
   4   Copyright(c) 2007-2012 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28 #include <linux/module.h>
  29 #include <linux/types.h>
  30 #include <linux/init.h>
  31 #include <linux/vmalloc.h>
  32 #include <linux/pagemap.h>
  33 #include <linux/netdevice.h>
  34 #include <linux/tcp.h>
  35 #ifdef NETIF_F_TSO
  36 #include <net/checksum.h>
  37 #ifdef NETIF_F_TSO6
  38 #include <linux/ipv6.h>
  39 #include <net/ip6_checksum.h>
  40 #endif
  41 #endif
  42 #ifdef SIOCGMIIPHY
  43 #include <linux/mii.h>
  44 #endif
  45 #ifdef SIOCETHTOOL
  46 #include <linux/ethtool.h>
  47 #endif
  48 #include <linux/if_vlan.h>
  49 #ifdef CONFIG_PM_RUNTIME
  50 #include <linux/pm_runtime.h>
  51 #endif /* CONFIG_PM_RUNTIME */
  52
  53 #include "igb.h"
  54 #include "igb_vmdq.h"
  55
  56 #include <linux/uio_driver.h>
  57
  58 #define DRV_DEBUG
  59 #define DRV_HW_PERF
  60 #define VERSION_SUFFIX
  61
  62 #define MAJ 3
  63 #define MIN 4
  64 #define BUILD 8
  65 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." __stringify(BUILD) VERSION_SUFFIX DRV_DEBUG DRV_HW_PERF
  66
  67 char igb_driver_name[] = "igb";
  68 char igb_driver_version[] = DRV_VERSION;
  69 static const char igb_driver_string[] =
  70                                 "Intel(R) Gigabit Ethernet Network Driver";
  71 static const char igb_copyright[] = "Copyright (c) 2007-2011 Intel Corporation.";
  72
  73 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
  74         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER) },
  75         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER) },
  76         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES) },
  77         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII) },
  78         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER) },
  79         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER) },
  80         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER) },
  81         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES) },
  82         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII) },
  83         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL) },
  84         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII) },
  85         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES) },
  86         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE) },
  87         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP) },
  88         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576) },
  89         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS) },
  90         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES) },
  91         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER) },
  92         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES) },
  93         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD) },
  94         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2) },
  95         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER) },
  96         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER) },
  97         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES) },
  98         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER) },
  99         /* required last entry */
 100         {0, }
 101 };
 102
 103 //MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
 104 static void igb_set_sriov_capability(struct igb_adapter *adapter) __attribute__((__unused__));
 105 void igb_reset(struct igb_adapter *);
 106 static int igb_setup_all_tx_resources(struct igb_adapter *);
 107 static int igb_setup_all_rx_resources(struct igb_adapter *);
 108 static void igb_free_all_tx_resources(struct igb_adapter *);
 109 static void igb_free_all_rx_resources(struct igb_adapter *);
 110 static void igb_setup_mrqc(struct igb_adapter *);
 111 void igb_update_stats(struct igb_adapter *);
 112 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
 113 static void __devexit igb_remove(struct pci_dev *pdev);
 114 #ifdef HAVE_HW_TIME_STAMP
 115 static void igb_init_hw_timer(struct igb_adapter *adapter);
 116 #endif
 117 static int igb_sw_init(struct igb_adapter *);
 118 static int igb_open(struct net_device *);
 119 static int igb_close(struct net_device *);
 120 static void igb_configure_tx(struct igb_adapter *);
 121 static void igb_configure_rx(struct igb_adapter *);
 122 static void igb_clean_all_tx_rings(struct igb_adapter *);
 123 static void igb_clean_all_rx_rings(struct igb_adapter *);
 124 static void igb_clean_tx_ring(struct igb_ring *);
 125 static void igb_set_rx_mode(struct net_device *);
 126 static void igb_update_phy_info(unsigned long);
 127 static void igb_watchdog(unsigned long);
 128 static void igb_watchdog_task(struct work_struct *);
 129 static void igb_dma_err_task(struct work_struct *);
 130 static void igb_dma_err_timer(unsigned long data);
 131 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
 132 static struct net_device_stats *igb_get_stats(struct net_device *);
 133 static int igb_change_mtu(struct net_device *, int);
 134 void igb_full_sync_mac_table(struct igb_adapter *adapter);
 135 static int igb_set_mac(struct net_device *, void *);
 136 static void igb_set_uta(struct igb_adapter *adapter);
 137 static irqreturn_t igb_intr(int irq, void *);
 138 static irqreturn_t igb_intr_msi(int irq, void *);
 139 static irqreturn_t igb_msix_other(int irq, void *);
 140 static irqreturn_t igb_msix_ring(int irq, void *);
 141 #ifdef IGB_DCA
 142 static void igb_update_dca(struct igb_q_vector *);
 143 static void igb_setup_dca(struct igb_adapter *);
 144 #endif /* IGB_DCA */
 145 static int igb_poll(struct napi_struct *, int);
 146 static bool igb_clean_tx_irq(struct igb_q_vector *);
 147 static bool igb_clean_rx_irq(struct igb_q_vector *, int);
 148 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
 149 static void igb_tx_timeout(struct net_device *);
 150 static void igb_reset_task(struct work_struct *);
 151 #ifdef HAVE_VLAN_RX_REGISTER
 152 static void igb_vlan_mode(struct net_device *, struct vlan_group *);
 153 #endif
 154 #ifdef HAVE_VLAN_PROTOCOL
 155 static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16);
 156 static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16);
 157 #elif defined HAVE_INT_NDO_VLAN_RX_ADD_VID
 158 static int igb_vlan_rx_add_vid(struct net_device *, u16);
 159 static int igb_vlan_rx_kill_vid(struct net_device *, u16);
 160 #else
 161 static void igb_vlan_rx_add_vid(struct net_device *, u16);
 162 static void igb_vlan_rx_kill_vid(struct net_device *, u16);
 163 #endif
 164 static void igb_restore_vlan(struct igb_adapter *);
 165 void igb_rar_set(struct igb_adapter *adapter, u32 index);
 166 static void igb_ping_all_vfs(struct igb_adapter *);
 167 static void igb_msg_task(struct igb_adapter *);
 168 static void igb_vmm_control(struct igb_adapter *);
 169 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
 170 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
 171 static void igb_process_mdd_event(struct igb_adapter *);
 172 #ifdef IFLA_VF_MAX
 173 static int igb_ndo_set_vf_mac( struct net_device *netdev, int vf, u8 *mac);
 174 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
 175                                int vf, u16 vlan, u8 qos);
 176 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
 177 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
 178                                  struct ifla_vf_info *ivi);
 179 static void igb_check_vf_rate_limit(struct igb_adapter *);
 180 #endif
 181 static int igb_vf_configure(struct igb_adapter *adapter, int vf);
 182 static int igb_check_vf_assignment(struct igb_adapter *adapter);
 183 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
 184 static int igb_find_enabled_vfs(struct igb_adapter *adapter);
 185 #endif
 186 #ifdef CONFIG_PM
 187 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
 188 static int igb_suspend(struct device *dev);
 189 static int igb_resume(struct device *dev);
 190 #ifdef CONFIG_PM_RUNTIME
 191 static int igb_runtime_suspend(struct device *dev);
 192 static int igb_runtime_resume(struct device *dev);
 193 static int igb_runtime_idle(struct device *dev);
 194 #endif /* CONFIG_PM_RUNTIME */
 195 static const struct dev_pm_ops igb_pm_ops = {
 196 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,34)
 197         .suspend = igb_suspend,
 198         .resume = igb_resume,
 199         .freeze = igb_suspend,
 200         .thaw = igb_resume,
 201         .poweroff = igb_suspend,
 202         .restore = igb_resume,
 203 #ifdef CONFIG_PM_RUNTIME
 204         .runtime_suspend = igb_runtime_suspend,
 205         .runtime_resume = igb_runtime_resume,
 206         .runtime_idle = igb_runtime_idle,
 207 #endif
 208 #else /* Linux >= 2.6.34 */
 209         SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
 210 #ifdef CONFIG_PM_RUNTIME
 211         SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
 212                         igb_runtime_idle)
 213 #endif /* CONFIG_PM_RUNTIME */
 214 #endif /* Linux version */
 215 };
 216 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
 217 #endif /* CONFIG_PM */
 218 #ifndef USE_REBOOT_NOTIFIER
 219 static void igb_shutdown(struct pci_dev *);
 220 #else
 221 static int igb_notify_reboot(struct notifier_block *, unsigned long, void *);
 222 static struct notifier_block igb_notifier_reboot = {
 223         .notifier_call  = igb_notify_reboot,
 224         .next           = NULL,
 225         .priority       = 0
 226 };
 227 #endif
 228 #ifdef IGB_DCA
 229 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
 230 static struct notifier_block dca_notifier = {
 231         .notifier_call  = igb_notify_dca,
 232         .next           = NULL,
 233         .priority       = 0
 234 };
 235 #endif
 236 #ifdef CONFIG_NET_POLL_CONTROLLER
 237 /* for netdump / net console */
 238 static void igb_netpoll(struct net_device *);
 239 #endif
 240
 241 #ifdef HAVE_PCI_ERS
 242 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
 243                      pci_channel_state_t);
 244 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
 245 static void igb_io_resume(struct pci_dev *);
 246
 247 static struct pci_error_handlers igb_err_handler = {
 248         .error_detected = igb_io_error_detected,
 249         .slot_reset = igb_io_slot_reset,
 250         .resume = igb_io_resume,
 251 };
 252 #endif
 253
 254 static void igb_init_fw(struct igb_adapter *adapter);
 255 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
 256
 257 static struct pci_driver igb_driver = {
 258         .name     = igb_driver_name,
 259         .id_table = igb_pci_tbl,
 260         .probe    = igb_probe,
 261         .remove   = __devexit_p(igb_remove),
 262 #ifdef CONFIG_PM
 263 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
 264         .driver.pm = &igb_pm_ops,
 265 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
 266 #endif /* CONFIG_PM */
 267 #ifndef USE_REBOOT_NOTIFIER
 268         .shutdown = igb_shutdown,
 269 #endif
 270 #ifdef HAVE_PCI_ERS
 271         .err_handler = &igb_err_handler
 272 #endif
 273 };
 274
 275 //MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 276 //MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
 277 //MODULE_LICENSE("GPL");
 278 //MODULE_VERSION(DRV_VERSION);
 279
 280 static void igb_vfta_set(struct igb_adapter *adapter, u32 vid, bool add)
 281 {
 282         struct e1000_hw *hw = &adapter->hw;
 283         struct e1000_host_mng_dhcp_cookie *mng_cookie = &hw->mng_cookie;
 284         u32 index = (vid >> E1000_VFTA_ENTRY_SHIFT) & E1000_VFTA_ENTRY_MASK;
 285         u32 mask = 1 << (vid & E1000_VFTA_ENTRY_BIT_SHIFT_MASK);
 286         u32 vfta;
 287
 288         /*
 289          * if this is the management vlan the only option is to add it in so
 290          * that the management pass through will continue to work
 291          */
 292         if ((mng_cookie->status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
 293             (vid == mng_cookie->vlan_id))
 294                 add = TRUE;
 295
 296         vfta = adapter->shadow_vfta[index];
 297
 298         if (add)
 299                 vfta |= mask;
 300         else
 301                 vfta &= ~mask;
 302
 303         e1000_write_vfta(hw, index, vfta);
 304         adapter->shadow_vfta[index] = vfta;
 305 }
 306
 307 #ifdef HAVE_HW_TIME_STAMP
 308 /**
 309  * igb_read_clock - read raw cycle counter (to be used by time counter)
 310  */
 311 static cycle_t igb_read_clock(const struct cyclecounter *tc)
 312 {
 313         struct igb_adapter *adapter =
 314                 container_of(tc, struct igb_adapter, cycles);
 315         struct e1000_hw *hw = &adapter->hw;
 316         u64 stamp = 0;
 317         int shift = 0;
 318
 319         /*
 320          * The timestamp latches on lowest register read. For the 82580
 321          * the lowest register is SYSTIMR instead of SYSTIML.  However we never
 322          * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
 323          */
 324         if (hw->mac.type >= e1000_82580) {
 325                 stamp = E1000_READ_REG(hw, E1000_SYSTIMR) >> 8;
 326                 shift = IGB_82580_TSYNC_SHIFT;
 327         }
 328
 329         stamp |= (u64)E1000_READ_REG(hw, E1000_SYSTIML) << shift;
 330         stamp |= (u64)E1000_READ_REG(hw, E1000_SYSTIMH) << (shift + 32);
 331         return stamp;
 332 }
 333
 334 #endif /* SIOCSHWTSTAMP */
 335 static int debug = NETIF_MSG_DRV | NETIF_MSG_PROBE;
 336 //module_param(debug, int, 0);
 337 //MODULE_PARM_DESC(debug, "Debug level (0=none, ..., 16=all)");
 338
 339 /**
 340  * igb_init_module - Driver Registration Routine
 341  *
 342  * igb_init_module is the first routine called when the driver is
 343  * loaded. All it does is register with the PCI subsystem.
 344  **/
 345 static int __init igb_init_module(void)
 346 {
 347         int ret;
 348
 349         printk(KERN_INFO "%s - version %s\n",
 350                igb_driver_string, igb_driver_version);
 351
 352         printk(KERN_INFO "%s\n", igb_copyright);
 353 #ifdef IGB_SYSFS
 354 /* only use IGB_PROCFS if IGB_SYSFS is not defined */
 355 #else
 356 #ifdef IGB_PROCFS
 357         if (igb_procfs_topdir_init())
 358                 printk(KERN_INFO "Procfs failed to initialize topdir\n");
 359 #endif /* IGB_PROCFS */
 360 #endif /* IGB_SYSFS  */
 361
 362 #ifdef IGB_DCA
 363         dca_register_notify(&dca_notifier);
 364 #endif
 365         ret = pci_register_driver(&igb_driver);
 366 #ifdef USE_REBOOT_NOTIFIER
 367         if (ret >= 0) {
 368                 register_reboot_notifier(&igb_notifier_reboot);
 369         }
 370 #endif
 371         return ret;
 372 }
 373
 374 #undef module_init
 375 #define module_init(x) static int x(void)  __attribute__((__unused__));
 376 module_init(igb_init_module);
 377
 378 /**
 379  * igb_exit_module - Driver Exit Cleanup Routine
 380  *
 381  * igb_exit_module is called just before the driver is removed
 382  * from memory.
 383  **/
 384 static void __exit igb_exit_module(void)
 385 {
 386 #ifdef IGB_DCA
 387         dca_unregister_notify(&dca_notifier);
 388 #endif
 389 #ifdef USE_REBOOT_NOTIFIER
 390         unregister_reboot_notifier(&igb_notifier_reboot);
 391 #endif
 392         pci_unregister_driver(&igb_driver);
 393
 394 #ifdef IGB_SYSFS
 395 /* only compile IGB_PROCFS if IGB_SYSFS is not defined */
 396 #else
 397 #ifdef IGB_PROCFS
 398         igb_procfs_topdir_exit();
 399 #endif /* IGB_PROCFS */
 400 #endif /* IGB_SYSFS */
 401 }
 402
 403 #undef module_exit
 404 #define module_exit(x) static void x(void)  __attribute__((__unused__));
 405 module_exit(igb_exit_module);
 406
 407 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
 408 /**
 409  * igb_cache_ring_register - Descriptor ring to register mapping
 410  * @adapter: board private structure to initialize
 411  *
 412  * Once we know the feature-set enabled for the device, we'll cache
 413  * the register offset the descriptor ring is assigned to.
 414  **/
 415 static void igb_cache_ring_register(struct igb_adapter *adapter)
 416 {
 417         int i = 0, j = 0;
 418         u32 rbase_offset = adapter->vfs_allocated_count;
 419
 420         switch (adapter->hw.mac.type) {
 421         case e1000_82576:
 422                 /* The queues are allocated for virtualization such that VF 0
 423                  * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
 424                  * In order to avoid collision we start at the first free queue
 425                  * and continue consuming queues in the same sequence
 426                  */
 427                 if ((adapter->rss_queues > 1) && adapter->vmdq_pools) {
 428                         for (; i < adapter->rss_queues; i++)
 429                                 adapter->rx_ring[i]->reg_idx = rbase_offset +
 430                                                                Q_IDX_82576(i);
 431                 }
 432         case e1000_82575:
 433         case e1000_82580:
 434         case e1000_i350:
 435         default:
 436                 for (; i < adapter->num_rx_queues; i++)
 437                         adapter->rx_ring[i]->reg_idx = rbase_offset + i;
 438                 for (; j < adapter->num_tx_queues; j++)
 439                         adapter->tx_ring[j]->reg_idx = rbase_offset + j;
 440                 break;
 441         }
 442 }
 443
 444 static void igb_free_queues(struct igb_adapter *adapter)
 445 {
 446         int i;
 447
 448         for (i = 0; i < adapter->num_tx_queues; i++) {
 449                 kfree(adapter->tx_ring[i]);
 450                 adapter->tx_ring[i] = NULL;
 451         }
 452         for (i = 0; i < adapter->num_rx_queues; i++) {
 453                 kfree(adapter->rx_ring[i]);
 454                 adapter->rx_ring[i] = NULL;
 455         }
 456         adapter->num_rx_queues = 0;
 457         adapter->num_tx_queues = 0;
 458 }
 459
 460 /**
 461  * igb_alloc_queues - Allocate memory for all rings
 462  * @adapter: board private structure to initialize
 463  *
 464  * We allocate one ring per queue at run-time since we don't know the
 465  * number of queues at compile-time.
 466  **/
 467 static int igb_alloc_queues(struct igb_adapter *adapter)
 468 {
 469         struct igb_ring *ring;
 470         int i;
 471 #ifdef HAVE_DEVICE_NUMA_NODE
 472         int orig_node = adapter->node;
 473 #endif /* HAVE_DEVICE_NUMA_NODE */
 474
 475         for (i = 0; i < adapter->num_tx_queues; i++) {
 476 #ifdef HAVE_DEVICE_NUMA_NODE
 477                 if (orig_node == -1) {
 478                         int cur_node = next_online_node(adapter->node);
 479                         if (cur_node == MAX_NUMNODES)
 480                                 cur_node = first_online_node;
 481                         adapter->node = cur_node;
 482                 }
 483 #endif /* HAVE_DEVICE_NUMA_NODE */
 484                 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
 485                                     adapter->node);
 486                 if (!ring)
 487                         ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
 488                 if (!ring)
 489                         goto err;
 490                 ring->count = adapter->tx_ring_count;
 491                 ring->queue_index = i;
 492                 ring->dev = pci_dev_to_dev(adapter->pdev);
 493                 ring->netdev = adapter->netdev;
 494                 ring->numa_node = adapter->node;
 495                 /* For 82575, context index must be unique per ring. */
 496                 if (adapter->hw.mac.type == e1000_82575)
 497                         set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
 498                 adapter->tx_ring[i] = ring;
 499         }
 500 #ifdef HAVE_DEVICE_NUMA_NODE
 501         /* Restore the adapter's original node */
 502         adapter->node = orig_node;
 503 #endif /* HAVE_DEVICE_NUMA_NODE */
 504
 505         for (i = 0; i < adapter->num_rx_queues; i++) {
 506 #ifdef HAVE_DEVICE_NUMA_NODE
 507                 if (orig_node == -1) {
 508                         int cur_node = next_online_node(adapter->node);
 509                         if (cur_node == MAX_NUMNODES)
 510                                 cur_node = first_online_node;
 511                         adapter->node = cur_node;
 512                 }
 513 #endif /* HAVE_DEVICE_NUMA_NODE */
 514                 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
 515                                     adapter->node);
 516                 if (!ring)
 517                         ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
 518                 if (!ring)
 519                         goto err;
 520                 ring->count = adapter->rx_ring_count;
 521                 ring->queue_index = i;
 522                 ring->dev = pci_dev_to_dev(adapter->pdev);
 523                 ring->netdev = adapter->netdev;
 524                 ring->numa_node = adapter->node;
 525 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
 526                 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
 527 #endif
 528 #ifndef HAVE_NDO_SET_FEATURES
 529                 /* enable rx checksum */
 530                 set_bit(IGB_RING_FLAG_RX_CSUM, &ring->flags);
 531
 532 #endif
 533                 /* set flag indicating ring supports SCTP checksum offload */
 534                 if (adapter->hw.mac.type >= e1000_82576)
 535                         set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
 536
 537                 /* On i350, loopback VLAN packets have the tag byte-swapped. */
 538                 if (adapter->hw.mac.type == e1000_i350)
 539                         set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
 540
 541                 adapter->rx_ring[i] = ring;
 542         }
 543 #ifdef HAVE_DEVICE_NUMA_NODE
 544         /* Restore the adapter's original node */
 545         adapter->node = orig_node;
 546 #endif /* HAVE_DEVICE_NUMA_NODE */
 547
 548         igb_cache_ring_register(adapter);
 549
 550         return E1000_SUCCESS;
 551
 552 err:
 553 #ifdef HAVE_DEVICE_NUMA_NODE
 554         /* Restore the adapter's original node */
 555         adapter->node = orig_node;
 556 #endif /* HAVE_DEVICE_NUMA_NODE */
 557         igb_free_queues(adapter);
 558
 559         return -ENOMEM;
 560 }
 561
 562 static void igb_configure_lli(struct igb_adapter *adapter)
 563 {
 564         struct e1000_hw *hw = &adapter->hw;
 565         u16 port;
 566
 567         /* LLI should only be enabled for MSI-X or MSI interrupts */
 568         if (!adapter->msix_entries && !(adapter->flags & IGB_FLAG_HAS_MSI))
 569                 return;
 570
 571         if (adapter->lli_port) {
 572                 /* use filter 0 for port */
 573                 port = htons((u16)adapter->lli_port);
 574                 E1000_WRITE_REG(hw, E1000_IMIR(0),
 575                         (port | E1000_IMIR_PORT_IM_EN));
 576                 E1000_WRITE_REG(hw, E1000_IMIREXT(0),
 577                         (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
 578         }
 579
 580         if (adapter->flags & IGB_FLAG_LLI_PUSH) {
 581                 /* use filter 1 for push flag */
 582                 E1000_WRITE_REG(hw, E1000_IMIR(1),
 583                         (E1000_IMIR_PORT_BP | E1000_IMIR_PORT_IM_EN));
 584                 E1000_WRITE_REG(hw, E1000_IMIREXT(1),
 585                         (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_PSH));
 586         }
 587
 588         if (adapter->lli_size) {
 589                 /* use filter 2 for size */
 590                 E1000_WRITE_REG(hw, E1000_IMIR(2),
 591                         (E1000_IMIR_PORT_BP | E1000_IMIR_PORT_IM_EN));
 592                 E1000_WRITE_REG(hw, E1000_IMIREXT(2),
 593                         (adapter->lli_size | E1000_IMIREXT_CTRL_BP));
 594         }
 595
 596 }
 597
 598 /**
 599  *  igb_write_ivar - configure ivar for given MSI-X vector
 600  *  @hw: pointer to the HW structure
 601  *  @msix_vector: vector number we are allocating to a given ring
 602  *  @index: row index of IVAR register to write within IVAR table
 603  *  @offset: column offset of in IVAR, should be multiple of 8
 604  *
 605  *  This function is intended to handle the writing of the IVAR register
 606  *  for adapters 82576 and newer.  The IVAR table consists of 2 columns,
 607  *  each containing an cause allocation for an Rx and Tx ring, and a
 608  *  variable number of rows depending on the number of queues supported.
 609  **/
 610 static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
 611                            int index, int offset)
 612 {
 613         u32 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 614
 615         /* clear any bits that are currently set */
 616         ivar &= ~((u32)0xFF << offset);
 617
 618         /* write vector and valid bit */
 619         ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
 620
 621         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 622 }
 623
 624 #define IGB_N0_QUEUE -1
 625 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
 626 {
 627         struct igb_adapter *adapter = q_vector->adapter;
 628         struct e1000_hw *hw = &adapter->hw;
 629         int rx_queue = IGB_N0_QUEUE;
 630         int tx_queue = IGB_N0_QUEUE;
 631         u32 msixbm = 0;
 632
 633         if (q_vector->rx.ring)
 634                 rx_queue = q_vector->rx.ring->reg_idx;
 635         if (q_vector->tx.ring)
 636                 tx_queue = q_vector->tx.ring->reg_idx;
 637
 638         switch (hw->mac.type) {
 639         case e1000_82575:
 640                 /* The 82575 assigns vectors using a bitmask, which matches the
 641                    bitmask for the EICR/EIMS/EIMC registers.  To assign one
 642                    or more queues to a vector, we write the appropriate bits
 643                    into the MSIXBM register for that vector. */
 644                 if (rx_queue > IGB_N0_QUEUE)
 645                         msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
 646                 if (tx_queue > IGB_N0_QUEUE)
 647                         msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
 648                 if (!adapter->msix_entries && msix_vector == 0)
 649                         msixbm |= E1000_EIMS_OTHER;
 650                 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), msix_vector, msixbm);
 651                 q_vector->eims_value = msixbm;
 652                 break;
 653         case e1000_82576:
 654                 /*
 655                  * 82576 uses a table that essentially consists of 2 columns
 656                  * with 8 rows.  The ordering is column-major so we use the
 657                  * lower 3 bits as the row index, and the 4th bit as the
 658                  * column offset.
 659                  */
 660                 if (rx_queue > IGB_N0_QUEUE)
 661                         igb_write_ivar(hw, msix_vector,
 662                                        rx_queue & 0x7,
 663                                        (rx_queue & 0x8) << 1);
 664                 if (tx_queue > IGB_N0_QUEUE)
 665                         igb_write_ivar(hw, msix_vector,
 666                                        tx_queue & 0x7,
 667                                        ((tx_queue & 0x8) << 1) + 8);
 668                 q_vector->eims_value = 1 << msix_vector;
 669                 break;
 670         case e1000_82580:
 671         case e1000_i350:
 672                 /*
 673                  * On 82580 and newer adapters the scheme is similar to 82576
 674                  * however instead of ordering column-major we have things
 675                  * ordered row-major.  So we traverse the table by using
 676                  * bit 0 as the column offset, and the remaining bits as the
 677                  * row index.
 678                  */
 679                 if (rx_queue > IGB_N0_QUEUE)
 680                         igb_write_ivar(hw, msix_vector,
 681                                        rx_queue >> 1,
 682                                        (rx_queue & 0x1) << 4);
 683                 if (tx_queue > IGB_N0_QUEUE)
 684                         igb_write_ivar(hw, msix_vector,
 685                                        tx_queue >> 1,
 686                                        ((tx_queue & 0x1) << 4) + 8);
 687                 q_vector->eims_value = 1 << msix_vector;
 688                 break;
 689         default:
 690                 BUG();
 691                 break;
 692         }
 693
 694         /* add q_vector eims value to global eims_enable_mask */
 695         adapter->eims_enable_mask |= q_vector->eims_value;
 696
 697         /* configure q_vector to set itr on first interrupt */
 698         q_vector->set_itr = 1;
 699 }
 700
 701 /**
 702  * igb_configure_msix - Configure MSI-X hardware
 703  *
 704  * igb_configure_msix sets up the hardware to properly
 705  * generate MSI-X interrupts.
 706  **/
 707 static void igb_configure_msix(struct igb_adapter *adapter)
 708 {
 709         u32 tmp;
 710         int i, vector = 0;
 711         struct e1000_hw *hw = &adapter->hw;
 712
 713         adapter->eims_enable_mask = 0;
 714
 715         /* set vector for other causes, i.e. link changes */
 716         switch (hw->mac.type) {
 717         case e1000_82575:
 718                 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
 719                 /* enable MSI-X PBA support*/
 720                 tmp |= E1000_CTRL_EXT_PBA_CLR;
 721
 722                 /* Auto-Mask interrupts upon ICR read. */
 723                 tmp |= E1000_CTRL_EXT_EIAME;
 724                 tmp |= E1000_CTRL_EXT_IRCA;
 725
 726                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
 727
 728                 /* enable msix_other interrupt */
 729                 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), vector++,
 730                                       E1000_EIMS_OTHER);
 731                 adapter->eims_other = E1000_EIMS_OTHER;
 732
 733                 break;
 734
 735         case e1000_82576:
 736         case e1000_82580:
 737         case e1000_i350:
 738                 /* Turn on MSI-X capability first, or our settings
 739                  * won't stick.  And it will take days to debug. */
 740                 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE |
 741                                 E1000_GPIE_PBA | E1000_GPIE_EIAME |
 742                                 E1000_GPIE_NSICR);
 743
 744                 /* enable msix_other interrupt */
 745                 adapter->eims_other = 1 << vector;
 746                 tmp = (vector++ | E1000_IVAR_VALID) << 8;
 747
 748                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmp);
 749                 break;
 750         default:
 751                 /* do nothing, since nothing else supports MSI-X */
 752                 break;
 753         } /* switch (hw->mac.type) */
 754
 755         adapter->eims_enable_mask |= adapter->eims_other;
 756
 757         for (i = 0; i < adapter->num_q_vectors; i++)
 758                 igb_assign_vector(adapter->q_vector[i], vector++);
 759
 760         E1000_WRITE_FLUSH(hw);
 761 }
 762
 763 /**
 764  * igb_request_msix - Initialize MSI-X interrupts
 765  *
 766  * igb_request_msix allocates MSI-X vectors and requests interrupts from the
 767  * kernel.
 768  **/
 769 static int igb_request_msix(struct igb_adapter *adapter)
 770 {
 771         struct net_device *netdev = adapter->netdev;
 772         struct e1000_hw *hw = &adapter->hw;
 773         int i, err = 0, vector = 0;
 774
 775         err = request_irq(adapter->msix_entries[vector].vector,
 776                           &igb_msix_other, 0, netdev->name, adapter);
 777         if (err)
 778                 goto out;
 779         vector++;
 780
 781         for (i = 0; i < adapter->num_q_vectors; i++) {
 782                 struct igb_q_vector *q_vector = adapter->q_vector[i];
 783
 784                 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
 785
 786                 if (q_vector->rx.ring && q_vector->tx.ring)
 787                         sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
 788                                 q_vector->rx.ring->queue_index);
 789                 else if (q_vector->tx.ring)
 790                         sprintf(q_vector->name, "%s-tx-%u", netdev->name,
 791                                 q_vector->tx.ring->queue_index);
 792                 else if (q_vector->rx.ring)
 793                         sprintf(q_vector->name, "%s-rx-%u", netdev->name,
 794                                 q_vector->rx.ring->queue_index);
 795                 else
 796                         sprintf(q_vector->name, "%s-unused", netdev->name);
 797
 798                 err = request_irq(adapter->msix_entries[vector].vector,
 799                                   igb_msix_ring, 0, q_vector->name,
 800                                   q_vector);
 801                 if (err)
 802                         goto out;
 803                 vector++;
 804         }
 805
 806         igb_configure_msix(adapter);
 807         return 0;
 808 out:
 809         return err;
 810 }
 811
 812 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
 813 {
 814         if (adapter->msix_entries) {
 815                 pci_disable_msix(adapter->pdev);
 816                 kfree(adapter->msix_entries);
 817                 adapter->msix_entries = NULL;
 818         } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
 819                 pci_disable_msi(adapter->pdev);
 820         }
 821 }
 822
 823 /**
 824  * igb_free_q_vectors - Free memory allocated for interrupt vectors
 825  * @adapter: board private structure to initialize
 826  *
 827  * This function frees the memory allocated to the q_vectors.  In addition if
 828  * NAPI is enabled it will delete any references to the NAPI struct prior
 829  * to freeing the q_vector.
 830  **/
 831 static void igb_free_q_vectors(struct igb_adapter *adapter)
 832 {
 833         int v_idx;
 834
 835         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
 836                 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
 837                 adapter->q_vector[v_idx] = NULL;
 838                 if (!q_vector)
 839                         continue;
 840                 netif_napi_del(&q_vector->napi);
 841 #ifndef IGB_NO_LRO
 842                 if (q_vector->lrolist) {
 843                         __skb_queue_purge(&q_vector->lrolist->active);
 844                         vfree(q_vector->lrolist);
 845                         q_vector->lrolist = NULL;
 846                 }
 847 #endif
 848                 kfree(q_vector);
 849         }
 850         adapter->num_q_vectors = 0;
 851 }
 852
 853 /**
 854  * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
 855  *
 856  * This function resets the device so that it has 0 rx queues, tx queues, and
 857  * MSI-X interrupts allocated.
 858  */
 859 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
 860 {
 861         igb_free_queues(adapter);
 862         igb_free_q_vectors(adapter);
 863         igb_reset_interrupt_capability(adapter);
 864 }
 865
 866 /**
 867  * igb_process_mdd_event
 868  * @adapter - board private structure
 869  *
 870  * Identify a malicious VF, disable the VF TX/RX queues and log a message.
 871  */
 872 static void igb_process_mdd_event(struct igb_adapter *adapter)
 873 {
 874         struct e1000_hw *hw = &adapter->hw;
 875         u32 lvmmc, vfte, vfre, mdfb;
 876         u8 vf_queue;
 877
 878         lvmmc = E1000_READ_REG(hw, E1000_LVMMC);
 879         vf_queue = lvmmc >> 29;
 880
 881         /* VF index cannot be bigger or equal to VFs allocated */
 882         if (vf_queue >= adapter->vfs_allocated_count)
 883                 return;
 884
 885         netdev_info(adapter->netdev,
 886                     "VF %d misbehaved. VF queues are disabled. "
 887                     "VM misbehavior code is 0x%x\n", vf_queue, lvmmc);
 888
 889         /* Disable VFTE and VFRE related bits */
 890         vfte = E1000_READ_REG(hw, E1000_VFTE);
 891         vfte &= ~(1 << vf_queue);
 892         E1000_WRITE_REG(hw, E1000_VFTE, vfte);
 893
 894         vfre = E1000_READ_REG(hw, E1000_VFRE);
 895         vfre &= ~(1 << vf_queue);
 896         E1000_WRITE_REG(hw, E1000_VFRE, vfre);
 897
 898         /* Disable MDFB related bit */
 899         mdfb = E1000_READ_REG(hw, E1000_MDFB);
 900         mdfb &= ~(1 << vf_queue);
 901         E1000_WRITE_REG(hw, E1000_MDFB, mdfb);
 902
 903         /* Reset the specific VF */
 904         E1000_WRITE_REG(hw, E1000_VTCTRL(vf_queue), E1000_VTCTRL_RST);
 905 }
 906
 907 /**
 908  * igb_disable_mdd
 909  * @adapter - board private structure
 910  *
 911  * Disable MDD behavior in the HW
 912  **/
 913 static void igb_disable_mdd(struct igb_adapter *adapter)
 914 {
 915         struct e1000_hw *hw = &adapter->hw;
 916         u32 reg;
 917
 918         if (hw->mac.type != e1000_i350)
 919                 return;
 920
 921         reg = E1000_READ_REG(hw, E1000_DTXCTL);
 922         reg &= (~E1000_DTXCTL_MDP_EN);
 923         E1000_WRITE_REG(hw, E1000_DTXCTL, reg);
 924 }
 925
 926 /**
 927  * igb_enable_mdd
 928  * @adapter - board private structure
 929  *
 930  * Enable the HW to detect malicious driver and sends an interrupt to
 931  * the driver.
 932  *
 933  * Only available on i350 device
 934  **/
 935 static void igb_enable_mdd(struct igb_adapter *adapter)
 936 {
 937         struct e1000_hw *hw = &adapter->hw;
 938         u32 reg;
 939
 940         if (hw->mac.type != e1000_i350)
 941                 return;
 942
 943         reg = E1000_READ_REG(hw, E1000_DTXCTL);
 944         reg |= E1000_DTXCTL_MDP_EN;
 945         E1000_WRITE_REG(hw, E1000_DTXCTL, reg);
 946 }
 947
 948 /**
 949  * igb_reset_sriov_capability - disable SR-IOV if enabled
 950  *
 951  * Attempt to disable single root IO virtualization capabilites present in the
 952  * kernel.
 953  **/
 954 static void igb_reset_sriov_capability(struct igb_adapter *adapter)
 955 {
 956         struct pci_dev *pdev = adapter->pdev;
 957         struct e1000_hw *hw = &adapter->hw;
 958
 959         /* reclaim resources allocated to VFs */
 960         if (adapter->vf_data) {
 961                 if (!igb_check_vf_assignment(adapter)) {
 962                         /*
 963                          * disable iov and allow time for transactions to
 964                          * clear
 965                          */
 966                         pci_disable_sriov(pdev);
 967                         msleep(500);
 968
 969                         dev_info(pci_dev_to_dev(pdev), "IOV Disabled\n");
 970                 } else {
 971                         dev_info(pci_dev_to_dev(pdev), "IOV Not Disabled\n "
 972                                         "VF(s) are assigned to guests!\n");
 973                 }
 974                 /* Disable Malicious Driver Detection */
 975                 igb_disable_mdd(adapter);
 976
 977                 /* free vf data storage */
 978                 kfree(adapter->vf_data);
 979                 adapter->vf_data = NULL;
 980
 981                 /* switch rings back to PF ownership */
 982                 E1000_WRITE_REG(hw, E1000_IOVCTL,
 983                                 E1000_IOVCTL_REUSE_VFQ);
 984                 E1000_WRITE_FLUSH(hw);
 985                 msleep(100);
 986         }
 987
 988         adapter->vfs_allocated_count = 0;
 989 }
 990
 991 /**
 992  * igb_set_sriov_capability - setup SR-IOV if supported
 993  *
 994  * Attempt to enable single root IO virtualization capabilites present in the
 995  * kernel.
 996  **/
 997 static void igb_set_sriov_capability(struct igb_adapter *adapter)
 998 {
 999         struct pci_dev *pdev = adapter->pdev;
1000         int old_vfs = 0;
1001         int i;
1002
1003 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
1004         old_vfs = igb_find_enabled_vfs(adapter);
1005 #endif
1006         if (old_vfs) {
1007                 dev_info(pci_dev_to_dev(pdev),
1008                                 "%d pre-allocated VFs found - override "
1009                                 "max_vfs setting of %d\n", old_vfs,
1010                                 adapter->vfs_allocated_count);
1011                 adapter->vfs_allocated_count = old_vfs;
1012         }
1013         /* no VFs requested, do nothing */
1014         if (!adapter->vfs_allocated_count)
1015                 return;
1016
1017         /* allocate vf data storage */
1018         adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
1019                                    sizeof(struct vf_data_storage),
1020                                    GFP_KERNEL);
1021
1022         if (adapter->vf_data) {
1023                 if (!old_vfs) {
1024                         if (pci_enable_sriov(pdev,
1025                                         adapter->vfs_allocated_count))
1026                                 goto err_out;
1027                 }
1028                 for (i = 0; i < adapter->vfs_allocated_count; i++)
1029                         igb_vf_configure(adapter, i);
1030
1031                 /* DMA Coalescing is not supported in IOV mode. */
1032                 if (adapter->hw.mac.type >= e1000_i350)
1033                 adapter->dmac = IGB_DMAC_DISABLE;
1034                 if (adapter->hw.mac.type < e1000_i350)
1035                 adapter->flags |= IGB_FLAG_DETECT_BAD_DMA;
1036                 return;
1037
1038         }
1039
1040 err_out:
1041         kfree(adapter->vf_data);
1042         adapter->vf_data = NULL;
1043         adapter->vfs_allocated_count = 0;
1044         dev_warn(pci_dev_to_dev(pdev),
1045                         "Failed to initialize SR-IOV virtualization\n");
1046 }
1047
1048 /**
1049  * igb_set_interrupt_capability - set MSI or MSI-X if supported
1050  *
1051  * Attempt to configure interrupts using the best available
1052  * capabilities of the hardware and kernel.
1053  **/
1054 static void igb_set_interrupt_capability(struct igb_adapter *adapter)
1055 {
1056         struct pci_dev *pdev = adapter->pdev;
1057         int err;
1058         int numvecs, i;
1059
1060         /* Number of supported queues. */
1061         adapter->num_rx_queues = adapter->rss_queues;
1062
1063         if (adapter->vmdq_pools > 1)
1064                 adapter->num_rx_queues += adapter->vmdq_pools - 1;
1065
1066 #ifdef HAVE_TX_MQ
1067         if (adapter->vmdq_pools)
1068                 adapter->num_tx_queues = adapter->vmdq_pools;
1069         else
1070                 adapter->num_tx_queues = adapter->num_rx_queues;
1071 #else
1072         adapter->num_tx_queues = max_t(u32, 1, adapter->vmdq_pools);
1073 #endif
1074
1075         switch (adapter->int_mode) {
1076         case IGB_INT_MODE_MSIX:
1077                 /* start with one vector for every rx queue */
1078                 numvecs = adapter->num_rx_queues;
1079
1080                 /* if tx handler is seperate add 1 for every tx queue */
1081                 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1082                         numvecs += adapter->num_tx_queues;
1083
1084                 /* store the number of vectors reserved for queues */
1085                 adapter->num_q_vectors = numvecs;
1086
1087                 /* add 1 vector for link status interrupts */
1088                 numvecs++;
1089                 adapter->msix_entries = kcalloc(numvecs,
1090                                                 sizeof(struct msix_entry),
1091                                                 GFP_KERNEL);
1092                 if (adapter->msix_entries) {
1093                         for (i = 0; i < numvecs; i++)
1094                                 adapter->msix_entries[i].entry = i;
1095
1096                         err = pci_enable_msix(pdev,
1097                                               adapter->msix_entries, numvecs);
1098                         if (err == 0)
1099                                 break;
1100                 }
1101                 /* MSI-X failed, so fall through and try MSI */
1102                 dev_warn(pci_dev_to_dev(pdev), "Failed to initialize MSI-X interrupts. "
1103                          "Falling back to MSI interrupts.\n");
1104                 igb_reset_interrupt_capability(adapter);
1105         case IGB_INT_MODE_MSI:
1106                 if (!pci_enable_msi(pdev))
1107                         adapter->flags |= IGB_FLAG_HAS_MSI;
1108                 else
1109                         dev_warn(pci_dev_to_dev(pdev), "Failed to initialize MSI "
1110                                  "interrupts.  Falling back to legacy "
1111                                  "interrupts.\n");
1112                 /* Fall through */
1113         case IGB_INT_MODE_LEGACY:
1114                 /* disable advanced features and set number of queues to 1 */
1115                 igb_reset_sriov_capability(adapter);
1116                 adapter->vmdq_pools = 0;
1117                 adapter->rss_queues = 1;
1118                 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1119                 adapter->num_rx_queues = 1;
1120                 adapter->num_tx_queues = 1;
1121                 adapter->num_q_vectors = 1;
1122                 /* Don't do anything; this is system default */
1123                 break;
1124         }
1125
1126 #ifdef HAVE_TX_MQ
1127         /* Notify the stack of the (possibly) reduced Tx Queue count. */
1128 #ifdef CONFIG_NETDEVICES_MULTIQUEUE
1129         adapter->netdev->egress_subqueue_count = adapter->num_tx_queues;
1130 #else
1131         adapter->netdev->real_num_tx_queues =
1132                         (adapter->vmdq_pools ? 1 : adapter->num_tx_queues);
1133 #endif /* CONFIG_NETDEVICES_MULTIQUEUE */
1134 #endif /* HAVE_TX_MQ */
1135 }
1136
1137 /**
1138  * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1139  * @adapter: board private structure to initialize
1140  *
1141  * We allocate one q_vector per queue interrupt.  If allocation fails we
1142  * return -ENOMEM.
1143  **/
1144 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1145 {
1146         struct igb_q_vector *q_vector;
1147         struct e1000_hw *hw = &adapter->hw;
1148         int v_idx;
1149 #ifdef HAVE_DEVICE_NUMA_NODE
1150         int orig_node = adapter->node;
1151 #endif /* HAVE_DEVICE_NUMA_NODE */
1152
1153         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1154 #ifdef HAVE_DEVICE_NUMA_NODE
1155                 if ((adapter->num_q_vectors == (adapter->num_rx_queues +
1156                                                 adapter->num_tx_queues)) &&
1157                     (adapter->num_rx_queues == v_idx))
1158                         adapter->node = orig_node;
1159                 if (orig_node == -1) {
1160                         int cur_node = next_online_node(adapter->node);
1161                         if (cur_node == MAX_NUMNODES)
1162                                 cur_node = first_online_node;
1163                         adapter->node = cur_node;
1164                 }
1165 #endif /* HAVE_DEVICE_NUMA_NODE */
1166                 q_vector = kzalloc_node(sizeof(struct igb_q_vector), GFP_KERNEL,
1167                                         adapter->node);
1168                 if (!q_vector)
1169                         q_vector = kzalloc(sizeof(struct igb_q_vector),
1170                                            GFP_KERNEL);
1171                 if (!q_vector)
1172                         goto err_out;
1173                 q_vector->adapter = adapter;
1174                 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1175                 q_vector->itr_val = IGB_START_ITR;
1176                 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1177                 adapter->q_vector[v_idx] = q_vector;
1178 #ifndef IGB_NO_LRO
1179                 if (v_idx < adapter->num_rx_queues) {
1180                         int size = sizeof(struct igb_lro_list);
1181                         q_vector->lrolist = vzalloc_node(size, q_vector->numa_node);
1182                         if (!q_vector->lrolist)
1183                                 q_vector->lrolist = vzalloc(size);
1184                         if (!q_vector->lrolist)
1185                                 goto err_out;
1186                         __skb_queue_head_init(&q_vector->lrolist->active);
1187                 }
1188 #endif /* IGB_NO_LRO */
1189         }
1190 #ifdef HAVE_DEVICE_NUMA_NODE
1191         /* Restore the adapter's original node */
1192         adapter->node = orig_node;
1193 #endif /* HAVE_DEVICE_NUMA_NODE */
1194
1195         return 0;
1196
1197 err_out:
1198 #ifdef HAVE_DEVICE_NUMA_NODE
1199         /* Restore the adapter's original node */
1200         adapter->node = orig_node;
1201 #endif /* HAVE_DEVICE_NUMA_NODE */
1202         igb_free_q_vectors(adapter);
1203         return -ENOMEM;
1204 }
1205
1206 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1207                                       int ring_idx, int v_idx)
1208 {
1209         struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1210
1211         q_vector->rx.ring = adapter->rx_ring[ring_idx];
1212         q_vector->rx.ring->q_vector = q_vector;
1213         q_vector->rx.count++;
1214         q_vector->itr_val = adapter->rx_itr_setting;
1215         if (q_vector->itr_val && q_vector->itr_val <= 3)
1216                 q_vector->itr_val = IGB_START_ITR;
1217 }
1218
1219 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1220                                       int ring_idx, int v_idx)
1221 {
1222         struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1223
1224         q_vector->tx.ring = adapter->tx_ring[ring_idx];
1225         q_vector->tx.ring->q_vector = q_vector;
1226         q_vector->tx.count++;
1227         q_vector->itr_val = adapter->tx_itr_setting;
1228         q_vector->tx.work_limit = adapter->tx_work_limit;
1229         if (q_vector->itr_val && q_vector->itr_val <= 3)
1230                 q_vector->itr_val = IGB_START_ITR;
1231 }
1232
1233 /**
1234  * igb_map_ring_to_vector - maps allocated queues to vectors
1235  *
1236  * This function maps the recently allocated queues to vectors.
1237  **/
1238 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1239 {
1240         int i;
1241         int v_idx = 0;
1242
1243         if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1244             (adapter->num_q_vectors < adapter->num_tx_queues))
1245                 return -ENOMEM;
1246
1247         if (adapter->num_q_vectors >=
1248             (adapter->num_rx_queues + adapter->num_tx_queues)) {
1249                 for (i = 0; i < adapter->num_rx_queues; i++)
1250                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1251                 for (i = 0; i < adapter->num_tx_queues; i++)
1252                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1253         } else {
1254                 for (i = 0; i < adapter->num_rx_queues; i++) {
1255                         if (i < adapter->num_tx_queues)
1256                                 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1257                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1258                 }
1259                 for (; i < adapter->num_tx_queues; i++)
1260                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1261         }
1262         return 0;
1263 }
1264
1265 /**
1266  * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1267  *
1268  * This function initializes the interrupts and allocates all of the queues.
1269  **/
1270 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1271 {
1272         struct pci_dev *pdev = adapter->pdev;
1273         int err;
1274
1275         igb_set_interrupt_capability(adapter);
1276
1277         err = igb_alloc_q_vectors(adapter);
1278         if (err) {
1279                 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for vectors\n");
1280                 goto err_alloc_q_vectors;
1281         }
1282
1283         err = igb_alloc_queues(adapter);
1284         if (err) {
1285                 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for queues\n");
1286                 goto err_alloc_queues;
1287         }
1288
1289         err = igb_map_ring_to_vector(adapter);
1290         if (err) {
1291                 dev_err(pci_dev_to_dev(pdev), "Invalid q_vector to ring mapping\n");
1292                 goto err_map_queues;
1293         }
1294
1295
1296         return 0;
1297 err_map_queues:
1298         igb_free_queues(adapter);
1299 err_alloc_queues:
1300         igb_free_q_vectors(adapter);
1301 err_alloc_q_vectors:
1302         igb_reset_interrupt_capability(adapter);
1303         return err;
1304 }
1305
1306 /**
1307  * igb_request_irq - initialize interrupts
1308  *
1309  * Attempts to configure interrupts using the best available
1310  * capabilities of the hardware and kernel.
1311  **/
1312 static int igb_request_irq(struct igb_adapter *adapter)
1313 {
1314         struct net_device *netdev = adapter->netdev;
1315         struct pci_dev *pdev = adapter->pdev;
1316         int err = 0;
1317
1318         if (adapter->msix_entries) {
1319                 err = igb_request_msix(adapter);
1320                 if (!err)
1321                         goto request_done;
1322                 /* fall back to MSI */
1323                 igb_clear_interrupt_scheme(adapter);
1324                 igb_reset_sriov_capability(adapter);
1325                 if (!pci_enable_msi(pdev))
1326                         adapter->flags |= IGB_FLAG_HAS_MSI;
1327                 igb_free_all_tx_resources(adapter);
1328                 igb_free_all_rx_resources(adapter);
1329                 adapter->num_tx_queues = 1;
1330                 adapter->num_rx_queues = 1;
1331                 adapter->num_q_vectors = 1;
1332                 err = igb_alloc_q_vectors(adapter);
1333                 if (err) {
1334                         dev_err(pci_dev_to_dev(pdev),
1335                                 "Unable to allocate memory for vectors\n");
1336                         goto request_done;
1337                 }
1338                 err = igb_alloc_queues(adapter);
1339                 if (err) {
1340                         dev_err(pci_dev_to_dev(pdev),
1341                                 "Unable to allocate memory for queues\n");
1342                         igb_free_q_vectors(adapter);
1343                         goto request_done;
1344                 }
1345                 igb_setup_all_tx_resources(adapter);
1346                 igb_setup_all_rx_resources(adapter);
1347         }
1348
1349         igb_assign_vector(adapter->q_vector[0], 0);
1350
1351         if (adapter->flags & IGB_FLAG_HAS_MSI) {
1352                 err = request_irq(pdev->irq, &igb_intr_msi, 0,
1353                                   netdev->name, adapter);
1354                 if (!err)
1355                         goto request_done;
1356
1357                 /* fall back to legacy interrupts */
1358                 igb_reset_interrupt_capability(adapter);
1359                 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1360         }
1361
1362         err = request_irq(pdev->irq, &igb_intr, IRQF_SHARED,
1363                           netdev->name, adapter);
1364
1365         if (err)
1366                 dev_err(pci_dev_to_dev(pdev), "Error %d getting interrupt\n",
1367                         err);
1368
1369 request_done:
1370         return err;
1371 }
1372
1373 static void igb_free_irq(struct igb_adapter *adapter)
1374 {
1375         if (adapter->msix_entries) {
1376                 int vector = 0, i;
1377
1378                 free_irq(adapter->msix_entries[vector++].vector, adapter);
1379
1380                 for (i = 0; i < adapter->num_q_vectors; i++)
1381                         free_irq(adapter->msix_entries[vector++].vector,
1382                                  adapter->q_vector[i]);
1383         } else {
1384                 free_irq(adapter->pdev->irq, adapter);
1385         }
1386 }
1387
1388 /**
1389  * igb_irq_disable - Mask off interrupt generation on the NIC
1390  * @adapter: board private structure
1391  **/
1392 static void igb_irq_disable(struct igb_adapter *adapter)
1393 {
1394         struct e1000_hw *hw = &adapter->hw;
1395
1396         /*
1397          * we need to be careful when disabling interrupts.  The VFs are also
1398          * mapped into these registers and so clearing the bits can cause
1399          * issues on the VF drivers so we only need to clear what we set
1400          */
1401         if (adapter->msix_entries) {
1402                 u32 regval = E1000_READ_REG(hw, E1000_EIAM);
1403                 E1000_WRITE_REG(hw, E1000_EIAM, regval & ~adapter->eims_enable_mask);
1404                 E1000_WRITE_REG(hw, E1000_EIMC, adapter->eims_enable_mask);
1405                 regval = E1000_READ_REG(hw, E1000_EIAC);
1406                 E1000_WRITE_REG(hw, E1000_EIAC, regval & ~adapter->eims_enable_mask);
1407         }
1408
1409         E1000_WRITE_REG(hw, E1000_IAM, 0);
1410         E1000_WRITE_REG(hw, E1000_IMC, ~0);
1411         E1000_WRITE_FLUSH(hw);
1412
1413         if (adapter->msix_entries) {
1414                 int vector = 0, i;
1415
1416                 synchronize_irq(adapter->msix_entries[vector++].vector);
1417
1418                 for (i = 0; i < adapter->num_q_vectors; i++)
1419                         synchronize_irq(adapter->msix_entries[vector++].vector);
1420         } else {
1421                 synchronize_irq(adapter->pdev->irq);
1422         }
1423 }
1424
1425 /**
1426  * igb_irq_enable - Enable default interrupt generation settings
1427  * @adapter: board private structure
1428  **/
1429 static void igb_irq_enable(struct igb_adapter *adapter)
1430 {
1431         struct e1000_hw *hw = &adapter->hw;
1432
1433         if (adapter->msix_entries) {
1434                 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1435                 u32 regval = E1000_READ_REG(hw, E1000_EIAC);
1436                 E1000_WRITE_REG(hw, E1000_EIAC, regval | adapter->eims_enable_mask);
1437                 regval = E1000_READ_REG(hw, E1000_EIAM);
1438                 E1000_WRITE_REG(hw, E1000_EIAM, regval | adapter->eims_enable_mask);
1439                 E1000_WRITE_REG(hw, E1000_EIMS, adapter->eims_enable_mask);
1440                 if (adapter->vfs_allocated_count) {
1441                         E1000_WRITE_REG(hw, E1000_MBVFIMR, 0xFF);
1442                         ims |= E1000_IMS_VMMB;
1443                         /* For I350 device only enable MDD interrupts*/
1444                         if ((adapter->mdd) &&
1445                             (adapter->hw.mac.type == e1000_i350))
1446                                 ims |= E1000_IMS_MDDET;
1447                 }
1448                 E1000_WRITE_REG(hw, E1000_IMS, ims);
1449         } else {
1450                 E1000_WRITE_REG(hw, E1000_IMS, IMS_ENABLE_MASK |
1451                                 E1000_IMS_DRSTA);
1452                 E1000_WRITE_REG(hw, E1000_IAM, IMS_ENABLE_MASK |
1453                                 E1000_IMS_DRSTA);
1454         }
1455 }
1456
1457 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1458 {
1459         struct e1000_hw *hw = &adapter->hw;
1460         u16 vid = adapter->hw.mng_cookie.vlan_id;
1461         u16 old_vid = adapter->mng_vlan_id;
1462
1463         if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1464                 /* add VID to filter table */
1465                 igb_vfta_set(adapter, vid, TRUE);
1466                 adapter->mng_vlan_id = vid;
1467         } else {
1468                 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1469         }
1470
1471         if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1472             (vid != old_vid) &&
1473 #ifdef HAVE_VLAN_RX_REGISTER
1474             !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1475 #else
1476             !test_bit(old_vid, adapter->active_vlans)) {
1477 #endif
1478                 /* remove VID from filter table */
1479                 igb_vfta_set(adapter, old_vid, FALSE);
1480         }
1481 }
1482
1483 /**
1484  * igb_release_hw_control - release control of the h/w to f/w
1485  * @adapter: address of board private structure
1486  *
1487  * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1488  * For ASF and Pass Through versions of f/w this means that the
1489  * driver is no longer loaded.
1490  *
1491  **/
1492 static void igb_release_hw_control(struct igb_adapter *adapter)
1493 {
1494         struct e1000_hw *hw = &adapter->hw;
1495         u32 ctrl_ext;
1496
1497         /* Let firmware take over control of h/w */
1498         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1499         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1500                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1501 }
1502
1503 /**
1504  * igb_get_hw_control - get control of the h/w from f/w
1505  * @adapter: address of board private structure
1506  *
1507  * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1508  * For ASF and Pass Through versions of f/w this means that
1509  * the driver is loaded.
1510  *
1511  **/
1512 static void igb_get_hw_control(struct igb_adapter *adapter)
1513 {
1514         struct e1000_hw *hw = &adapter->hw;
1515         u32 ctrl_ext;
1516
1517         /* Let firmware know the driver has taken over */
1518         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1519         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1520                         ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1521 }
1522
1523 /**
1524  * igb_configure - configure the hardware for RX and TX
1525  * @adapter: private board structure
1526  **/
1527 static void igb_configure(struct igb_adapter *adapter)
1528 {
1529         struct net_device *netdev = adapter->netdev;
1530         int i;
1531
1532         igb_get_hw_control(adapter);
1533         igb_set_rx_mode(netdev);
1534
1535         igb_restore_vlan(adapter);
1536
1537         igb_setup_tctl(adapter);
1538         igb_setup_mrqc(adapter);
1539         igb_setup_rctl(adapter);
1540
1541         igb_configure_tx(adapter);
1542         igb_configure_rx(adapter);
1543
1544         e1000_rx_fifo_flush_82575(&adapter->hw);
1545 #ifdef CONFIG_NETDEVICES_MULTIQUEUE
1546
1547         if (adapter->num_tx_queues > 1)
1548                 netdev->features |= NETIF_F_MULTI_QUEUE;
1549         else
1550                 netdev->features &= ~NETIF_F_MULTI_QUEUE;
1551 #endif
1552
1553         /* call igb_desc_unused which always leaves
1554          * at least 1 descriptor unused to make sure
1555          * next_to_use != next_to_clean */
1556         for (i = 0; i < adapter->num_rx_queues; i++) {
1557                 struct igb_ring *ring = adapter->rx_ring[i];
1558                 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
1559         }
1560 }
1561
1562 /**
1563  * igb_power_up_link - Power up the phy/serdes link
1564  * @adapter: address of board private structure
1565  **/
1566 void igb_power_up_link(struct igb_adapter *adapter)
1567 {
1568         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1569                 e1000_power_up_phy(&adapter->hw);
1570         else
1571                 e1000_power_up_fiber_serdes_link(&adapter->hw);
1572
1573         e1000_phy_hw_reset(&adapter->hw);
1574 }
1575
1576 /**
1577  * igb_power_down_link - Power down the phy/serdes link
1578  * @adapter: address of board private structure
1579  */
1580 static void igb_power_down_link(struct igb_adapter *adapter)
1581 {
1582         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1583                 e1000_power_down_phy(&adapter->hw);
1584         else
1585                 e1000_shutdown_fiber_serdes_link(&adapter->hw);
1586 }
1587
1588 /**
1589  * igb_up - Open the interface and prepare it to handle traffic
1590  * @adapter: board private structure
1591  **/
1592 int igb_up(struct igb_adapter *adapter)
1593 {
1594         struct e1000_hw *hw = &adapter->hw;
1595         int i;
1596
1597         /* hardware has been reset, we need to reload some things */
1598         igb_configure(adapter);
1599
1600         clear_bit(__IGB_DOWN, &adapter->state);
1601
1602         for (i = 0; i < adapter->num_q_vectors; i++)
1603                 napi_enable(&(adapter->q_vector[i]->napi));
1604
1605         if (adapter->msix_entries)
1606                 igb_configure_msix(adapter);
1607         else
1608                 igb_assign_vector(adapter->q_vector[0], 0);
1609
1610         igb_configure_lli(adapter);
1611
1612         /* Clear any pending interrupts. */
1613         E1000_READ_REG(hw, E1000_ICR);
1614         igb_irq_enable(adapter);
1615
1616         /* notify VFs that reset has been completed */
1617         if (adapter->vfs_allocated_count) {
1618                 u32 reg_data = E1000_READ_REG(hw, E1000_CTRL_EXT);
1619                 reg_data |= E1000_CTRL_EXT_PFRSTD;
1620                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg_data);
1621         }
1622
1623         netif_tx_start_all_queues(adapter->netdev);
1624
1625         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
1626                 schedule_work(&adapter->dma_err_task);
1627         /* start the watchdog. */
1628         hw->mac.get_link_status = 1;
1629         schedule_work(&adapter->watchdog_task);
1630
1631         return 0;
1632 }
1633
1634 void igb_down(struct igb_adapter *adapter)
1635 {
1636         struct net_device *netdev = adapter->netdev;
1637         struct e1000_hw *hw = &adapter->hw;
1638         u32 tctl, rctl;
1639         int i;
1640
1641         /* signal that we're down so the interrupt handler does not
1642          * reschedule our watchdog timer */
1643         set_bit(__IGB_DOWN, &adapter->state);
1644
1645         /* disable receives in the hardware */
1646         rctl = E1000_READ_REG(hw, E1000_RCTL);
1647         E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
1648         /* flush and sleep below */
1649
1650         netif_tx_stop_all_queues(netdev);
1651
1652         /* disable transmits in the hardware */
1653         tctl = E1000_READ_REG(hw, E1000_TCTL);
1654         tctl &= ~E1000_TCTL_EN;
1655         E1000_WRITE_REG(hw, E1000_TCTL, tctl);
1656         /* flush both disables and wait for them to finish */
1657         E1000_WRITE_FLUSH(hw);
1658         usleep_range(10000, 20000);
1659
1660         for (i = 0; i < adapter->num_q_vectors; i++)
1661                 napi_disable(&(adapter->q_vector[i]->napi));
1662
1663         igb_irq_disable(adapter);
1664
1665         del_timer_sync(&adapter->watchdog_timer);
1666         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
1667                 del_timer_sync(&adapter->dma_err_timer);
1668         del_timer_sync(&adapter->phy_info_timer);
1669
1670         netif_carrier_off(netdev);
1671
1672         /* record the stats before reset*/
1673         igb_update_stats(adapter);
1674
1675         adapter->link_speed = 0;
1676         adapter->link_duplex = 0;
1677
1678 #ifdef HAVE_PCI_ERS
1679         if (!pci_channel_offline(adapter->pdev))
1680                 igb_reset(adapter);
1681 #else
1682         igb_reset(adapter);
1683 #endif
1684         igb_clean_all_tx_rings(adapter);
1685         igb_clean_all_rx_rings(adapter);
1686 #ifdef IGB_DCA
1687
1688         /* since we reset the hardware DCA settings were cleared */
1689         igb_setup_dca(adapter);
1690 #endif
1691 }
1692
1693 void igb_reinit_locked(struct igb_adapter *adapter)
1694 {
1695         WARN_ON(in_interrupt());
1696         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1697                 usleep_range(1000, 2000);
1698         igb_down(adapter);
1699         igb_up(adapter);
1700         clear_bit(__IGB_RESETTING, &adapter->state);
1701 }
1702
1703 void igb_reset(struct igb_adapter *adapter)
1704 {
1705         struct pci_dev *pdev = adapter->pdev;
1706         struct e1000_hw *hw = &adapter->hw;
1707         struct e1000_mac_info *mac = &hw->mac;
1708         struct e1000_fc_info *fc = &hw->fc;
1709         u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1710         u16 hwm;
1711
1712         /* Repartition Pba for greater than 9k mtu
1713          * To take effect CTRL.RST is required.
1714          */
1715         switch (mac->type) {
1716         case e1000_i350:
1717         case e1000_82580:
1718                 pba = E1000_READ_REG(hw, E1000_RXPBS);
1719                 pba = e1000_rxpbs_adjust_82580(pba);
1720                 break;
1721         case e1000_82576:
1722                 pba = E1000_READ_REG(hw, E1000_RXPBS);
1723                 pba &= E1000_RXPBS_SIZE_MASK_82576;
1724                 break;
1725         case e1000_82575:
1726         default:
1727                 pba = E1000_PBA_34K;
1728                 break;
1729         }
1730
1731         if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1732             (mac->type < e1000_82576)) {
1733                 /* adjust PBA for jumbo frames */
1734                 E1000_WRITE_REG(hw, E1000_PBA, pba);
1735
1736                 /* To maintain wire speed transmits, the Tx FIFO should be
1737                  * large enough to accommodate two full transmit packets,
1738                  * rounded up to the next 1KB and expressed in KB.  Likewise,
1739                  * the Rx FIFO should be large enough to accommodate at least
1740                  * one full receive packet and is similarly rounded up and
1741                  * expressed in KB. */
1742                 pba = E1000_READ_REG(hw, E1000_PBA);
1743                 /* upper 16 bits has Tx packet buffer allocation size in KB */
1744                 tx_space = pba >> 16;
1745                 /* lower 16 bits has Rx packet buffer allocation size in KB */
1746                 pba &= 0xffff;
1747                 /* the tx fifo also stores 16 bytes of information about the tx
1748                  * but don't include ethernet FCS because hardware appends it */
1749                 min_tx_space = (adapter->max_frame_size +
1750                                 sizeof(union e1000_adv_tx_desc) -
1751                                 ETH_FCS_LEN) * 2;
1752                 min_tx_space = ALIGN(min_tx_space, 1024);
1753                 min_tx_space >>= 10;
1754                 /* software strips receive CRC, so leave room for it */
1755                 min_rx_space = adapter->max_frame_size;
1756                 min_rx_space = ALIGN(min_rx_space, 1024);
1757                 min_rx_space >>= 10;
1758
1759                 /* If current Tx allocation is less than the min Tx FIFO size,
1760                  * and the min Tx FIFO size is less than the current Rx FIFO
1761                  * allocation, take space away from current Rx allocation */
1762                 if (tx_space < min_tx_space &&
1763                     ((min_tx_space - tx_space) < pba)) {
1764                         pba = pba - (min_tx_space - tx_space);
1765
1766                         /* if short on rx space, rx wins and must trump tx
1767                          * adjustment */
1768                         if (pba < min_rx_space)
1769                                 pba = min_rx_space;
1770                 }
1771                 E1000_WRITE_REG(hw, E1000_PBA, pba);
1772         }
1773
1774         /* flow control settings */
1775         /* The high water mark must be low enough to fit one full frame
1776          * (or the size used for early receive) above it in the Rx FIFO.
1777          * Set it to the lower of:
1778          * - 90% of the Rx FIFO size, or
1779          * - the full Rx FIFO size minus one full frame */
1780         hwm = min(((pba << 10) * 9 / 10),
1781                         ((pba << 10) - 2 * adapter->max_frame_size));
1782
1783         fc->high_water = hwm & 0xFFF0;  /* 16-byte granularity */
1784         fc->low_water = fc->high_water - 16;
1785         fc->pause_time = 0xFFFF;
1786         fc->send_xon = 1;
1787         fc->current_mode = fc->requested_mode;
1788
1789         /* disable receive for all VFs and wait one second */
1790         if (adapter->vfs_allocated_count) {
1791                 int i;
1792                 /*
1793                  * Clear all flags except indication that the PF has set
1794                  * the VF MAC addresses administratively
1795                  */
1796                 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1797                         adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
1798
1799                 /* ping all the active vfs to let them know we are going down */
1800                 igb_ping_all_vfs(adapter);
1801
1802                 /* disable transmits and receives */
1803                 E1000_WRITE_REG(hw, E1000_VFRE, 0);
1804                 E1000_WRITE_REG(hw, E1000_VFTE, 0);
1805         }
1806
1807         /* Allow time for pending master requests to run */
1808         e1000_reset_hw(hw);
1809         E1000_WRITE_REG(hw, E1000_WUC, 0);
1810
1811         if (e1000_init_hw(hw))
1812                 dev_err(pci_dev_to_dev(pdev), "Hardware Error\n");
1813
1814         igb_init_dmac(adapter, pba);
1815         /* Re-initialize the thermal sensor on i350 devices. */
1816         if (mac->type == e1000_i350 && hw->bus.func == 0) {
1817                 /*
1818                  * If present, re-initialize the external thermal sensor
1819                  * interface.
1820                  */
1821                 if (adapter->ets)
1822                         e1000_set_i2c_bb(hw);
1823                 e1000_init_thermal_sensor_thresh(hw);
1824         }
1825         if (!netif_running(adapter->netdev))
1826                 igb_power_down_link(adapter);
1827
1828         igb_update_mng_vlan(adapter);
1829
1830         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1831         E1000_WRITE_REG(hw, E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1832
1833         e1000_get_phy_info(hw);
1834 }
1835
1836 #ifdef HAVE_NDO_SET_FEATURES
1837 static netdev_features_t igb_fix_features(struct net_device *netdev,
1838                                           netdev_features_t features)
1839 {
1840         /*
1841          * Since there is no support for separate tx vlan accel
1842          * enabled make sure tx flag is cleared if rx is.
1843          */
1844         if (!(features & NETIF_F_HW_VLAN_RX))
1845                 features &= ~NETIF_F_HW_VLAN_TX;
1846
1847         /* If Rx checksum is disabled, then LRO should also be disabled */
1848         if (!(features & NETIF_F_RXCSUM))
1849                 features &= ~NETIF_F_LRO;
1850
1851         return features;
1852 }
1853
1854 static int igb_set_features(struct net_device *netdev,
1855                             netdev_features_t features)
1856 {
1857         u32 changed = netdev->features ^ features;
1858
1859         if (changed & NETIF_F_HW_VLAN_RX)
1860                 igb_vlan_mode(netdev, features);
1861
1862         return 0;
1863 }
1864
1865 #endif /* HAVE_NDO_SET_FEATURES */
1866 #ifdef HAVE_NET_DEVICE_OPS
1867 static const struct net_device_ops igb_netdev_ops = {
1868         .ndo_open               = igb_open,
1869         .ndo_stop               = igb_close,
1870         .ndo_start_xmit         = igb_xmit_frame,
1871         .ndo_get_stats          = igb_get_stats,
1872         .ndo_set_rx_mode        = igb_set_rx_mode,
1873         .ndo_set_mac_address    = igb_set_mac,
1874         .ndo_change_mtu         = igb_change_mtu,
1875         .ndo_do_ioctl           = igb_ioctl,
1876         .ndo_tx_timeout         = igb_tx_timeout,
1877         .ndo_validate_addr      = eth_validate_addr,
1878         .ndo_vlan_rx_add_vid    = igb_vlan_rx_add_vid,
1879         .ndo_vlan_rx_kill_vid   = igb_vlan_rx_kill_vid,
1880 #ifdef IFLA_VF_MAX
1881         .ndo_set_vf_mac         = igb_ndo_set_vf_mac,
1882         .ndo_set_vf_vlan        = igb_ndo_set_vf_vlan,
1883         .ndo_set_vf_tx_rate     = igb_ndo_set_vf_bw,
1884         .ndo_get_vf_config      = igb_ndo_get_vf_config,
1885 #endif
1886 #ifdef CONFIG_NET_POLL_CONTROLLER
1887         .ndo_poll_controller    = igb_netpoll,
1888 #endif
1889 #ifdef HAVE_NDO_SET_FEATURES
1890         .ndo_fix_features       = igb_fix_features,
1891         .ndo_set_features       = igb_set_features,
1892 #endif
1893 #ifdef HAVE_VLAN_RX_REGISTER
1894         .ndo_vlan_rx_register   = igb_vlan_mode,
1895 #endif
1896 };
1897
1898 #ifdef CONFIG_IGB_VMDQ_NETDEV
1899 static const struct net_device_ops igb_vmdq_ops = {
1900         .ndo_open               = &igb_vmdq_open,
1901         .ndo_stop               = &igb_vmdq_close,
1902         .ndo_start_xmit         = &igb_vmdq_xmit_frame,
1903         .ndo_get_stats          = &igb_vmdq_get_stats,
1904         .ndo_set_rx_mode        = &igb_vmdq_set_rx_mode,
1905         .ndo_validate_addr      = eth_validate_addr,
1906         .ndo_set_mac_address    = &igb_vmdq_set_mac,
1907         .ndo_change_mtu         = &igb_vmdq_change_mtu,
1908         .ndo_tx_timeout         = &igb_vmdq_tx_timeout,
1909         .ndo_vlan_rx_register   = &igb_vmdq_vlan_rx_register,
1910         .ndo_vlan_rx_add_vid    = &igb_vmdq_vlan_rx_add_vid,
1911         .ndo_vlan_rx_kill_vid   = &igb_vmdq_vlan_rx_kill_vid,
1912 };
1913
1914 #endif /* CONFIG_IGB_VMDQ_NETDEV */
1915 #endif /* HAVE_NET_DEVICE_OPS */
1916 #ifdef CONFIG_IGB_VMDQ_NETDEV
1917 void igb_assign_vmdq_netdev_ops(struct net_device *vnetdev)
1918 {
1919 #ifdef HAVE_NET_DEVICE_OPS
1920         vnetdev->netdev_ops = &igb_vmdq_ops;
1921 #else
1922         dev->open = &igb_vmdq_open;
1923         dev->stop = &igb_vmdq_close;
1924         dev->hard_start_xmit = &igb_vmdq_xmit_frame;
1925         dev->get_stats = &igb_vmdq_get_stats;
1926 #ifdef HAVE_SET_RX_MODE
1927         dev->set_rx_mode = &igb_vmdq_set_rx_mode;
1928 #endif
1929         dev->set_multicast_list = &igb_vmdq_set_rx_mode;
1930         dev->set_mac_address = &igb_vmdq_set_mac;
1931         dev->change_mtu = &igb_vmdq_change_mtu;
1932 #ifdef HAVE_TX_TIMEOUT
1933         dev->tx_timeout = &igb_vmdq_tx_timeout;
1934 #endif
1935 #ifdef NETIF_F_HW_VLAN_TX
1936         dev->vlan_rx_register = &igb_vmdq_vlan_rx_register;
1937         dev->vlan_rx_add_vid = &igb_vmdq_vlan_rx_add_vid;
1938         dev->vlan_rx_kill_vid = &igb_vmdq_vlan_rx_kill_vid;
1939 #endif
1940 #endif
1941         igb_vmdq_set_ethtool_ops(vnetdev);
1942         vnetdev->watchdog_timeo = 5 * HZ;
1943
1944 }
1945
1946 int igb_init_vmdq_netdevs(struct igb_adapter *adapter)
1947 {
1948         int pool, err = 0, base_queue;
1949         struct net_device *vnetdev;
1950         struct igb_vmdq_adapter *vmdq_adapter;
1951
1952         for (pool = 1; pool < adapter->vmdq_pools; pool++) {
1953                 int qpp = (!adapter->rss_queues ? 1 : adapter->rss_queues);
1954                 base_queue = pool * qpp;
1955                 vnetdev = alloc_etherdev(sizeof(struct igb_vmdq_adapter));
1956                 if (!vnetdev) {
1957                         err = -ENOMEM;
1958                         break;
1959                 }
1960                 vmdq_adapter = netdev_priv(vnetdev);
1961                 vmdq_adapter->vnetdev = vnetdev;
1962                 vmdq_adapter->real_adapter = adapter;
1963                 vmdq_adapter->rx_ring = adapter->rx_ring[base_queue];
1964                 vmdq_adapter->tx_ring = adapter->tx_ring[base_queue];
1965                 igb_assign_vmdq_netdev_ops(vnetdev);
1966                 snprintf(vnetdev->name, IFNAMSIZ, "%sv%d",
1967                          adapter->netdev->name, pool);
1968                 vnetdev->features = adapter->netdev->features;
1969 #ifdef HAVE_NETDEV_VLAN_FEATURES
1970                 vnetdev->vlan_features = adapter->netdev->vlan_features;
1971 #endif
1972                 adapter->vmdq_netdev[pool-1] = vnetdev;
1973                 err = register_netdev(vnetdev);
1974                 if (err)
1975                         break;
1976         }
1977         return err;
1978 }
1979
1980 int igb_remove_vmdq_netdevs(struct igb_adapter *adapter)
1981 {
1982         int pool, err = 0;
1983
1984         for (pool = 1; pool < adapter->vmdq_pools; pool++) {
1985                 unregister_netdev(adapter->vmdq_netdev[pool-1]);
1986                 free_netdev(adapter->vmdq_netdev[pool-1]);
1987                 adapter->vmdq_netdev[pool-1] = NULL;
1988         }
1989         return err;
1990 }
1991 #endif /* CONFIG_IGB_VMDQ_NETDEV */
1992
1993 /**
1994  * igb_probe - Device Initialization Routine
1995  * @pdev: PCI device information struct
1996  * @ent: entry in igb_pci_tbl
1997  *
1998  * Returns 0 on success, negative on failure
1999  *
2000  * igb_probe initializes an adapter identified by a pci_dev structure.
2001  * The OS initialization, configuring of the adapter private structure,
2002  * and a hardware reset occur.
2003  **/
2004 static int __devinit igb_probe(struct pci_dev *pdev,
2005                                const struct pci_device_id *ent)
2006 {
2007         struct net_device *netdev;
2008         struct igb_adapter *adapter;
2009         struct e1000_hw *hw;
2010         u16 eeprom_data = 0;
2011         u8 pba_str[E1000_PBANUM_LENGTH];
2012         s32 ret_val;
2013         static int global_quad_port_a; /* global quad port a indication */
2014         int i, err, pci_using_dac;
2015         static int cards_found;
2016
2017         err = pci_enable_device_mem(pdev);
2018         if (err)
2019                 return err;
2020
2021         pci_using_dac = 0;
2022         err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
2023         if (!err) {
2024                 err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
2025                 if (!err)
2026                         pci_using_dac = 1;
2027         } else {
2028                 err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
2029                 if (err) {
2030                         err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
2031                         if (err) {
2032                                 IGB_ERR("No usable DMA configuration, "
2033                                         "aborting\n");
2034                                 goto err_dma;
2035                         }
2036                 }
2037         }
2038
2039 #ifndef HAVE_ASPM_QUIRKS
2040         /* 82575 requires that the pci-e link partner disable the L0s state */
2041         switch (pdev->device) {
2042         case E1000_DEV_ID_82575EB_COPPER:
2043         case E1000_DEV_ID_82575EB_FIBER_SERDES:
2044         case E1000_DEV_ID_82575GB_QUAD_COPPER:
2045                 pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S);
2046         default:
2047                 break;
2048         }
2049
2050 #endif /* HAVE_ASPM_QUIRKS */
2051         err = pci_request_selected_regions(pdev,
2052                                            pci_select_bars(pdev,
2053                                                            IORESOURCE_MEM),
2054                                            igb_driver_name);
2055         if (err)
2056                 goto err_pci_reg;
2057
2058         pci_enable_pcie_error_reporting(pdev);
2059
2060         pci_set_master(pdev);
2061
2062         err = -ENOMEM;
2063 #ifdef HAVE_TX_MQ
2064         netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
2065                                    IGB_MAX_TX_QUEUES);
2066 #else
2067         netdev = alloc_etherdev(sizeof(struct igb_adapter));
2068 #endif /* HAVE_TX_MQ */
2069         if (!netdev)
2070                 goto err_alloc_etherdev;
2071
2072         SET_MODULE_OWNER(netdev);
2073         SET_NETDEV_DEV(netdev, &pdev->dev);
2074
2075         pci_set_drvdata(pdev, netdev);
2076         adapter = netdev_priv(netdev);
2077         adapter->netdev = netdev;
2078         adapter->pdev = pdev;
2079         hw = &adapter->hw;
2080         hw->back = adapter;
2081         adapter->port_num = hw->bus.func;
2082         adapter->msg_enable = (1 << debug) - 1;
2083
2084 #ifdef HAVE_PCI_ERS
2085         err = pci_save_state(pdev);
2086         if (err)
2087                 goto err_ioremap;
2088 #endif
2089         err = -EIO;
2090         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
2091                               pci_resource_len(pdev, 0));
2092         if (!hw->hw_addr)
2093                 goto err_ioremap;
2094
2095 #ifdef HAVE_NET_DEVICE_OPS
2096         netdev->netdev_ops = &igb_netdev_ops;
2097 #else /* HAVE_NET_DEVICE_OPS */
2098         netdev->open = &igb_open;
2099         netdev->stop = &igb_close;
2100         netdev->get_stats = &igb_get_stats;
2101 #ifdef HAVE_SET_RX_MODE
2102         netdev->set_rx_mode = &igb_set_rx_mode;
2103 #endif
2104         netdev->set_multicast_list = &igb_set_rx_mode;
2105         netdev->set_mac_address = &igb_set_mac;
2106         netdev->change_mtu = &igb_change_mtu;
2107         netdev->do_ioctl = &igb_ioctl;
2108 #ifdef HAVE_TX_TIMEOUT
2109         netdev->tx_timeout = &igb_tx_timeout;
2110 #endif
2111         netdev->vlan_rx_register = igb_vlan_mode;
2112         netdev->vlan_rx_add_vid = igb_vlan_rx_add_vid;
2113         netdev->vlan_rx_kill_vid = igb_vlan_rx_kill_vid;
2114 #ifdef CONFIG_NET_POLL_CONTROLLER
2115         netdev->poll_controller = igb_netpoll;
2116 #endif
2117         netdev->hard_start_xmit = &igb_xmit_frame;
2118 #endif /* HAVE_NET_DEVICE_OPS */
2119         igb_set_ethtool_ops(netdev);
2120 #ifdef HAVE_TX_TIMEOUT
2121         netdev->watchdog_timeo = 5 * HZ;
2122 #endif
2123
2124         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
2125
2126         adapter->bd_number = cards_found;
2127
2128         /* setup the private structure */
2129         err = igb_sw_init(adapter);
2130         if (err)
2131                 goto err_sw_init;
2132
2133         e1000_get_bus_info(hw);
2134
2135         hw->phy.autoneg_wait_to_complete = FALSE;
2136         hw->mac.adaptive_ifs = FALSE;
2137
2138         /* Copper options */
2139         if (hw->phy.media_type == e1000_media_type_copper) {
2140 #ifdef ETH_TP_MDI_X
2141                 hw->phy.mdix = ETH_TP_MDI_INVALID;
2142 #else
2143                 hw->phy.mdix = AUTO_ALL_MODES;
2144 #endif /* ETH_TP_MDI_X */
2145                 hw->phy.disable_polarity_correction = FALSE;
2146                 hw->phy.ms_type = e1000_ms_hw_default;
2147         }
2148
2149         if (e1000_check_reset_block(hw))
2150                 dev_info(pci_dev_to_dev(pdev),
2151                         "PHY reset is blocked due to SOL/IDER session.\n");
2152
2153         /*
2154          * features is initialized to 0 in allocation, it might have bits
2155          * set by igb_sw_init so we should use an or instead of an
2156          * assignment.
2157          */
2158         netdev->features |= NETIF_F_SG |
2159                             NETIF_F_IP_CSUM |
2160 #ifdef NETIF_F_IPV6_CSUM
2161                             NETIF_F_IPV6_CSUM |
2162 #endif
2163 #ifdef NETIF_F_TSO
2164                             NETIF_F_TSO |
2165 #ifdef NETIF_F_TSO6
2166                             NETIF_F_TSO6 |
2167 #endif
2168 #endif /* NETIF_F_TSO */
2169 #ifdef NETIF_F_RXHASH
2170                             NETIF_F_RXHASH |
2171 #endif
2172 #ifdef HAVE_NDO_SET_FEATURES
2173                             NETIF_F_RXCSUM |
2174 #endif
2175                             NETIF_F_HW_VLAN_RX |
2176                             NETIF_F_HW_VLAN_TX;
2177
2178 #ifdef HAVE_NDO_SET_FEATURES
2179         /* copy netdev features into list of user selectable features */
2180         netdev->hw_features |= netdev->features;
2181 #ifndef IGB_NO_LRO
2182
2183         /* give us the option of enabling LRO later */
2184         netdev->hw_features |= NETIF_F_LRO;
2185 #endif
2186 #else
2187 #ifdef NETIF_F_GRO
2188
2189         /* this is only needed on kernels prior to 2.6.39 */
2190         netdev->features |= NETIF_F_GRO;
2191 #endif
2192 #endif
2193
2194         /* set this bit last since it cannot be part of hw_features */
2195         netdev->features |= NETIF_F_HW_VLAN_FILTER;
2196
2197 #ifdef HAVE_NETDEV_VLAN_FEATURES
2198         netdev->vlan_features |= NETIF_F_TSO |
2199                                  NETIF_F_TSO6 |
2200                                  NETIF_F_IP_CSUM |
2201                                  NETIF_F_IPV6_CSUM |
2202                                  NETIF_F_SG;
2203
2204 #endif
2205         if (pci_using_dac)
2206                 netdev->features |= NETIF_F_HIGHDMA;
2207
2208         if (hw->mac.type >= e1000_82576)
2209                 netdev->features |= NETIF_F_SCTP_CSUM;
2210
2211         adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
2212
2213         /* before reading the NVM, reset the controller to put the device in a
2214          * known good starting state */
2215         e1000_reset_hw(hw);
2216
2217         /* make sure the NVM is good */
2218         if (e1000_validate_nvm_checksum(hw) < 0) {
2219                 dev_err(pci_dev_to_dev(pdev), "The NVM Checksum Is Not"
2220                         " Valid\n");
2221                 err = -EIO;
2222                 goto err_eeprom;
2223         }
2224
2225         /* copy the MAC address out of the NVM */
2226         if (e1000_read_mac_addr(hw))
2227                 dev_err(pci_dev_to_dev(pdev), "NVM Read Error\n");
2228         memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2229 #ifdef ETHTOOL_GPERMADDR
2230         memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
2231
2232         if (!is_valid_ether_addr(netdev->perm_addr)) {
2233 #else
2234         if (!is_valid_ether_addr(netdev->dev_addr)) {
2235 #endif
2236                 dev_err(pci_dev_to_dev(pdev), "Invalid MAC Address\n");
2237                 err = -EIO;
2238                 goto err_eeprom;
2239         }
2240
2241         memcpy(&adapter->mac_table[0].addr, hw->mac.addr, netdev->addr_len);
2242         adapter->mac_table[0].queue = adapter->vfs_allocated_count;
2243         adapter->mac_table[0].state = (IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE);
2244         igb_rar_set(adapter, 0);
2245
2246         /* get firmware version for ethtool -i */
2247         e1000_read_nvm(&adapter->hw, 5, 1, &adapter->fw_version);
2248         setup_timer(&adapter->watchdog_timer, &igb_watchdog,
2249                     (unsigned long) adapter);
2250         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2251                 setup_timer(&adapter->dma_err_timer, &igb_dma_err_timer,
2252                             (unsigned long) adapter);
2253         setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
2254                     (unsigned long) adapter);
2255
2256         INIT_WORK(&adapter->reset_task, igb_reset_task);
2257         INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2258         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2259                 INIT_WORK(&adapter->dma_err_task, igb_dma_err_task);
2260
2261         /* Initialize link properties that are user-changeable */
2262         adapter->fc_autoneg = true;
2263         hw->mac.autoneg = true;
2264         hw->phy.autoneg_advertised = 0x2f;
2265
2266         hw->fc.requested_mode = e1000_fc_default;
2267         hw->fc.current_mode = e1000_fc_default;
2268
2269         e1000_validate_mdi_setting(hw);
2270
2271         /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2272          * enable the ACPI Magic Packet filter
2273          */
2274
2275         if (hw->bus.func == 0)
2276                 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
2277         else if (hw->mac.type >= e1000_82580)
2278                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2279                                  NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2280                                  &eeprom_data);
2281         else if (hw->bus.func == 1)
2282                 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
2283
2284         if (eeprom_data & IGB_EEPROM_APME)
2285                 adapter->eeprom_wol |= E1000_WUFC_MAG;
2286
2287         /* now that we have the eeprom settings, apply the special cases where
2288          * the eeprom may be wrong or the board simply won't support wake on
2289          * lan on a particular port */
2290         switch (pdev->device) {
2291         case E1000_DEV_ID_82575GB_QUAD_COPPER:
2292                 adapter->eeprom_wol = 0;
2293                 break;
2294         case E1000_DEV_ID_82575EB_FIBER_SERDES:
2295         case E1000_DEV_ID_82576_FIBER:
2296         case E1000_DEV_ID_82576_SERDES:
2297                 /* Wake events only supported on port A for dual fiber
2298                  * regardless of eeprom setting */
2299                 if (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_FUNC_1)
2300                         adapter->eeprom_wol = 0;
2301                 break;
2302         case E1000_DEV_ID_82576_QUAD_COPPER:
2303         case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
2304                 /* if quad port adapter, disable WoL on all but port A */
2305                 if (global_quad_port_a != 0)
2306                         adapter->eeprom_wol = 0;
2307                 else
2308                         adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2309                 /* Reset for multiple quad port adapters */
2310                 if (++global_quad_port_a == 4)
2311                         global_quad_port_a = 0;
2312                 break;
2313         }
2314
2315         /* initialize the wol settings based on the eeprom settings */
2316         adapter->wol = adapter->eeprom_wol;
2317         device_set_wakeup_enable(pci_dev_to_dev(adapter->pdev), adapter->wol);
2318
2319         /* reset the hardware with the new settings */
2320         igb_reset(adapter);
2321
2322         /* let the f/w know that the h/w is now under the control of the
2323          * driver. */
2324         igb_get_hw_control(adapter);
2325
2326         strncpy(netdev->name, "eth%d", IFNAMSIZ);
2327         err = register_netdev(netdev);
2328         if (err)
2329                 goto err_register;
2330
2331 #ifdef CONFIG_IGB_VMDQ_NETDEV
2332         err = igb_init_vmdq_netdevs(adapter);
2333         if (err)
2334                 goto err_register;
2335 #endif
2336         /* carrier off reporting is important to ethtool even BEFORE open */
2337         netif_carrier_off(netdev);
2338
2339 #ifdef IGB_DCA
2340         if (dca_add_requester(&pdev->dev) == E1000_SUCCESS) {
2341                 adapter->flags |= IGB_FLAG_DCA_ENABLED;
2342                 dev_info(pci_dev_to_dev(pdev), "DCA enabled\n");
2343                 igb_setup_dca(adapter);
2344         }
2345
2346 #endif
2347 #ifdef HAVE_HW_TIME_STAMP
2348         /* do hw tstamp init after resetting */
2349         igb_init_hw_timer(adapter);
2350
2351 #endif
2352         dev_info(pci_dev_to_dev(pdev), "Intel(R) Gigabit Ethernet Network Connection\n");
2353         /* print bus type/speed/width info */
2354         dev_info(pci_dev_to_dev(pdev), "%s: (PCIe:%s:%s) ",
2355                  netdev->name,
2356                  ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5GT/s" :
2357                   (hw->bus.speed == e1000_bus_speed_5000) ? "5.0GT/s" :
2358                                                             "unknown"),
2359                  ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4\n" :
2360                   (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2\n" :
2361                   (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1\n" :
2362                    "unknown"));
2363         dev_info(pci_dev_to_dev(pdev), "%s: MAC: ", netdev->name);
2364         for (i = 0; i < 6; i++)
2365                 printk("%2.2x%c", netdev->dev_addr[i], i == 5 ? '\n' : ':');
2366
2367         ret_val = e1000_read_pba_string(hw, pba_str, E1000_PBANUM_LENGTH);
2368         if (ret_val)
2369                 strncpy(pba_str, "Unknown", sizeof(pba_str) - 1);
2370         dev_info(pci_dev_to_dev(pdev), "%s: PBA No: %s\n", netdev->name,
2371                  pba_str);
2372
2373
2374         /* Initialize the thermal sensor on i350 devices. */
2375         if (hw->mac.type == e1000_i350 && hw->bus.func == 0) {
2376                 u16 ets_word;
2377
2378                 /*
2379                  * Read the NVM to determine if this i350 device supports an
2380                  * external thermal sensor.
2381                  */
2382                 e1000_read_nvm(hw, NVM_ETS_CFG, 1, &ets_word);
2383                 if (ets_word != 0x0000 && ets_word != 0xFFFF)
2384                         adapter->ets = true;
2385                 else
2386                         adapter->ets = false;
2387 #ifdef IGB_SYSFS
2388                 igb_sysfs_init(adapter);
2389 #else
2390 #ifdef IGB_PROCFS
2391                 igb_procfs_init(adapter);
2392 #endif /* IGB_PROCFS */
2393 #endif /* IGB_SYSFS */
2394         } else {
2395                 adapter->ets = false;
2396         }
2397
2398         switch (hw->mac.type) {
2399         case e1000_i350:
2400                 /* Enable EEE for internal copper PHY devices */
2401                 if (hw->phy.media_type == e1000_media_type_copper)
2402                         e1000_set_eee_i350(hw);
2403
2404                 /* send driver version info to firmware */
2405                 igb_init_fw(adapter);
2406                 break;
2407         default:
2408                 break;
2409         }
2410 #ifndef IGB_NO_LRO
2411         if (netdev->features & NETIF_F_LRO)
2412                 dev_info(pci_dev_to_dev(pdev), "Internal LRO is enabled \n");
2413         else
2414                 dev_info(pci_dev_to_dev(pdev), "LRO is disabled \n");
2415 #endif
2416         dev_info(pci_dev_to_dev(pdev),
2417                  "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2418                  adapter->msix_entries ? "MSI-X" :
2419                  (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
2420                  adapter->num_rx_queues, adapter->num_tx_queues);
2421
2422         cards_found++;
2423
2424         pm_runtime_put_noidle(&pdev->dev);
2425         return 0;
2426
2427 err_register:
2428         igb_release_hw_control(adapter);
2429 err_eeprom:
2430         if (!e1000_check_reset_block(hw))
2431                 e1000_phy_hw_reset(hw);
2432
2433         if (hw->flash_address)
2434                 iounmap(hw->flash_address);
2435 err_sw_init:
2436         igb_clear_interrupt_scheme(adapter);
2437         igb_reset_sriov_capability(adapter);
2438         iounmap(hw->hw_addr);
2439 err_ioremap:
2440         free_netdev(netdev);
2441 err_alloc_etherdev:
2442         pci_release_selected_regions(pdev,
2443                                      pci_select_bars(pdev, IORESOURCE_MEM));
2444 err_pci_reg:
2445 err_dma:
2446         pci_disable_device(pdev);
2447         return err;
2448 }
2449
2450 /**
2451  * igb_remove - Device Removal Routine
2452  * @pdev: PCI device information struct
2453  *
2454  * igb_remove is called by the PCI subsystem to alert the driver
2455  * that it should release a PCI device.  The could be caused by a
2456  * Hot-Plug event, or because the driver is going to be removed from
2457  * memory.
2458  **/
2459 static void __devexit igb_remove(struct pci_dev *pdev)
2460 {
2461         struct net_device *netdev = pci_get_drvdata(pdev);
2462         struct igb_adapter *adapter = netdev_priv(netdev);
2463         struct e1000_hw *hw = &adapter->hw;
2464
2465         pm_runtime_get_noresume(&pdev->dev);
2466
2467         /* flush_scheduled work may reschedule our watchdog task, so
2468          * explicitly disable watchdog tasks from being rescheduled  */
2469         set_bit(__IGB_DOWN, &adapter->state);
2470         del_timer_sync(&adapter->watchdog_timer);
2471         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2472                 del_timer_sync(&adapter->dma_err_timer);
2473         del_timer_sync(&adapter->phy_info_timer);
2474
2475         flush_scheduled_work();
2476
2477 #ifdef IGB_DCA
2478         if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2479                 dev_info(pci_dev_to_dev(pdev), "DCA disabled\n");
2480                 dca_remove_requester(&pdev->dev);
2481                 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2482                 E1000_WRITE_REG(hw, E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_DISABLE);
2483         }
2484 #endif
2485
2486         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
2487          * would have already happened in close and is redundant. */
2488         igb_release_hw_control(adapter);
2489
2490         unregister_netdev(netdev);
2491 #ifdef CONFIG_IGB_VMDQ_NETDEV
2492         igb_remove_vmdq_netdevs(adapter);
2493 #endif
2494
2495         igb_clear_interrupt_scheme(adapter);
2496         igb_reset_sriov_capability(adapter);
2497
2498         iounmap(hw->hw_addr);
2499         if (hw->flash_address)
2500                 iounmap(hw->flash_address);
2501         pci_release_selected_regions(pdev,
2502                                      pci_select_bars(pdev, IORESOURCE_MEM));
2503
2504         kfree(adapter->mac_table);
2505         kfree(adapter->shadow_vfta);
2506         free_netdev(netdev);
2507
2508         pci_disable_pcie_error_reporting(pdev);
2509
2510         pci_disable_device(pdev);
2511
2512 #ifdef IGB_SYSFS
2513         igb_sysfs_exit(adapter);
2514 #else
2515 #ifdef IGB_PROCFS
2516         igb_procfs_exit(adapter);
2517 #endif /* IGB_PROCFS */
2518 #endif /* IGB_SYSFS */
2519 }
2520
2521 #ifdef HAVE_HW_TIME_STAMP
2522 /**
2523  * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2524  * @adapter: board private structure to initialize
2525  *
2526  * igb_init_hw_timer initializes the function pointer and values for the hw
2527  * timer found in hardware.
2528  **/
2529 static void igb_init_hw_timer(struct igb_adapter *adapter)
2530 {
2531         struct e1000_hw *hw = &adapter->hw;
2532
2533         switch (hw->mac.type) {
2534         case e1000_i350:
2535         case e1000_82580:
2536                 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2537                 adapter->cycles.read = igb_read_clock;
2538                 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2539                 adapter->cycles.mult = 1;
2540                 /*
2541                  * The 82580 timesync updates the system timer every 8ns by 8ns
2542                  * and the value cannot be shifted.  Instead we need to shift
2543                  * the registers to generate a 64bit timer value.  As a result
2544                  * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2545                  * 24 in order to generate a larger value for synchronization.
2546                  */
2547                 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2548                 /* disable system timer temporarily by setting bit 31 */
2549                 E1000_WRITE_REG(hw, E1000_TSAUXC, 0x80000000);
2550                 E1000_WRITE_FLUSH(hw);
2551
2552                 /* Set registers so that rollover occurs soon to test this. */
2553                 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x00000000);
2554                 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x80000000);
2555                 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x000000FF);
2556                 E1000_WRITE_FLUSH(hw);
2557
2558                 /* enable system timer by clearing bit 31 */
2559                 E1000_WRITE_REG(hw, E1000_TSAUXC, 0x0);
2560                 E1000_WRITE_FLUSH(hw);
2561
2562                 timecounter_init(&adapter->clock,
2563                                  &adapter->cycles,
2564                                  ktime_to_ns(ktime_get_real()));
2565                 /*
2566                  * Synchronize our NIC clock against system wall clock. NIC
2567                  * time stamp reading requires ~3us per sample, each sample
2568                  * was pretty stable even under load => only require 10
2569                  * samples for each offset comparison.
2570                  */
2571                 memset(&adapter->compare, 0, sizeof(adapter->compare));
2572                 adapter->compare.source = &adapter->clock;
2573                 adapter->compare.target = ktime_get_real;
2574                 adapter->compare.num_samples = 10;
2575                 timecompare_update(&adapter->compare, 0);
2576                 break;
2577         case e1000_82576:
2578                 /*
2579                  * Initialize hardware timer: we keep it running just in case
2580                  * that some program needs it later on.
2581                  */
2582                 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2583                 adapter->cycles.read = igb_read_clock;
2584                 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2585                 adapter->cycles.mult = 1;
2586                 /**
2587                  * Scale the NIC clock cycle by a large factor so that
2588                  * relatively small clock corrections can be added or
2589                  * subtracted at each clock tick. The drawbacks of a large
2590                  * factor are a) that the clock register overflows more quickly
2591                  * (not such a big deal) and b) that the increment per tick has
2592                  * to fit into 24 bits.  As a result we need to use a shift of
2593                  * 19 so we can fit a value of 16 into the TIMINCA register.
2594                  */
2595                 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2596                 E1000_WRITE_REG(hw, E1000_TIMINCA,
2597                                 (1 << E1000_TIMINCA_16NS_SHIFT) |
2598                                 (16 << IGB_82576_TSYNC_SHIFT));
2599
2600                 /* Set registers so that rollover occurs soon to test this. */
2601                 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x00000000);
2602                 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0xFF800000);
2603                 E1000_WRITE_FLUSH(hw);
2604
2605                 timecounter_init(&adapter->clock,
2606                                  &adapter->cycles,
2607                                  ktime_to_ns(ktime_get_real()));
2608                 /*
2609                  * Synchronize our NIC clock against system wall clock. NIC
2610                  * time stamp reading requires ~3us per sample, each sample
2611                  * was pretty stable even under load => only require 10
2612                  * samples for each offset comparison.
2613                  */
2614                 memset(&adapter->compare, 0, sizeof(adapter->compare));
2615                 adapter->compare.source = &adapter->clock;
2616                 adapter->compare.target = ktime_get_real;
2617                 adapter->compare.num_samples = 10;
2618                 timecompare_update(&adapter->compare, 0);
2619                 break;
2620         case e1000_82575:
2621                 /* 82575 does not support timesync */
2622         default:
2623                 break;
2624         }
2625 }
2626
2627 #endif /* HAVE_HW_TIME_STAMP */
2628 /**
2629  * igb_sw_init - Initialize general software structures (struct igb_adapter)
2630  * @adapter: board private structure to initialize
2631  *
2632  * igb_sw_init initializes the Adapter private data structure.
2633  * Fields are initialized based on PCI device information and
2634  * OS network device settings (MTU size).
2635  **/
2636 static int igb_sw_init(struct igb_adapter *adapter)
2637 {
2638         struct e1000_hw *hw = &adapter->hw;
2639         struct net_device *netdev = adapter->netdev;
2640         struct pci_dev *pdev = adapter->pdev;
2641
2642         /* PCI config space info */
2643
2644         hw->vendor_id = pdev->vendor;
2645         hw->device_id = pdev->device;
2646         hw->subsystem_vendor_id = pdev->subsystem_vendor;
2647         hw->subsystem_device_id = pdev->subsystem_device;
2648
2649         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
2650
2651         pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2652
2653         /* set default ring sizes */
2654         adapter->tx_ring_count = IGB_DEFAULT_TXD;
2655         adapter->rx_ring_count = IGB_DEFAULT_RXD;
2656
2657         /* set default work limits */
2658         adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2659
2660         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2661                                               VLAN_HLEN;
2662
2663         /* Initialize the hardware-specific values */
2664         if (e1000_setup_init_funcs(hw, TRUE)) {
2665                 dev_err(pci_dev_to_dev(pdev), "Hardware Initialization Failure\n");
2666                 return -EIO;
2667         }
2668
2669         adapter->mac_table = kzalloc(sizeof(struct igb_mac_addr) *
2670                                      hw->mac.rar_entry_count,
2671                                      GFP_ATOMIC);
2672
2673         /* Setup and initialize a copy of the hw vlan table array */
2674         adapter->shadow_vfta = (u32 *)kzalloc(sizeof(u32) * E1000_VFTA_ENTRIES,
2675                                         GFP_ATOMIC);
2676 #ifdef NO_KNI
2677         /* These calls may decrease the number of queues */
2678         igb_set_sriov_capability(adapter);
2679
2680         if (igb_init_interrupt_scheme(adapter)) {
2681                 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for queues\n");
2682                 return -ENOMEM;
2683         }
2684
2685         /* Explicitly disable IRQ since the NIC can be in any state. */
2686         igb_irq_disable(adapter);
2687
2688         set_bit(__IGB_DOWN, &adapter->state);
2689 #endif
2690         return 0;
2691 }
2692
2693 /**
2694  * igb_open - Called when a network interface is made active
2695  * @netdev: network interface device structure
2696  *
2697  * Returns 0 on success, negative value on failure
2698  *
2699  * The open entry point is called when a network interface is made
2700  * active by the system (IFF_UP).  At this point all resources needed
2701  * for transmit and receive operations are allocated, the interrupt
2702  * handler is registered with the OS, the watchdog timer is started,
2703  * and the stack is notified that the interface is ready.
2704  **/
2705 static int __igb_open(struct net_device *netdev, bool resuming)
2706 {
2707         struct igb_adapter *adapter = netdev_priv(netdev);
2708         struct e1000_hw *hw = &adapter->hw;
2709 #ifdef CONFIG_PM_RUNTIME
2710         struct pci_dev *pdev = adapter->pdev;
2711 #endif /* CONFIG_PM_RUNTIME */
2712         int err;
2713         int i;
2714
2715         /* disallow open during test */
2716         if (test_bit(__IGB_TESTING, &adapter->state)) {
2717                 WARN_ON(resuming);
2718                 return -EBUSY;
2719         }
2720
2721 #ifdef CONFIG_PM_RUNTIME
2722         if (!resuming)
2723                 pm_runtime_get_sync(&pdev->dev);
2724 #endif /* CONFIG_PM_RUNTIME */
2725
2726         netif_carrier_off(netdev);
2727
2728         /* allocate transmit descriptors */
2729         err = igb_setup_all_tx_resources(adapter);
2730         if (err)
2731                 goto err_setup_tx;
2732
2733         /* allocate receive descriptors */
2734         err = igb_setup_all_rx_resources(adapter);
2735         if (err)
2736                 goto err_setup_rx;
2737
2738         igb_power_up_link(adapter);
2739
2740         /* before we allocate an interrupt, we must be ready to handle it.
2741          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2742          * as soon as we call pci_request_irq, so we have to setup our
2743          * clean_rx handler before we do so.  */
2744         igb_configure(adapter);
2745
2746         err = igb_request_irq(adapter);
2747         if (err)
2748                 goto err_req_irq;
2749
2750         /* From here on the code is the same as igb_up() */
2751         clear_bit(__IGB_DOWN, &adapter->state);
2752
2753         for (i = 0; i < adapter->num_q_vectors; i++)
2754                 napi_enable(&(adapter->q_vector[i]->napi));
2755         igb_configure_lli(adapter);
2756
2757         /* Clear any pending interrupts. */
2758         E1000_READ_REG(hw, E1000_ICR);
2759
2760         igb_irq_enable(adapter);
2761
2762         /* notify VFs that reset has been completed */
2763         if (adapter->vfs_allocated_count) {
2764                 u32 reg_data = E1000_READ_REG(hw, E1000_CTRL_EXT);
2765                 reg_data |= E1000_CTRL_EXT_PFRSTD;
2766                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg_data);
2767         }
2768
2769         netif_tx_start_all_queues(netdev);
2770
2771         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2772                 schedule_work(&adapter->dma_err_task);
2773
2774         /* start the watchdog. */
2775         hw->mac.get_link_status = 1;
2776         schedule_work(&adapter->watchdog_task);
2777
2778         return E1000_SUCCESS;
2779
2780 err_req_irq:
2781         igb_release_hw_control(adapter);
2782         igb_power_down_link(adapter);
2783         igb_free_all_rx_resources(adapter);
2784 err_setup_rx:
2785         igb_free_all_tx_resources(adapter);
2786 err_setup_tx:
2787         igb_reset(adapter);
2788
2789 #ifdef CONFIG_PM_RUNTIME
2790         if (!resuming)
2791                 pm_runtime_put(&pdev->dev);
2792 #endif /* CONFIG_PM_RUNTIME */
2793
2794         return err;
2795 }
2796
2797 static int igb_open(struct net_device *netdev)
2798 {
2799         return __igb_open(netdev, false);
2800 }
2801
2802 /**
2803  * igb_close - Disables a network interface
2804  * @netdev: network interface device structure
2805  *
2806  * Returns 0, this is not allowed to fail
2807  *
2808  * The close entry point is called when an interface is de-activated
2809  * by the OS.  The hardware is still under the driver's control, but
2810  * needs to be disabled.  A global MAC reset is issued to stop the
2811  * hardware, and all transmit and receive resources are freed.
2812  **/
2813 static int __igb_close(struct net_device *netdev, bool suspending)
2814 {
2815         struct igb_adapter *adapter = netdev_priv(netdev);
2816 #ifdef CONFIG_PM_RUNTIME
2817         struct pci_dev *pdev = adapter->pdev;
2818 #endif /* CONFIG_PM_RUNTIME */
2819
2820         WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2821
2822 #ifdef CONFIG_PM_RUNTIME
2823         if (!suspending)
2824                 pm_runtime_get_sync(&pdev->dev);
2825 #endif /* CONFIG_PM_RUNTIME */
2826
2827         igb_down(adapter);
2828
2829         igb_release_hw_control(adapter);
2830
2831         igb_free_irq(adapter);
2832
2833         igb_free_all_tx_resources(adapter);
2834         igb_free_all_rx_resources(adapter);
2835
2836 #ifdef CONFIG_PM_RUNTIME
2837         if (!suspending)
2838                 pm_runtime_put_sync(&pdev->dev);
2839 #endif /* CONFIG_PM_RUNTIME */
2840
2841         return 0;
2842 }
2843
2844 static int igb_close(struct net_device *netdev)
2845 {
2846         return __igb_close(netdev, false);
2847 }
2848
2849 /**
2850  * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2851  * @tx_ring: tx descriptor ring (for a specific queue) to setup
2852  *
2853  * Return 0 on success, negative on failure
2854  **/
2855 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2856 {
2857         struct device *dev = tx_ring->dev;
2858         int orig_node = dev_to_node(dev);
2859         int size;
2860
2861         size = sizeof(struct igb_tx_buffer) * tx_ring->count;
2862         tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
2863         if (!tx_ring->tx_buffer_info)
2864                 tx_ring->tx_buffer_info = vzalloc(size);
2865         if (!tx_ring->tx_buffer_info)
2866                 goto err;
2867
2868         /* round up to nearest 4K */
2869         tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2870         tx_ring->size = ALIGN(tx_ring->size, 4096);
2871
2872         set_dev_node(dev, tx_ring->numa_node);
2873         tx_ring->desc = dma_alloc_coherent(dev,
2874                                            tx_ring->size,
2875                                            &tx_ring->dma,
2876                                            GFP_KERNEL);
2877         set_dev_node(dev, orig_node);
2878         if (!tx_ring->desc)
2879                 tx_ring->desc = dma_alloc_coherent(dev,
2880                                                    tx_ring->size,
2881                                                    &tx_ring->dma,
2882                                                    GFP_KERNEL);
2883
2884         if (!tx_ring->desc)
2885                 goto err;
2886
2887         tx_ring->next_to_use = 0;
2888         tx_ring->next_to_clean = 0;
2889
2890         return 0;
2891
2892 err:
2893         vfree(tx_ring->tx_buffer_info);
2894         dev_err(dev,
2895                 "Unable to allocate memory for the transmit descriptor ring\n");
2896         return -ENOMEM;
2897 }
2898
2899 /**
2900  * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2901  *                                (Descriptors) for all queues
2902  * @adapter: board private structure
2903  *
2904  * Return 0 on success, negative on failure
2905  **/
2906 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2907 {
2908         struct pci_dev *pdev = adapter->pdev;
2909         int i, err = 0;
2910
2911         for (i = 0; i < adapter->num_tx_queues; i++) {
2912                 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2913                 if (err) {
2914                         dev_err(pci_dev_to_dev(pdev),
2915                                 "Allocation for Tx Queue %u failed\n", i);
2916                         for (i--; i >= 0; i--)
2917                                 igb_free_tx_resources(adapter->tx_ring[i]);
2918                         break;
2919                 }
2920         }
2921
2922         return err;
2923 }
2924
2925 /**
2926  * igb_setup_tctl - configure the transmit control registers
2927  * @adapter: Board private structure
2928  **/
2929 void igb_setup_tctl(struct igb_adapter *adapter)
2930 {
2931         struct e1000_hw *hw = &adapter->hw;
2932         u32 tctl;
2933
2934         /* disable queue 0 which is enabled by default on 82575 and 82576 */
2935         E1000_WRITE_REG(hw, E1000_TXDCTL(0), 0);
2936
2937         /* Program the Transmit Control Register */
2938         tctl = E1000_READ_REG(hw, E1000_TCTL);
2939         tctl &= ~E1000_TCTL_CT;
2940         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2941                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2942
2943         e1000_config_collision_dist(hw);
2944
2945         /* Enable transmits */
2946         tctl |= E1000_TCTL_EN;
2947
2948         E1000_WRITE_REG(hw, E1000_TCTL, tctl);
2949 }
2950
2951 /**
2952  * igb_configure_tx_ring - Configure transmit ring after Reset
2953  * @adapter: board private structure
2954  * @ring: tx ring to configure
2955  *
2956  * Configure a transmit ring after a reset.
2957  **/
2958 void igb_configure_tx_ring(struct igb_adapter *adapter,
2959                            struct igb_ring *ring)
2960 {
2961         struct e1000_hw *hw = &adapter->hw;
2962         u32 txdctl = 0;
2963         u64 tdba = ring->dma;
2964         int reg_idx = ring->reg_idx;
2965
2966         /* disable the queue */
2967         E1000_WRITE_REG(hw, E1000_TXDCTL(reg_idx), 0);
2968         E1000_WRITE_FLUSH(hw);
2969         mdelay(10);
2970
2971         E1000_WRITE_REG(hw, E1000_TDLEN(reg_idx),
2972                         ring->count * sizeof(union e1000_adv_tx_desc));
2973         E1000_WRITE_REG(hw, E1000_TDBAL(reg_idx),
2974                         tdba & 0x00000000ffffffffULL);
2975         E1000_WRITE_REG(hw, E1000_TDBAH(reg_idx), tdba >> 32);
2976
2977         ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2978         E1000_WRITE_REG(hw, E1000_TDH(reg_idx), 0);
2979         writel(0, ring->tail);
2980
2981         txdctl |= IGB_TX_PTHRESH;
2982         txdctl |= IGB_TX_HTHRESH << 8;
2983         txdctl |= IGB_TX_WTHRESH << 16;
2984
2985         txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2986         E1000_WRITE_REG(hw, E1000_TXDCTL(reg_idx), txdctl);
2987 }
2988
2989 /**
2990  * igb_configure_tx - Configure transmit Unit after Reset
2991  * @adapter: board private structure
2992  *
2993  * Configure the Tx unit of the MAC after a reset.
2994  **/
2995 static void igb_configure_tx(struct igb_adapter *adapter)
2996 {
2997         int i;
2998
2999         for (i = 0; i < adapter->num_tx_queues; i++)
3000                 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
3001 }
3002
3003 /**
3004  * igb_setup_rx_resources - allocate Rx resources (Descriptors)
3005  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
3006  *
3007  * Returns 0 on success, negative on failure
3008  **/
3009 int igb_setup_rx_resources(struct igb_ring *rx_ring)
3010 {
3011         struct device *dev = rx_ring->dev;
3012         int orig_node = dev_to_node(dev);
3013         int size, desc_len;
3014
3015         size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3016         rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
3017         if (!rx_ring->rx_buffer_info)
3018                 rx_ring->rx_buffer_info = vzalloc(size);
3019         if (!rx_ring->rx_buffer_info)
3020                 goto err;
3021
3022         desc_len = sizeof(union e1000_adv_rx_desc);
3023
3024         /* Round up to nearest 4K */
3025         rx_ring->size = rx_ring->count * desc_len;
3026         rx_ring->size = ALIGN(rx_ring->size, 4096);
3027
3028         set_dev_node(dev, rx_ring->numa_node);
3029         rx_ring->desc = dma_alloc_coherent(dev,
3030                                            rx_ring->size,
3031                                            &rx_ring->dma,
3032                                            GFP_KERNEL);
3033         set_dev_node(dev, orig_node);
3034         if (!rx_ring->desc)
3035                 rx_ring->desc = dma_alloc_coherent(dev,
3036                                                    rx_ring->size,
3037                                                    &rx_ring->dma,
3038                                                    GFP_KERNEL);
3039
3040         if (!rx_ring->desc)
3041                 goto err;
3042
3043         rx_ring->next_to_clean = 0;
3044         rx_ring->next_to_use = 0;
3045
3046         return 0;
3047
3048 err:
3049         vfree(rx_ring->rx_buffer_info);
3050         rx_ring->rx_buffer_info = NULL;
3051         dev_err(dev, "Unable to allocate memory for the receive descriptor"
3052                 " ring\n");
3053         return -ENOMEM;
3054 }
3055
3056 /**
3057  * igb_setup_all_rx_resources - wrapper to allocate Rx resources
3058  *                                (Descriptors) for all queues
3059  * @adapter: board private structure
3060  *
3061  * Return 0 on success, negative on failure
3062  **/
3063 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
3064 {
3065         struct pci_dev *pdev = adapter->pdev;
3066         int i, err = 0;
3067
3068         for (i = 0; i < adapter->num_rx_queues; i++) {
3069                 err = igb_setup_rx_resources(adapter->rx_ring[i]);
3070                 if (err) {
3071                         dev_err(pci_dev_to_dev(pdev),
3072                                 "Allocation for Rx Queue %u failed\n", i);
3073                         for (i--; i >= 0; i--)
3074                                 igb_free_rx_resources(adapter->rx_ring[i]);
3075                         break;
3076                 }
3077         }
3078
3079         return err;
3080 }
3081
3082 /**
3083  * igb_setup_mrqc - configure the multiple receive queue control registers
3084  * @adapter: Board private structure
3085  **/
3086 static void igb_setup_mrqc(struct igb_adapter *adapter)
3087 {
3088         struct e1000_hw *hw = &adapter->hw;
3089         u32 mrqc, rxcsum;
3090         u32 j, num_rx_queues, shift = 0, shift2 = 0;
3091         union e1000_reta {
3092                 u32 dword;
3093                 u8  bytes[4];
3094         } reta;
3095         static const u8 rsshash[40] = {
3096                 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
3097                 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
3098                 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
3099                 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
3100
3101         /* Fill out hash function seeds */
3102         for (j = 0; j < 10; j++) {
3103                 u32 rsskey = rsshash[(j * 4)];
3104                 rsskey |= rsshash[(j * 4) + 1] << 8;
3105                 rsskey |= rsshash[(j * 4) + 2] << 16;
3106                 rsskey |= rsshash[(j * 4) + 3] << 24;
3107                 E1000_WRITE_REG_ARRAY(hw, E1000_RSSRK(0), j, rsskey);
3108         }
3109
3110         num_rx_queues = adapter->rss_queues;
3111
3112         if (adapter->vfs_allocated_count || adapter->vmdq_pools) {
3113                 /* 82575 and 82576 supports 2 RSS queues for VMDq */
3114                 switch (hw->mac.type) {
3115                 case e1000_i350:
3116                 case e1000_82580:
3117                         num_rx_queues = 1;
3118                         shift = 0;
3119                         break;
3120                 case e1000_82576:
3121                         shift = 3;
3122                         num_rx_queues = 2;
3123                         break;
3124                 case e1000_82575:
3125                         shift = 2;
3126                         shift2 = 6;
3127                 default:
3128                         break;
3129                 }
3130         } else {
3131                 if (hw->mac.type == e1000_82575)
3132                         shift = 6;
3133         }
3134
3135         for (j = 0; j < (32 * 4); j++) {
3136                 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
3137                 if (shift2)
3138                         reta.bytes[j & 3] |= num_rx_queues << shift2;
3139                 if ((j & 3) == 3)
3140                         E1000_WRITE_REG(hw, E1000_RETA(j >> 2), reta.dword);
3141         }
3142
3143         /*
3144          * Disable raw packet checksumming so that RSS hash is placed in
3145          * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
3146          * offloads as they are enabled by default
3147          */
3148         rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
3149         rxcsum |= E1000_RXCSUM_PCSD;
3150
3151         if (adapter->hw.mac.type >= e1000_82576)
3152                 /* Enable Receive Checksum Offload for SCTP */
3153                 rxcsum |= E1000_RXCSUM_CRCOFL;
3154
3155         /* Don't need to set TUOFL or IPOFL, they default to 1 */
3156         E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
3157
3158         /* If VMDq is enabled then we set the appropriate mode for that, else
3159          * we default to RSS so that an RSS hash is calculated per packet even
3160          * if we are only using one queue */
3161         if (adapter->vfs_allocated_count || adapter->vmdq_pools) {
3162                 if (hw->mac.type > e1000_82575) {
3163                         /* Set the default pool for the PF's first queue */
3164                         u32 vtctl = E1000_READ_REG(hw, E1000_VT_CTL);
3165                         vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
3166                                    E1000_VT_CTL_DISABLE_DEF_POOL);
3167                         vtctl |= adapter->vfs_allocated_count <<
3168                                 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
3169                         E1000_WRITE_REG(hw, E1000_VT_CTL, vtctl);
3170                 } else if (adapter->rss_queues > 1) {
3171                         /* set default queue for pool 1 to queue 2 */
3172                         E1000_WRITE_REG(hw, E1000_VT_CTL,
3173                                         adapter->rss_queues << 7);
3174                 }
3175                 if (adapter->rss_queues > 1)
3176                         mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
3177                 else
3178                         mrqc = E1000_MRQC_ENABLE_VMDQ;
3179         } else {
3180                 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
3181         }
3182
3183         igb_vmm_control(adapter);
3184
3185         /*
3186          * Generate RSS hash based on TCP port numbers and/or
3187          * IPv4/v6 src and dst addresses since UDP cannot be
3188          * hashed reliably due to IP fragmentation
3189          */
3190         mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
3191                 E1000_MRQC_RSS_FIELD_IPV4_TCP |
3192                 E1000_MRQC_RSS_FIELD_IPV6 |
3193                 E1000_MRQC_RSS_FIELD_IPV6_TCP |
3194                 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
3195
3196         E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
3197 }
3198
3199 /**
3200  * igb_setup_rctl - configure the receive control registers
3201  * @adapter: Board private structure
3202  **/
3203 void igb_setup_rctl(struct igb_adapter *adapter)
3204 {
3205         struct e1000_hw *hw = &adapter->hw;
3206         u32 rctl;
3207
3208         rctl = E1000_READ_REG(hw, E1000_RCTL);
3209
3210         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3211         rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
3212
3213         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
3214                 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
3215
3216         /*
3217          * enable stripping of CRC. It's unlikely this will break BMC
3218          * redirection as it did with e1000. Newer features require
3219          * that the HW strips the CRC.
3220          */
3221         rctl |= E1000_RCTL_SECRC;
3222
3223         /* disable store bad packets and clear size bits. */
3224         rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
3225
3226         /* enable LPE to prevent packets larger than max_frame_size */
3227         rctl |= E1000_RCTL_LPE;
3228
3229         /* disable queue 0 to prevent tail write w/o re-config */
3230         E1000_WRITE_REG(hw, E1000_RXDCTL(0), 0);
3231
3232         /* Attention!!!  For SR-IOV PF driver operations you must enable
3233          * queue drop for all VF and PF queues to prevent head of line blocking
3234          * if an un-trusted VF does not provide descriptors to hardware.
3235          */
3236         if (adapter->vfs_allocated_count) {
3237                 /* set all queue drop enable bits */
3238                 E1000_WRITE_REG(hw, E1000_QDE, ALL_QUEUES);
3239         }
3240
3241         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3242 }
3243
3244 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
3245                                    int vfn)
3246 {
3247         struct e1000_hw *hw = &adapter->hw;
3248         u32 vmolr;
3249
3250         /* if it isn't the PF check to see if VFs are enabled and
3251          * increase the size to support vlan tags */
3252         if (vfn < adapter->vfs_allocated_count &&
3253             adapter->vf_data[vfn].vlans_enabled)
3254                 size += VLAN_HLEN;
3255
3256 #ifdef CONFIG_IGB_VMDQ_NETDEV
3257         if (vfn >= adapter->vfs_allocated_count) {
3258                 int queue = vfn - adapter->vfs_allocated_count;
3259                 struct igb_vmdq_adapter *vadapter;
3260
3261                 vadapter = netdev_priv(adapter->vmdq_netdev[queue-1]);
3262                 if (vadapter->vlgrp)
3263                         size += VLAN_HLEN;
3264         }
3265 #endif
3266         vmolr = E1000_READ_REG(hw, E1000_VMOLR(vfn));
3267         vmolr &= ~E1000_VMOLR_RLPML_MASK;
3268         vmolr |= size | E1000_VMOLR_LPE;
3269         E1000_WRITE_REG(hw, E1000_VMOLR(vfn), vmolr);
3270
3271         return 0;
3272 }
3273
3274 /**
3275  * igb_rlpml_set - set maximum receive packet size
3276  * @adapter: board private structure
3277  *
3278  * Configure maximum receivable packet size.
3279  **/
3280 static void igb_rlpml_set(struct igb_adapter *adapter)
3281 {
3282         u32 max_frame_size = adapter->max_frame_size;
3283         struct e1000_hw *hw = &adapter->hw;
3284         u16 pf_id = adapter->vfs_allocated_count;
3285
3286         if (adapter->vmdq_pools && hw->mac.type != e1000_82575) {
3287                 int i;
3288                 for (i = 0; i < adapter->vmdq_pools; i++)
3289                         igb_set_vf_rlpml(adapter, max_frame_size, pf_id + i);
3290                 /*
3291                  * If we're in VMDQ or SR-IOV mode, then set global RLPML
3292                  * to our max jumbo frame size, in case we need to enable
3293                  * jumbo frames on one of the rings later.
3294                  * This will not pass over-length frames into the default
3295                  * queue because it's gated by the VMOLR.RLPML.
3296                  */
3297                 max_frame_size = MAX_JUMBO_FRAME_SIZE;
3298         }
3299         /* Set VF RLPML for the PF device. */
3300         if (adapter->vfs_allocated_count)
3301                 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
3302
3303         E1000_WRITE_REG(hw, E1000_RLPML, max_frame_size);
3304 }
3305
3306 static inline void igb_set_vf_vlan_strip(struct igb_adapter *adapter,
3307                                         int vfn, bool enable)
3308 {
3309         struct e1000_hw *hw = &adapter->hw;
3310         u32 val;
3311         void __iomem *reg;
3312
3313         if (hw->mac.type < e1000_82576)
3314                 return;
3315
3316         if (hw->mac.type == e1000_i350)
3317                 reg = hw->hw_addr + E1000_DVMOLR(vfn);
3318         else
3319                 reg = hw->hw_addr + E1000_VMOLR(vfn);
3320
3321         val = readl(reg);
3322         if (enable)
3323                 val |= E1000_VMOLR_STRVLAN;
3324         else
3325                 val &= ~(E1000_VMOLR_STRVLAN);
3326         writel(val, reg);
3327 }
3328 static inline void igb_set_vmolr(struct igb_adapter *adapter,
3329                                  int vfn, bool aupe)
3330 {
3331         struct e1000_hw *hw = &adapter->hw;
3332         u32 vmolr;
3333
3334         /*
3335          * This register exists only on 82576 and newer so if we are older then
3336          * we should exit and do nothing
3337          */
3338         if (hw->mac.type < e1000_82576)
3339                 return;
3340
3341         vmolr = E1000_READ_REG(hw, E1000_VMOLR(vfn));
3342
3343         if (aupe)
3344                 vmolr |= E1000_VMOLR_AUPE;        /* Accept untagged packets */
3345         else
3346                 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
3347
3348         /* clear all bits that might not be set */
3349         vmolr &= ~E1000_VMOLR_RSSE;
3350
3351         if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
3352                 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3353
3354         vmolr |= E1000_VMOLR_BAM;          /* Accept broadcast */
3355         vmolr |= E1000_VMOLR_LPE;          /* Accept long packets */
3356
3357         E1000_WRITE_REG(hw, E1000_VMOLR(vfn), vmolr);
3358 }
3359
3360 /**
3361  * igb_configure_rx_ring - Configure a receive ring after Reset
3362  * @adapter: board private structure
3363  * @ring: receive ring to be configured
3364  *
3365  * Configure the Rx unit of the MAC after a reset.
3366  **/
3367 void igb_configure_rx_ring(struct igb_adapter *adapter,
3368                            struct igb_ring *ring)
3369 {
3370         struct e1000_hw *hw = &adapter->hw;
3371         u64 rdba = ring->dma;
3372         int reg_idx = ring->reg_idx;
3373         u32 srrctl = 0, rxdctl = 0;
3374
3375         /* disable the queue */
3376         E1000_WRITE_REG(hw, E1000_RXDCTL(reg_idx), 0);
3377
3378         /* Set DMA base address registers */
3379         E1000_WRITE_REG(hw, E1000_RDBAL(reg_idx),
3380                         rdba & 0x00000000ffffffffULL);
3381         E1000_WRITE_REG(hw, E1000_RDBAH(reg_idx), rdba >> 32);
3382         E1000_WRITE_REG(hw, E1000_RDLEN(reg_idx),
3383                        ring->count * sizeof(union e1000_adv_rx_desc));
3384
3385         /* initialize head and tail */
3386         ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
3387         E1000_WRITE_REG(hw, E1000_RDH(reg_idx), 0);
3388         writel(0, ring->tail);
3389
3390         /* set descriptor configuration */
3391 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
3392         srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3393 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
3394         srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3395 #else
3396         srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3397 #endif
3398         srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3399 #else /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
3400         srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
3401                  E1000_SRRCTL_BSIZEPKT_SHIFT;
3402         srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
3403 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
3404 #ifdef IGB_PER_PKT_TIMESTAMP
3405         if (hw->mac.type >= e1000_82580)
3406                 srrctl |= E1000_SRRCTL_TIMESTAMP;
3407 #endif
3408         /*
3409          * We should set the drop enable bit if:
3410          *  SR-IOV is enabled
3411          *   or
3412          *  Flow Control is disabled and number of RX queues > 1
3413          *
3414          *  This allows us to avoid head of line blocking for security
3415          *  and performance reasons.
3416          */
3417         if (adapter->vfs_allocated_count ||
3418             (adapter->num_rx_queues > 1 &&
3419              (hw->fc.requested_mode == e1000_fc_none ||
3420               hw->fc.requested_mode == e1000_fc_rx_pause)))
3421                 srrctl |= E1000_SRRCTL_DROP_EN;
3422
3423         E1000_WRITE_REG(hw, E1000_SRRCTL(reg_idx), srrctl);
3424
3425         /* set filtering for VMDQ pools */
3426         igb_set_vmolr(adapter, reg_idx & 0x7, true);
3427
3428         rxdctl |= IGB_RX_PTHRESH;
3429         rxdctl |= IGB_RX_HTHRESH << 8;
3430         rxdctl |= IGB_RX_WTHRESH << 16;
3431
3432         /* enable receive descriptor fetching */
3433         rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3434         E1000_WRITE_REG(hw, E1000_RXDCTL(reg_idx), rxdctl);
3435 }
3436
3437 /**
3438  * igb_configure_rx - Configure receive Unit after Reset
3439  * @adapter: board private structure
3440  *
3441  * Configure the Rx unit of the MAC after a reset.
3442  **/
3443 static void igb_configure_rx(struct igb_adapter *adapter)
3444 {
3445         int i;
3446
3447         /* set UTA to appropriate mode */
3448         igb_set_uta(adapter);
3449
3450         igb_full_sync_mac_table(adapter);
3451         /* Setup the HW Rx Head and Tail Descriptor Pointers and
3452          * the Base and Length of the Rx Descriptor Ring */
3453         for (i = 0; i < adapter->num_rx_queues; i++)
3454                 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
3455 }
3456
3457 /**
3458  * igb_free_tx_resources - Free Tx Resources per Queue
3459  * @tx_ring: Tx descriptor ring for a specific queue
3460  *
3461  * Free all transmit software resources
3462  **/
3463 void igb_free_tx_resources(struct igb_ring *tx_ring)
3464 {
3465         igb_clean_tx_ring(tx_ring);
3466
3467         vfree(tx_ring->tx_buffer_info);
3468         tx_ring->tx_buffer_info = NULL;
3469
3470         /* if not set, then don't free */
3471         if (!tx_ring->desc)
3472                 return;
3473
3474         dma_free_coherent(tx_ring->dev, tx_ring->size,
3475                           tx_ring->desc, tx_ring->dma);
3476
3477         tx_ring->desc = NULL;
3478 }
3479
3480 /**
3481  * igb_free_all_tx_resources - Free Tx Resources for All Queues
3482  * @adapter: board private structure
3483  *
3484  * Free all transmit software resources
3485  **/
3486 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3487 {
3488         int i;
3489
3490         for (i = 0; i < adapter->num_tx_queues; i++)
3491                 igb_free_tx_resources(adapter->tx_ring[i]);
3492 }
3493
3494 void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
3495                                     struct igb_tx_buffer *tx_buffer)
3496 {
3497         if (tx_buffer->skb) {
3498                 dev_kfree_skb_any(tx_buffer->skb);
3499                 if (dma_unmap_len(tx_buffer, len))
3500                         dma_unmap_single(ring->dev,
3501                                          dma_unmap_addr(tx_buffer, dma),
3502                                          dma_unmap_len(tx_buffer, len),
3503                                          DMA_TO_DEVICE);
3504         } else if (dma_unmap_len(tx_buffer, len)) {
3505                 dma_unmap_page(ring->dev,
3506                                dma_unmap_addr(tx_buffer, dma),
3507                                dma_unmap_len(tx_buffer, len),
3508                                DMA_TO_DEVICE);
3509         }
3510         tx_buffer->next_to_watch = NULL;
3511         tx_buffer->skb = NULL;
3512         dma_unmap_len_set(tx_buffer, len, 0);
3513         /* buffer_info must be completely set up in the transmit path */
3514 }
3515
3516 /**
3517  * igb_clean_tx_ring - Free Tx Buffers
3518  * @tx_ring: ring to be cleaned
3519  **/
3520 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3521 {
3522         struct igb_tx_buffer *buffer_info;
3523         unsigned long size;
3524         u16 i;
3525
3526         if (!tx_ring->tx_buffer_info)
3527                 return;
3528         /* Free all the Tx ring sk_buffs */
3529
3530         for (i = 0; i < tx_ring->count; i++) {
3531                 buffer_info = &tx_ring->tx_buffer_info[i];
3532                 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3533         }
3534
3535 #ifdef CONFIG_BQL
3536         netdev_tx_reset_queue(txring_txq(tx_ring));
3537 #endif /* CONFIG_BQL */
3538
3539         size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3540         memset(tx_ring->tx_buffer_info, 0, size);
3541
3542         /* Zero out the descriptor ring */
3543         memset(tx_ring->desc, 0, tx_ring->size);
3544
3545         tx_ring->next_to_use = 0;
3546         tx_ring->next_to_clean = 0;
3547 }
3548
3549 /**
3550  * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3551  * @adapter: board private structure
3552  **/
3553 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3554 {
3555         int i;
3556
3557         for (i = 0; i < adapter->num_tx_queues; i++)
3558                 igb_clean_tx_ring(adapter->tx_ring[i]);
3559 }
3560
3561 /**
3562  * igb_free_rx_resources - Free Rx Resources
3563  * @rx_ring: ring to clean the resources from
3564  *
3565  * Free all receive software resources
3566  **/
3567 void igb_free_rx_resources(struct igb_ring *rx_ring)
3568 {
3569         igb_clean_rx_ring(rx_ring);
3570
3571         vfree(rx_ring->rx_buffer_info);
3572         rx_ring->rx_buffer_info = NULL;
3573
3574         /* if not set, then don't free */
3575         if (!rx_ring->desc)
3576                 return;
3577
3578         dma_free_coherent(rx_ring->dev, rx_ring->size,
3579                           rx_ring->desc, rx_ring->dma);
3580
3581         rx_ring->desc = NULL;
3582 }
3583
3584 /**
3585  * igb_free_all_rx_resources - Free Rx Resources for All Queues
3586  * @adapter: board private structure
3587  *
3588  * Free all receive software resources
3589  **/
3590 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3591 {
3592         int i;
3593
3594         for (i = 0; i < adapter->num_rx_queues; i++)
3595                 igb_free_rx_resources(adapter->rx_ring[i]);
3596 }
3597
3598 /**
3599  * igb_clean_rx_ring - Free Rx Buffers per Queue
3600  * @rx_ring: ring to free buffers from
3601  **/
3602 void igb_clean_rx_ring(struct igb_ring *rx_ring)
3603 {
3604         unsigned long size;
3605 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
3606         const int bufsz = rx_ring->rx_buffer_len;
3607 #else
3608         const int bufsz = IGB_RX_HDR_LEN;
3609 #endif
3610         u16 i;
3611
3612         if (!rx_ring->rx_buffer_info)
3613                 return;
3614
3615         /* Free all the Rx ring sk_buffs */
3616         for (i = 0; i < rx_ring->count; i++) {
3617                 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
3618                 if (buffer_info->dma) {
3619                         dma_unmap_single(rx_ring->dev,
3620                                          buffer_info->dma,
3621                                          bufsz,
3622                                          DMA_FROM_DEVICE);
3623                         buffer_info->dma = 0;
3624                 }
3625
3626                 if (buffer_info->skb) {
3627                         dev_kfree_skb(buffer_info->skb);
3628                         buffer_info->skb = NULL;
3629                 }
3630 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
3631                 if (buffer_info->page_dma) {
3632                         dma_unmap_page(rx_ring->dev,
3633                                        buffer_info->page_dma,
3634                                        PAGE_SIZE / 2,
3635                                        DMA_FROM_DEVICE);
3636                         buffer_info->page_dma = 0;
3637                 }
3638                 if (buffer_info->page) {
3639                         put_page(buffer_info->page);
3640                         buffer_info->page = NULL;
3641                         buffer_info->page_offset = 0;
3642                 }
3643 #endif
3644         }
3645
3646         size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3647         memset(rx_ring->rx_buffer_info, 0, size);
3648
3649         /* Zero out the descriptor ring */
3650         memset(rx_ring->desc, 0, rx_ring->size);
3651
3652         rx_ring->next_to_clean = 0;
3653         rx_ring->next_to_use = 0;
3654 }
3655
3656 /**
3657  * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3658  * @adapter: board private structure
3659  **/
3660 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3661 {
3662         int i;
3663
3664         for (i = 0; i < adapter->num_rx_queues; i++)
3665                 igb_clean_rx_ring(adapter->rx_ring[i]);
3666 }
3667
3668 /**
3669  * igb_set_mac - Change the Ethernet Address of the NIC
3670  * @netdev: network interface device structure
3671  * @p: pointer to an address structure
3672  *
3673  * Returns 0 on success, negative on failure
3674  **/
3675 static int igb_set_mac(struct net_device *netdev, void *p)
3676 {
3677         struct igb_adapter *adapter = netdev_priv(netdev);
3678         struct e1000_hw *hw = &adapter->hw;
3679         struct sockaddr *addr = p;
3680
3681         if (!is_valid_ether_addr(addr->sa_data))
3682                 return -EADDRNOTAVAIL;
3683
3684         igb_del_mac_filter(adapter, hw->mac.addr,
3685                            adapter->vfs_allocated_count);
3686         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3687         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3688
3689         /* set the correct pool for the new PF MAC address in entry 0 */
3690         return igb_add_mac_filter(adapter, hw->mac.addr,
3691                            adapter->vfs_allocated_count);
3692 }
3693
3694 /**
3695  * igb_write_mc_addr_list - write multicast addresses to MTA
3696  * @netdev: network interface device structure
3697  *
3698  * Writes multicast address list to the MTA hash table.
3699  * Returns: -ENOMEM on failure
3700  *                0 on no addresses written
3701  *                X on writing X addresses to MTA
3702  **/
3703 int igb_write_mc_addr_list(struct net_device *netdev)
3704 {
3705         struct igb_adapter *adapter = netdev_priv(netdev);
3706         struct e1000_hw *hw = &adapter->hw;
3707 #ifdef NETDEV_HW_ADDR_T_MULTICAST
3708         struct netdev_hw_addr *ha;
3709 #else
3710         struct dev_mc_list *ha;
3711 #endif
3712         u8  *mta_list;
3713         int i, count;
3714 #ifdef CONFIG_IGB_VMDQ_NETDEV
3715         int vm;
3716 #endif
3717         count = netdev_mc_count(netdev);
3718 #ifdef CONFIG_IGB_VMDQ_NETDEV
3719         for (vm = 1; vm < adapter->vmdq_pools; vm++) {
3720                 if (!adapter->vmdq_netdev[vm])
3721                         break;
3722                 if (!netif_running(adapter->vmdq_netdev[vm]))
3723                         continue;
3724                 count += netdev_mc_count(adapter->vmdq_netdev[vm]);
3725         }
3726 #endif
3727
3728         if (!count) {
3729                 e1000_update_mc_addr_list(hw, NULL, 0);
3730                 return 0;
3731         }
3732         mta_list = kzalloc(count * 6, GFP_ATOMIC);
3733         if (!mta_list)
3734                 return -ENOMEM;
3735
3736         /* The shared function expects a packed array of only addresses. */
3737         i = 0;
3738         netdev_for_each_mc_addr(ha, netdev)
3739 #ifdef NETDEV_HW_ADDR_T_MULTICAST
3740                 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3741 #else
3742                 memcpy(mta_list + (i++ * ETH_ALEN), ha->dmi_addr, ETH_ALEN);
3743 #endif
3744 #ifdef CONFIG_IGB_VMDQ_NETDEV
3745         for (vm = 1; vm < adapter->vmdq_pools; vm++) {
3746                 if (!adapter->vmdq_netdev[vm])
3747                         break;
3748                 if (!netif_running(adapter->vmdq_netdev[vm]) ||
3749                     !netdev_mc_count(adapter->vmdq_netdev[vm]))
3750                         continue;
3751                 netdev_for_each_mc_addr(ha, adapter->vmdq_netdev[vm])
3752 #ifdef NETDEV_HW_ADDR_T_MULTICAST
3753                         memcpy(mta_list + (i++ * ETH_ALEN),
3754                                ha->addr, ETH_ALEN);
3755 #else
3756                         memcpy(mta_list + (i++ * ETH_ALEN),
3757                                ha->dmi_addr, ETH_ALEN);
3758 #endif
3759         }
3760 #endif
3761         e1000_update_mc_addr_list(hw, mta_list, i);
3762         kfree(mta_list);
3763
3764         return count;
3765 }
3766
3767 void igb_rar_set(struct igb_adapter *adapter, u32 index)
3768 {
3769         u32 rar_low, rar_high;
3770         struct e1000_hw *hw = &adapter->hw;
3771         u8 *addr = adapter->mac_table[index].addr;
3772         /* HW expects these in little endian so we reverse the byte order
3773          * from network order (big endian) to little endian
3774          */
3775         rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
3776                   ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
3777         rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
3778
3779         /* Indicate to hardware the Address is Valid. */
3780         if (adapter->mac_table[index].state & IGB_MAC_STATE_IN_USE)
3781                 rar_high |= E1000_RAH_AV;
3782
3783         if (hw->mac.type == e1000_82575)
3784                 rar_high |= E1000_RAH_POOL_1 * adapter->mac_table[index].queue;
3785         else
3786                 rar_high |= E1000_RAH_POOL_1 << adapter->mac_table[index].queue;
3787
3788         E1000_WRITE_REG(hw, E1000_RAL(index), rar_low);
3789         E1000_WRITE_FLUSH(hw);
3790         E1000_WRITE_REG(hw, E1000_RAH(index), rar_high);
3791         E1000_WRITE_FLUSH(hw);
3792 }
3793
3794 void igb_full_sync_mac_table(struct igb_adapter *adapter)
3795 {
3796         struct e1000_hw *hw = &adapter->hw;
3797         int i;
3798         for (i = 0; i < hw->mac.rar_entry_count; i++) {
3799                         igb_rar_set(adapter, i);
3800         }
3801 }
3802
3803 void igb_sync_mac_table(struct igb_adapter *adapter)
3804 {
3805         struct e1000_hw *hw = &adapter->hw;
3806         int i;
3807         for (i = 0; i < hw->mac.rar_entry_count; i++) {
3808                 if (adapter->mac_table[i].state & IGB_MAC_STATE_MODIFIED)
3809                         igb_rar_set(adapter, i);
3810                 adapter->mac_table[i].state &= ~(IGB_MAC_STATE_MODIFIED);
3811         }
3812 }
3813
3814 int igb_available_rars(struct igb_adapter *adapter)
3815 {
3816         struct e1000_hw *hw = &adapter->hw;
3817         int i, count = 0;
3818
3819         for (i = 0; i < hw->mac.rar_entry_count; i++) {
3820                 if (adapter->mac_table[i].state == 0)
3821                         count++;
3822         }
3823         return count;
3824 }
3825
3826 #ifdef HAVE_SET_RX_MODE
3827 /**
3828  * igb_write_uc_addr_list - write unicast addresses to RAR table
3829  * @netdev: network interface device structure
3830  *
3831  * Writes unicast address list to the RAR table.
3832  * Returns: -ENOMEM on failure/insufficient address space
3833  *                0 on no addresses written
3834  *                X on writing X addresses to the RAR table
3835  **/
3836 static int igb_write_uc_addr_list(struct net_device *netdev)
3837 {
3838         struct igb_adapter *adapter = netdev_priv(netdev);
3839         unsigned int vfn = adapter->vfs_allocated_count;
3840         int count = 0;
3841
3842         /* return ENOMEM indicating insufficient memory for addresses */
3843         if (netdev_uc_count(netdev) > igb_available_rars(adapter))
3844                 return -ENOMEM;
3845         if (!netdev_uc_empty(netdev)) {
3846 #ifdef NETDEV_HW_ADDR_T_UNICAST
3847                 struct netdev_hw_addr *ha;
3848 #else
3849                 struct dev_mc_list *ha;
3850 #endif
3851                 netdev_for_each_uc_addr(ha, netdev) {
3852 #ifdef NETDEV_HW_ADDR_T_UNICAST
3853                         igb_del_mac_filter(adapter, ha->addr, vfn);
3854                         igb_add_mac_filter(adapter, ha->addr, vfn);
3855 #else
3856                         igb_del_mac_filter(adapter, ha->da_addr, vfn);
3857                         igb_add_mac_filter(adapter, ha->da_addr, vfn);
3858 #endif
3859                         count++;
3860                 }
3861         }
3862         return count;
3863 }
3864
3865 #endif /* HAVE_SET_RX_MODE */
3866 /**
3867  * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3868  * @netdev: network interface device structure
3869  *
3870  * The set_rx_mode entry point is called whenever the unicast or multicast
3871  * address lists or the network interface flags are updated.  This routine is
3872  * responsible for configuring the hardware for proper unicast, multicast,
3873  * promiscuous mode, and all-multi behavior.
3874  **/
3875 static void igb_set_rx_mode(struct net_device *netdev)
3876 {
3877         struct igb_adapter *adapter = netdev_priv(netdev);
3878         struct e1000_hw *hw = &adapter->hw;
3879         unsigned int vfn = adapter->vfs_allocated_count;
3880         u32 rctl, vmolr = 0;
3881         int count;
3882
3883         /* Check for Promiscuous and All Multicast modes */
3884         rctl = E1000_READ_REG(hw, E1000_RCTL);
3885
3886         /* clear the effected bits */
3887         rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3888
3889         if (netdev->flags & IFF_PROMISC) {
3890                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3891                 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3892         } else {
3893                 if (netdev->flags & IFF_ALLMULTI) {
3894                         rctl |= E1000_RCTL_MPE;
3895                         vmolr |= E1000_VMOLR_MPME;
3896                 } else {
3897                         /*
3898                          * Write addresses to the MTA, if the attempt fails
3899                          * then we should just turn on promiscuous mode so
3900                          * that we can at least receive multicast traffic
3901                          */
3902                         count = igb_write_mc_addr_list(netdev);
3903                         if (count < 0) {
3904                                 rctl |= E1000_RCTL_MPE;
3905                                 vmolr |= E1000_VMOLR_MPME;
3906                         } else if (count) {
3907                                 vmolr |= E1000_VMOLR_ROMPE;
3908                         }
3909                 }
3910 #ifdef HAVE_SET_RX_MODE
3911                 /*
3912                  * Write addresses to available RAR registers, if there is not
3913                  * sufficient space to store all the addresses then enable
3914                  * unicast promiscuous mode
3915                  */
3916                 count = igb_write_uc_addr_list(netdev);
3917                 if (count < 0) {
3918                         rctl |= E1000_RCTL_UPE;
3919                         vmolr |= E1000_VMOLR_ROPE;
3920                 }
3921 #endif /* HAVE_SET_RX_MODE */
3922                 rctl |= E1000_RCTL_VFE;
3923         }
3924         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3925
3926         /*
3927          * In order to support SR-IOV and eventually VMDq it is necessary to set
3928          * the VMOLR to enable the appropriate modes.  Without this workaround
3929          * we will have issues with VLAN tag stripping not being done for frames
3930          * that are only arriving because we are the default pool
3931          */
3932         if (hw->mac.type < e1000_82576)
3933                 return;
3934
3935         vmolr |= E1000_READ_REG(hw, E1000_VMOLR(vfn)) &
3936                  ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3937         E1000_WRITE_REG(hw, E1000_VMOLR(vfn), vmolr);
3938         igb_restore_vf_multicasts(adapter);
3939 }
3940
3941 static void igb_check_wvbr(struct igb_adapter *adapter)
3942 {
3943         struct e1000_hw *hw = &adapter->hw;
3944         u32 wvbr = 0;
3945
3946         switch (hw->mac.type) {
3947         case e1000_82576:
3948         case e1000_i350:
3949                 if (!(wvbr = E1000_READ_REG(hw, E1000_WVBR)))
3950                         return;
3951                 break;
3952         default:
3953                 break;
3954         }
3955
3956         adapter->wvbr |= wvbr;
3957 }
3958
3959 #define IGB_STAGGERED_QUEUE_OFFSET 8
3960
3961 static void igb_spoof_check(struct igb_adapter *adapter)
3962 {
3963         int j;
3964
3965         if (!adapter->wvbr)
3966                 return;
3967
3968         for(j = 0; j < adapter->vfs_allocated_count; j++) {
3969                 if (adapter->wvbr & (1 << j) ||
3970                     adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3971                         DPRINTK(DRV, WARNING,
3972                                 "Spoof event(s) detected on VF %d\n", j);
3973                         adapter->wvbr &=
3974                                 ~((1 << j) |
3975                                   (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3976                 }
3977         }
3978 }
3979
3980 /* Need to wait a few seconds after link up to get diagnostic information from
3981  * the phy */
3982 static void igb_update_phy_info(unsigned long data)
3983 {
3984         struct igb_adapter *adapter = (struct igb_adapter *) data;
3985         e1000_get_phy_info(&adapter->hw);
3986 }
3987
3988 /**
3989  * igb_has_link - check shared code for link and determine up/down
3990  * @adapter: pointer to driver private info
3991  **/
3992 bool igb_has_link(struct igb_adapter *adapter)
3993 {
3994         struct e1000_hw *hw = &adapter->hw;
3995         bool link_active = FALSE;
3996
3997         /* get_link_status is set on LSC (link status) interrupt or
3998          * rx sequence error interrupt.  get_link_status will stay
3999          * false until the e1000_check_for_link establishes link
4000          * for copper adapters ONLY
4001          */
4002         switch (hw->phy.media_type) {
4003         case e1000_media_type_copper:
4004                 if (!hw->mac.get_link_status)
4005                         return true;
4006         case e1000_media_type_internal_serdes:
4007                 e1000_check_for_link(hw);
4008                 link_active = !hw->mac.get_link_status;
4009                 break;
4010         case e1000_media_type_unknown:
4011         default:
4012                 break;
4013         }
4014
4015         return link_active;
4016 }
4017
4018 /**
4019  * igb_watchdog - Timer Call-back
4020  * @data: pointer to adapter cast into an unsigned long
4021  **/
4022 static void igb_watchdog(unsigned long data)
4023 {
4024         struct igb_adapter *adapter = (struct igb_adapter *)data;
4025         /* Do the rest outside of interrupt context */
4026         schedule_work(&adapter->watchdog_task);
4027 }
4028
4029 static void igb_watchdog_task(struct work_struct *work)
4030 {
4031         struct igb_adapter *adapter = container_of(work,
4032                                                    struct igb_adapter,
4033                                                    watchdog_task);
4034         struct e1000_hw *hw = &adapter->hw;
4035         struct net_device *netdev = adapter->netdev;
4036         u32 link;
4037         int i;
4038         u32 thstat, ctrl_ext;
4039
4040
4041         link = igb_has_link(adapter);
4042         if (link) {
4043                 /* Cancel scheduled suspend requests. */
4044                 pm_runtime_resume(netdev->dev.parent);
4045
4046                 if (!netif_carrier_ok(netdev)) {
4047                         u32 ctrl;
4048                         e1000_get_speed_and_duplex(hw,
4049                                                    &adapter->link_speed,
4050                                                    &adapter->link_duplex);
4051
4052                         ctrl = E1000_READ_REG(hw, E1000_CTRL);
4053                         /* Links status message must follow this format */
4054                         printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
4055                                  "Flow Control: %s\n",
4056                                netdev->name,
4057                                adapter->link_speed,
4058                                adapter->link_duplex == FULL_DUPLEX ?
4059                                  "Full Duplex" : "Half Duplex",
4060                                ((ctrl & E1000_CTRL_TFCE) &&
4061                                 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX":
4062                                ((ctrl & E1000_CTRL_RFCE) ?  "RX" :
4063                                ((ctrl & E1000_CTRL_TFCE) ?  "TX" : "None")));
4064                         /* adjust timeout factor according to speed/duplex */
4065                         adapter->tx_timeout_factor = 1;
4066                         switch (adapter->link_speed) {
4067                         case SPEED_10:
4068                                 adapter->tx_timeout_factor = 14;
4069                                 break;
4070                         case SPEED_100:
4071                                 /* maybe add some timeout factor ? */
4072                                 break;
4073                         }
4074
4075                         netif_carrier_on(netdev);
4076                         netif_tx_wake_all_queues(netdev);
4077
4078                         igb_ping_all_vfs(adapter);
4079 #ifdef IFLA_VF_MAX
4080                         igb_check_vf_rate_limit(adapter);
4081 #endif /* IFLA_VF_MAX */
4082
4083                         /* link state has changed, schedule phy info update */
4084                         if (!test_bit(__IGB_DOWN, &adapter->state))
4085                                 mod_timer(&adapter->phy_info_timer,
4086                                           round_jiffies(jiffies + 2 * HZ));
4087                 }
4088         } else {
4089                 if (netif_carrier_ok(netdev)) {
4090                         adapter->link_speed = 0;
4091                         adapter->link_duplex = 0;
4092                         /* check for thermal sensor event on i350 */
4093                         if (hw->mac.type == e1000_i350) {
4094                                 thstat = E1000_READ_REG(hw, E1000_THSTAT);
4095                                 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
4096                                 if ((hw->phy.media_type ==
4097                                         e1000_media_type_copper) &&
4098                                         !(ctrl_ext &
4099                                         E1000_CTRL_EXT_LINK_MODE_SGMII)) {
4100                                         if (thstat & E1000_THSTAT_PWR_DOWN) {
4101                                                 printk(KERN_ERR "igb: %s The "
4102                                                 "network adapter was stopped "
4103                                                 "because it overheated.\n",
4104                                                 netdev->name);
4105                                         }
4106                                         if (thstat & E1000_THSTAT_LINK_THROTTLE) {
4107                                                 printk(KERN_INFO
4108                                                         "igb: %s The network "
4109                                                         "adapter supported "
4110                                                         "link speed "
4111                                                         "was downshifted "
4112                                                         "because it "
4113                                                         "overheated.\n",
4114                                                         netdev->name);
4115                                         }
4116                                 }
4117                         }
4118
4119                         /* Links status message must follow this format */
4120                         printk(KERN_INFO "igb: %s NIC Link is Down\n",
4121                                netdev->name);
4122                         netif_carrier_off(netdev);
4123                         netif_tx_stop_all_queues(netdev);
4124
4125                         igb_ping_all_vfs(adapter);
4126
4127                         /* link state has changed, schedule phy info update */
4128                         if (!test_bit(__IGB_DOWN, &adapter->state))
4129                                 mod_timer(&adapter->phy_info_timer,
4130                                           round_jiffies(jiffies + 2 * HZ));
4131
4132                         pm_schedule_suspend(netdev->dev.parent,
4133                                             MSEC_PER_SEC * 5);
4134                 }
4135         }
4136
4137         igb_update_stats(adapter);
4138
4139         for (i = 0; i < adapter->num_tx_queues; i++) {
4140                 struct igb_ring *tx_ring = adapter->tx_ring[i];
4141                 if (!netif_carrier_ok(netdev)) {
4142                         /* We've lost link, so the controller stops DMA,
4143                          * but we've got queued Tx work that's never going
4144                          * to get done, so reset controller to flush Tx.
4145                          * (Do the reset outside of interrupt context). */
4146                         if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
4147                                 adapter->tx_timeout_count++;
4148                                 schedule_work(&adapter->reset_task);
4149                                 /* return immediately since reset is imminent */
4150                                 return;
4151                         }
4152                 }
4153
4154                 /* Force detection of hung controller every watchdog period */
4155                 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
4156         }
4157
4158         /* Cause software interrupt to ensure rx ring is cleaned */
4159         if (adapter->msix_entries) {
4160                 u32 eics = 0;
4161                 for (i = 0; i < adapter->num_q_vectors; i++)
4162                         eics |= adapter->q_vector[i]->eims_value;
4163                 E1000_WRITE_REG(hw, E1000_EICS, eics);
4164         } else {
4165                 E1000_WRITE_REG(hw, E1000_ICS, E1000_ICS_RXDMT0);
4166         }
4167
4168         igb_spoof_check(adapter);
4169
4170         /* Reset the timer */
4171         if (!test_bit(__IGB_DOWN, &adapter->state))
4172                 mod_timer(&adapter->watchdog_timer,
4173                           round_jiffies(jiffies + 2 * HZ));
4174 }
4175
4176 static void igb_dma_err_task(struct work_struct *work)
4177 {
4178         struct igb_adapter *adapter = container_of(work,
4179                                                    struct igb_adapter,
4180                                                    dma_err_task);
4181         int vf;
4182         struct e1000_hw *hw = &adapter->hw;
4183         struct net_device *netdev = adapter->netdev;
4184         u32 hgptc;
4185         u32 ciaa, ciad;
4186
4187         hgptc = E1000_READ_REG(hw, E1000_HGPTC);
4188         if (hgptc) /* If incrementing then no need for the check below */
4189                 goto dma_timer_reset;
4190         /*
4191          * Check to see if a bad DMA write target from an errant or
4192          * malicious VF has caused a PCIe error.  If so then we can
4193          * issue a VFLR to the offending VF(s) and then resume without
4194          * requesting a full slot reset.
4195          */
4196
4197         for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4198                 ciaa = (vf << 16) | 0x80000000;
4199                 /* 32 bit read so align, we really want status at offset 6 */
4200                 ciaa |= PCI_COMMAND;
4201                 E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4202                 ciad = E1000_READ_REG(hw, E1000_CIAD);
4203                 ciaa &= 0x7FFFFFFF;
4204                 /* disable debug mode asap after reading data */
4205                 E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4206                 /* Get the upper 16 bits which will be the PCI status reg */
4207                 ciad >>= 16;
4208                 if (ciad & (PCI_STATUS_REC_MASTER_ABORT |
4209                             PCI_STATUS_REC_TARGET_ABORT |
4210                             PCI_STATUS_SIG_SYSTEM_ERROR)) {
4211                         netdev_err(netdev, "VF %d suffered error\n", vf);
4212                         /* Issue VFLR */
4213                         ciaa = (vf << 16) | 0x80000000;
4214                         ciaa |= 0xA8;
4215                         E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4216                         ciad = 0x00008000;  /* VFLR */
4217                         E1000_WRITE_REG(hw, E1000_CIAD, ciad);
4218                         ciaa &= 0x7FFFFFFF;
4219                         E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4220                 }
4221         }
4222 dma_timer_reset:
4223         /* Reset the timer */
4224         if (!test_bit(__IGB_DOWN, &adapter->state))
4225                 mod_timer(&adapter->dma_err_timer,
4226                           round_jiffies(jiffies + HZ / 10));
4227 }
4228
4229 /**
4230  * igb_dma_err_timer - Timer Call-back
4231  * @data: pointer to adapter cast into an unsigned long
4232  **/
4233 static void igb_dma_err_timer(unsigned long data)
4234 {
4235         struct igb_adapter *adapter = (struct igb_adapter *)data;
4236         /* Do the rest outside of interrupt context */
4237         schedule_work(&adapter->dma_err_task);
4238 }
4239
4240 enum latency_range {
4241         lowest_latency = 0,
4242         low_latency = 1,
4243         bulk_latency = 2,
4244         latency_invalid = 255
4245 };
4246
4247 /**
4248  * igb_update_ring_itr - update the dynamic ITR value based on packet size
4249  *
4250  *      Stores a new ITR value based on strictly on packet size.  This
4251  *      algorithm is less sophisticated than that used in igb_update_itr,
4252  *      due to the difficulty of synchronizing statistics across multiple
4253  *      receive rings.  The divisors and thresholds used by this function
4254  *      were determined based on theoretical maximum wire speed and testing
4255  *      data, in order to minimize response time while increasing bulk
4256  *      throughput.
4257  *      This functionality is controlled by the InterruptThrottleRate module
4258  *      parameter (see igb_param.c)
4259  *      NOTE:  This function is called only when operating in a multiqueue
4260  *             receive environment.
4261  * @q_vector: pointer to q_vector
4262  **/
4263 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
4264 {
4265         int new_val = q_vector->itr_val;
4266         int avg_wire_size = 0;
4267         struct igb_adapter *adapter = q_vector->adapter;
4268         unsigned int packets;
4269
4270         /* For non-gigabit speeds, just fix the interrupt rate at 4000
4271          * ints/sec - ITR timer value of 120 ticks.
4272          */
4273         if (adapter->link_speed != SPEED_1000) {
4274                 new_val = IGB_4K_ITR;
4275                 goto set_itr_val;
4276         }
4277
4278         packets = q_vector->rx.total_packets;
4279         if (packets)
4280                 avg_wire_size = q_vector->rx.total_bytes / packets;
4281
4282         packets = q_vector->tx.total_packets;
4283         if (packets)
4284                 avg_wire_size = max_t(u32, avg_wire_size,
4285                                       q_vector->tx.total_bytes / packets);
4286
4287         /* if avg_wire_size isn't set no work was done */
4288         if (!avg_wire_size)
4289                 goto clear_counts;
4290
4291         /* Add 24 bytes to size to account for CRC, preamble, and gap */
4292         avg_wire_size += 24;
4293
4294         /* Don't starve jumbo frames */
4295         avg_wire_size = min(avg_wire_size, 3000);
4296
4297         /* Give a little boost to mid-size frames */
4298         if ((avg_wire_size > 300) && (avg_wire_size < 1200))
4299                 new_val = avg_wire_size / 3;
4300         else
4301                 new_val = avg_wire_size / 2;
4302
4303         /* conservative mode (itr 3) eliminates the lowest_latency setting */
4304         if (new_val < IGB_20K_ITR &&
4305             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4306              (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4307                 new_val = IGB_20K_ITR;
4308
4309 set_itr_val:
4310         if (new_val != q_vector->itr_val) {
4311                 q_vector->itr_val = new_val;
4312                 q_vector->set_itr = 1;
4313         }
4314 clear_counts:
4315         q_vector->rx.total_bytes = 0;
4316         q_vector->rx.total_packets = 0;
4317         q_vector->tx.total_bytes = 0;
4318         q_vector->tx.total_packets = 0;
4319 }
4320
4321 /**
4322  * igb_update_itr - update the dynamic ITR value based on statistics
4323  *      Stores a new ITR value based on packets and byte
4324  *      counts during the last interrupt.  The advantage of per interrupt
4325  *      computation is faster updates and more accurate ITR for the current
4326  *      traffic pattern.  Constants in this function were computed
4327  *      based on theoretical maximum wire speed and thresholds were set based
4328  *      on testing data as well as attempting to minimize response time
4329  *      while increasing bulk throughput.
4330  *      this functionality is controlled by the InterruptThrottleRate module
4331  *      parameter (see igb_param.c)
4332  *      NOTE:  These calculations are only valid when operating in a single-
4333  *             queue environment.
4334  * @q_vector: pointer to q_vector
4335  * @ring_container: ring info to update the itr for
4336  **/
4337 static void igb_update_itr(struct igb_q_vector *q_vector,
4338                            struct igb_ring_container *ring_container)
4339 {
4340         unsigned int packets = ring_container->total_packets;
4341         unsigned int bytes = ring_container->total_bytes;
4342         u8 itrval = ring_container->itr;
4343
4344         /* no packets, exit with status unchanged */
4345         if (packets == 0)
4346                 return;
4347
4348         switch (itrval) {
4349         case lowest_latency:
4350                 /* handle TSO and jumbo frames */
4351                 if (bytes/packets > 8000)
4352                         itrval = bulk_latency;
4353                 else if ((packets < 5) && (bytes > 512))
4354                         itrval = low_latency;
4355                 break;
4356         case low_latency:  /* 50 usec aka 20000 ints/s */
4357                 if (bytes > 10000) {
4358                         /* this if handles the TSO accounting */
4359                         if (bytes/packets > 8000) {
4360                                 itrval = bulk_latency;
4361                         } else if ((packets < 10) || ((bytes/packets) > 1200)) {
4362                                 itrval = bulk_latency;
4363                         } else if ((packets > 35)) {
4364                                 itrval = lowest_latency;
4365                         }
4366                 } else if (bytes/packets > 2000) {
4367                         itrval = bulk_latency;
4368                 } else if (packets <= 2 && bytes < 512) {
4369                         itrval = lowest_latency;
4370                 }
4371                 break;
4372         case bulk_latency: /* 250 usec aka 4000 ints/s */
4373                 if (bytes > 25000) {
4374                         if (packets > 35)
4375                                 itrval = low_latency;
4376                 } else if (bytes < 1500) {
4377                         itrval = low_latency;
4378                 }
4379                 break;
4380         }
4381
4382         /* clear work counters since we have the values we need */
4383         ring_container->total_bytes = 0;
4384         ring_container->total_packets = 0;
4385
4386         /* write updated itr to ring container */
4387         ring_container->itr = itrval;
4388 }
4389
4390 static void igb_set_itr(struct igb_q_vector *q_vector)
4391 {
4392         struct igb_adapter *adapter = q_vector->adapter;
4393         u32 new_itr = q_vector->itr_val;
4394         u8 current_itr = 0;
4395
4396         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4397         if (adapter->link_speed != SPEED_1000) {
4398                 current_itr = 0;
4399                 new_itr = IGB_4K_ITR;
4400                 goto set_itr_now;
4401         }
4402
4403         igb_update_itr(q_vector, &q_vector->tx);
4404         igb_update_itr(q_vector, &q_vector->rx);
4405
4406         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
4407
4408         /* conservative mode (itr 3) eliminates the lowest_latency setting */
4409         if (current_itr == lowest_latency &&
4410             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4411              (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4412                 current_itr = low_latency;
4413
4414         switch (current_itr) {
4415         /* counts and packets in update_itr are dependent on these numbers */
4416         case lowest_latency:
4417                 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
4418                 break;
4419         case low_latency:
4420                 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
4421                 break;
4422         case bulk_latency:
4423                 new_itr = IGB_4K_ITR;  /* 4,000 ints/sec */
4424                 break;
4425         default:
4426                 break;
4427         }
4428
4429 set_itr_now:
4430         if (new_itr != q_vector->itr_val) {
4431                 /* this attempts to bias the interrupt rate towards Bulk
4432                  * by adding intermediate steps when interrupt rate is
4433                  * increasing */
4434                 new_itr = new_itr > q_vector->itr_val ?
4435                              max((new_itr * q_vector->itr_val) /
4436                                  (new_itr + (q_vector->itr_val >> 2)),
4437                                  new_itr) :
4438                              new_itr;
4439                 /* Don't write the value here; it resets the adapter's
4440                  * internal timer, and causes us to delay far longer than
4441                  * we should between interrupts.  Instead, we write the ITR
4442                  * value at the beginning of the next interrupt so the timing
4443                  * ends up being correct.
4444                  */
4445                 q_vector->itr_val = new_itr;
4446                 q_vector->set_itr = 1;
4447         }
4448 }
4449
4450 void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
4451                      u32 type_tucmd, u32 mss_l4len_idx)
4452 {
4453         struct e1000_adv_tx_context_desc *context_desc;
4454         u16 i = tx_ring->next_to_use;
4455
4456         context_desc = IGB_TX_CTXTDESC(tx_ring, i);
4457
4458         i++;
4459         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
4460
4461         /* set bits to identify this as an advanced context descriptor */
4462         type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
4463
4464         /* For 82575, context index must be unique per ring. */
4465         if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4466                 mss_l4len_idx |= tx_ring->reg_idx << 4;
4467
4468         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
4469         context_desc->seqnum_seed       = 0;
4470         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
4471         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
4472 }
4473
4474 static int igb_tso(struct igb_ring *tx_ring,
4475                    struct igb_tx_buffer *first,
4476                    u8 *hdr_len)
4477 {
4478 #ifdef NETIF_F_TSO
4479         struct sk_buff *skb = first->skb;
4480         u32 vlan_macip_lens, type_tucmd;
4481         u32 mss_l4len_idx, l4len;
4482
4483         if (!skb_is_gso(skb))
4484 #endif /* NETIF_F_TSO */
4485                 return 0;
4486 #ifdef NETIF_F_TSO
4487
4488         if (skb_header_cloned(skb)) {
4489                 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
4490                 if (err)
4491                         return err;
4492         }
4493
4494         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4495         type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
4496
4497         if (first->protocol == __constant_htons(ETH_P_IP)) {
4498                 struct iphdr *iph = ip_hdr(skb);
4499                 iph->tot_len = 0;
4500                 iph->check = 0;
4501                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4502                                                          iph->daddr, 0,
4503                                                          IPPROTO_TCP,
4504                                                          0);
4505                 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4506                 first->tx_flags |= IGB_TX_FLAGS_TSO |
4507                                    IGB_TX_FLAGS_CSUM |
4508                                    IGB_TX_FLAGS_IPV4;
4509 #ifdef NETIF_F_TSO6
4510         } else if (skb_is_gso_v6(skb)) {
4511                 ipv6_hdr(skb)->payload_len = 0;
4512                 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4513                                                        &ipv6_hdr(skb)->daddr,
4514                                                        0, IPPROTO_TCP, 0);
4515                 first->tx_flags |= IGB_TX_FLAGS_TSO |
4516                                    IGB_TX_FLAGS_CSUM;
4517 #endif
4518         }
4519
4520         /* compute header lengths */
4521         l4len = tcp_hdrlen(skb);
4522         *hdr_len = skb_transport_offset(skb) + l4len;
4523
4524         /* update gso size and bytecount with header size */
4525         first->gso_segs = skb_shinfo(skb)->gso_segs;
4526         first->bytecount += (first->gso_segs - 1) * *hdr_len;
4527
4528         /* MSS L4LEN IDX */
4529         mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
4530         mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
4531
4532         /* VLAN MACLEN IPLEN */
4533         vlan_macip_lens = skb_network_header_len(skb);
4534         vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4535         vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4536
4537         igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4538
4539         return 1;
4540 #endif  /* NETIF_F_TSO */
4541 }
4542
4543 static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
4544 {
4545         struct sk_buff *skb = first->skb;
4546         u32 vlan_macip_lens = 0;
4547         u32 mss_l4len_idx = 0;
4548         u32 type_tucmd = 0;
4549
4550         if (skb->ip_summed != CHECKSUM_PARTIAL) {
4551                 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
4552                         return;
4553         } else {
4554                 u8 l4_hdr = 0;
4555                 switch (first->protocol) {
4556                 case __constant_htons(ETH_P_IP):
4557                         vlan_macip_lens |= skb_network_header_len(skb);
4558                         type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4559                         l4_hdr = ip_hdr(skb)->protocol;
4560                         break;
4561 #ifdef NETIF_F_IPV6_CSUM
4562                 case __constant_htons(ETH_P_IPV6):
4563                         vlan_macip_lens |= skb_network_header_len(skb);
4564                         l4_hdr = ipv6_hdr(skb)->nexthdr;
4565                         break;
4566 #endif
4567                 default:
4568                         if (unlikely(net_ratelimit())) {
4569                                 dev_warn(tx_ring->dev,
4570                                  "partial checksum but proto=%x!\n",
4571                                  first->protocol);
4572                         }
4573                         break;
4574                 }
4575
4576                 switch (l4_hdr) {
4577                 case IPPROTO_TCP:
4578                         type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4579                         mss_l4len_idx = tcp_hdrlen(skb) <<
4580                                         E1000_ADVTXD_L4LEN_SHIFT;
4581                         break;
4582 #ifdef HAVE_SCTP
4583                 case IPPROTO_SCTP:
4584                         type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4585                         mss_l4len_idx = sizeof(struct sctphdr) <<
4586                                         E1000_ADVTXD_L4LEN_SHIFT;
4587                         break;
4588 #endif
4589                 case IPPROTO_UDP:
4590                         mss_l4len_idx = sizeof(struct udphdr) <<
4591                                         E1000_ADVTXD_L4LEN_SHIFT;
4592                         break;
4593                 default:
4594                         if (unlikely(net_ratelimit())) {
4595                                 dev_warn(tx_ring->dev,
4596                                  "partial checksum but l4 proto=%x!\n",
4597                                  l4_hdr);
4598                         }
4599                         break;
4600                 }
4601
4602                 /* update TX checksum flag */
4603                 first->tx_flags |= IGB_TX_FLAGS_CSUM;
4604         }
4605
4606         vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4607         vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4608
4609         igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4610 }
4611
4612 static __le32 igb_tx_cmd_type(u32 tx_flags)
4613 {
4614         /* set type for advanced descriptor with frame checksum insertion */
4615         __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA |
4616                                       E1000_ADVTXD_DCMD_IFCS |
4617                                       E1000_ADVTXD_DCMD_DEXT);
4618
4619         /* set HW vlan bit if vlan is present */
4620         if (tx_flags & IGB_TX_FLAGS_VLAN)
4621                 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE);
4622
4623         /* set timestamp bit if present */
4624         if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4625                 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP);
4626
4627         /* set segmentation bits for TSO */
4628         if (tx_flags & IGB_TX_FLAGS_TSO)
4629                 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE);
4630
4631         return cmd_type;
4632 }
4633
4634 static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
4635                                  union e1000_adv_tx_desc *tx_desc,
4636                                  u32 tx_flags, unsigned int paylen)
4637 {
4638         u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
4639
4640         /* 82575 requires a unique index per ring if any offload is enabled */
4641         if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) &&
4642             test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4643                 olinfo_status |= tx_ring->reg_idx << 4;
4644
4645         /* insert L4 checksum */
4646         if (tx_flags & IGB_TX_FLAGS_CSUM) {
4647                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4648
4649                 /* insert IPv4 checksum */
4650                 if (tx_flags & IGB_TX_FLAGS_IPV4)
4651                         olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4652         }
4653
4654         tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4655 }
4656
4657 /*
4658  * The largest size we can write to the descriptor is 65535.  In order to
4659  * maintain a power of two alignment we have to limit ourselves to 32K.
4660  */
4661 #define IGB_MAX_TXD_PWR 15
4662 #define IGB_MAX_DATA_PER_TXD    (1<<IGB_MAX_TXD_PWR)
4663
4664 static void igb_tx_map(struct igb_ring *tx_ring,
4665                        struct igb_tx_buffer *first,
4666                        const u8 hdr_len)
4667 {
4668         struct sk_buff *skb = first->skb;
4669         struct igb_tx_buffer *tx_buffer;
4670         union e1000_adv_tx_desc *tx_desc;
4671         dma_addr_t dma;
4672 #ifdef MAX_SKB_FRAGS
4673         struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
4674         unsigned int data_len = skb->data_len;
4675 #endif
4676         unsigned int size = skb_headlen(skb);
4677         unsigned int paylen = skb->len - hdr_len;
4678         __le32 cmd_type;
4679         u32 tx_flags = first->tx_flags;
4680         u16 i = tx_ring->next_to_use;
4681
4682         tx_desc = IGB_TX_DESC(tx_ring, i);
4683
4684         igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen);
4685         cmd_type = igb_tx_cmd_type(tx_flags);
4686
4687         dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4688         if (dma_mapping_error(tx_ring->dev, dma))
4689                 goto dma_error;
4690
4691         /* record length, and DMA address */
4692         dma_unmap_len_set(first, len, size);
4693         dma_unmap_addr_set(first, dma, dma);
4694         tx_desc->read.buffer_addr = cpu_to_le64(dma);
4695
4696 #ifdef MAX_SKB_FRAGS
4697         for (;;) {
4698 #endif
4699                 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
4700                         tx_desc->read.cmd_type_len =
4701                                 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD);
4702
4703                         i++;
4704                         tx_desc++;
4705                         if (i == tx_ring->count) {
4706                                 tx_desc = IGB_TX_DESC(tx_ring, 0);
4707                                 i = 0;
4708                         }
4709
4710                         dma += IGB_MAX_DATA_PER_TXD;
4711                         size -= IGB_MAX_DATA_PER_TXD;
4712
4713                         tx_desc->read.olinfo_status = 0;
4714                         tx_desc->read.buffer_addr = cpu_to_le64(dma);
4715                 }
4716
4717 #ifdef MAX_SKB_FRAGS
4718                 if (likely(!data_len))
4719                         break;
4720
4721                 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
4722
4723                 i++;
4724                 tx_desc++;
4725                 if (i == tx_ring->count) {
4726                         tx_desc = IGB_TX_DESC(tx_ring, 0);
4727                         i = 0;
4728                 }
4729
4730                 size = skb_frag_size(frag);
4731                 data_len -= size;
4732
4733                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
4734                                 DMA_TO_DEVICE);
4735                 if (dma_mapping_error(tx_ring->dev, dma))
4736                         goto dma_error;
4737
4738                 tx_buffer = &tx_ring->tx_buffer_info[i];
4739                 dma_unmap_len_set(tx_buffer, len, size);
4740                 dma_unmap_addr_set(tx_buffer, dma, dma);
4741
4742                 tx_desc->read.olinfo_status = 0;
4743                 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4744
4745                 frag++;
4746         }
4747
4748 #endif /* MAX_SKB_FRAGS */
4749 #ifdef CONFIG_BQL
4750         netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
4751 #endif /* CONFIG_BQL */
4752
4753         /* write last descriptor with RS and EOP bits */
4754         cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD);
4755         tx_desc->read.cmd_type_len = cmd_type;
4756
4757         /* set the timestamp */
4758         first->time_stamp = jiffies;
4759
4760         /*
4761          * Force memory writes to complete before letting h/w know there
4762          * are new descriptors to fetch.  (Only applicable for weak-ordered
4763          * memory model archs, such as IA-64).
4764          *
4765          * We also need this memory barrier to make certain all of the
4766          * status bits have been updated before next_to_watch is written.
4767          */
4768         wmb();
4769
4770         /* set next_to_watch value indicating a packet is present */
4771         first->next_to_watch = tx_desc;
4772
4773         i++;
4774         if (i == tx_ring->count)
4775                 i = 0;
4776
4777         tx_ring->next_to_use = i;
4778
4779         writel(i, tx_ring->tail);
4780
4781         /* we need this if more than one processor can write to our tail
4782          * at a time, it syncronizes IO on IA64/Altix systems */
4783         mmiowb();
4784
4785         return;
4786
4787 dma_error:
4788         dev_err(tx_ring->dev, "TX DMA map failed\n");
4789
4790         /* clear dma mappings for failed tx_buffer_info map */
4791         for (;;) {
4792                 tx_buffer= &tx_ring->tx_buffer_info[i];
4793                 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer);
4794                 if (tx_buffer == first)
4795                         break;
4796                 if (i == 0)
4797                         i = tx_ring->count;
4798                 i--;
4799         }
4800
4801         tx_ring->next_to_use = i;
4802 }
4803
4804 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4805 {
4806         struct net_device *netdev = netdev_ring(tx_ring);
4807
4808         if (netif_is_multiqueue(netdev))
4809                 netif_stop_subqueue(netdev, ring_queue_index(tx_ring));
4810         else
4811                 netif_stop_queue(netdev);
4812
4813         /* Herbert's original patch had:
4814          *  smp_mb__after_netif_stop_queue();
4815          * but since that doesn't exist yet, just open code it. */
4816         smp_mb();
4817
4818         /* We need to check again in a case another CPU has just
4819          * made room available. */
4820         if (igb_desc_unused(tx_ring) < size)
4821                 return -EBUSY;
4822
4823         /* A reprieve! */
4824         if (netif_is_multiqueue(netdev))
4825                 netif_wake_subqueue(netdev, ring_queue_index(tx_ring));
4826         else
4827                 netif_wake_queue(netdev);
4828
4829         tx_ring->tx_stats.restart_queue++;
4830
4831         return 0;
4832 }
4833
4834 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4835 {
4836         if (igb_desc_unused(tx_ring) >= size)
4837                 return 0;
4838         return __igb_maybe_stop_tx(tx_ring, size);
4839 }
4840
4841 netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4842                                 struct igb_ring *tx_ring)
4843 {
4844         struct igb_tx_buffer *first;
4845         int tso;
4846         u32 tx_flags = 0;
4847         __be16 protocol = vlan_get_protocol(skb);
4848         u8 hdr_len = 0;
4849
4850         /* need: 1 descriptor per page,
4851          *       + 2 desc gap to keep tail from touching head,
4852          *       + 1 desc for skb->data,
4853          *       + 1 desc for context descriptor,
4854          * otherwise try next time */
4855         if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
4856                 /* this is a hard error */
4857                 return NETDEV_TX_BUSY;
4858         }
4859
4860         /* record the location of the first descriptor for this packet */
4861         first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4862         first->skb = skb;
4863         first->bytecount = skb->len;
4864         first->gso_segs = 1;
4865
4866 #ifdef HAVE_HW_TIME_STAMP
4867 #ifdef SKB_SHARED_TX_IS_UNION
4868         if (unlikely(skb_shinfo(skb)->tx_flags.flags & SKBTX_HW_TSTAMP)) {
4869                 skb_shinfo(skb)->tx_flags.flags |= SKBTX_IN_PROGRESS;
4870                 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4871         }
4872 #else
4873         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4874                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4875                 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4876         }
4877 #endif
4878
4879 #endif
4880         if (vlan_tx_tag_present(skb)) {
4881                 tx_flags |= IGB_TX_FLAGS_VLAN;
4882                 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4883         }
4884
4885         /* record initial flags and protocol */
4886         first->tx_flags = tx_flags;
4887         first->protocol = protocol;
4888
4889         tso = igb_tso(tx_ring, first, &hdr_len);
4890         if (tso < 0)
4891                 goto out_drop;
4892         else if (!tso)
4893                 igb_tx_csum(tx_ring, first);
4894
4895         igb_tx_map(tx_ring, first, hdr_len);
4896
4897 #ifndef HAVE_TRANS_START_IN_QUEUE
4898         netdev_ring(tx_ring)->trans_start = jiffies;
4899
4900 #endif
4901         /* Make sure there is space in the ring for the next send. */
4902         igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
4903
4904         return NETDEV_TX_OK;
4905
4906 out_drop:
4907         igb_unmap_and_free_tx_resource(tx_ring, first);
4908
4909         return NETDEV_TX_OK;
4910 }
4911
4912 #ifdef HAVE_TX_MQ
4913 static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4914                                                     struct sk_buff *skb)
4915 {
4916         unsigned int r_idx = skb->queue_mapping;
4917
4918         if (r_idx >= adapter->num_tx_queues)
4919                 r_idx = r_idx % adapter->num_tx_queues;
4920
4921         return adapter->tx_ring[r_idx];
4922 }
4923 #else
4924 #define igb_tx_queue_mapping(_adapter, _skb) (_adapter)->tx_ring[0]
4925 #endif
4926
4927 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4928                                   struct net_device *netdev)
4929 {
4930         struct igb_adapter *adapter = netdev_priv(netdev);
4931
4932         if (test_bit(__IGB_DOWN, &adapter->state)) {
4933                 dev_kfree_skb_any(skb);
4934                 return NETDEV_TX_OK;
4935         }
4936
4937         if (skb->len <= 0) {
4938                 dev_kfree_skb_any(skb);
4939                 return NETDEV_TX_OK;
4940         }
4941
4942         /*
4943          * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4944          * in order to meet this minimum size requirement.
4945          */
4946         if (skb->len < 17) {
4947                 if (skb_padto(skb, 17))
4948                         return NETDEV_TX_OK;
4949                 skb->len = 17;
4950         }
4951
4952         return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
4953 }
4954
4955 /**
4956  * igb_tx_timeout - Respond to a Tx Hang
4957  * @netdev: network interface device structure
4958  **/
4959 static void igb_tx_timeout(struct net_device *netdev)
4960 {
4961         struct igb_adapter *adapter = netdev_priv(netdev);
4962         struct e1000_hw *hw = &adapter->hw;
4963
4964         /* Do the reset outside of interrupt context */
4965         adapter->tx_timeout_count++;
4966
4967         if (hw->mac.type >= e1000_82580)
4968                 hw->dev_spec._82575.global_device_reset = true;
4969
4970         schedule_work(&adapter->reset_task);
4971         E1000_WRITE_REG(hw, E1000_EICS,
4972                         (adapter->eims_enable_mask & ~adapter->eims_other));
4973 }
4974
4975 static void igb_reset_task(struct work_struct *work)
4976 {
4977         struct igb_adapter *adapter;
4978         adapter = container_of(work, struct igb_adapter, reset_task);
4979
4980         igb_reinit_locked(adapter);
4981 }
4982
4983 /**
4984  * igb_get_stats - Get System Network Statistics
4985  * @netdev: network interface device structure
4986  *
4987  * Returns the address of the device statistics structure.
4988  * The statistics are updated here and also from the timer callback.
4989  **/
4990 static struct net_device_stats *igb_get_stats(struct net_device *netdev)
4991 {
4992         struct igb_adapter *adapter = netdev_priv(netdev);
4993
4994         if (!test_bit(__IGB_RESETTING, &adapter->state))
4995                 igb_update_stats(adapter);
4996
4997 #ifdef HAVE_NETDEV_STATS_IN_NETDEV
4998         /* only return the current stats */
4999         return &netdev->stats;
5000 #else
5001         /* only return the current stats */
5002         return &adapter->net_stats;
5003 #endif /* HAVE_NETDEV_STATS_IN_NETDEV */
5004 }
5005
5006 /**
5007  * igb_change_mtu - Change the Maximum Transfer Unit
5008  * @netdev: network interface device structure
5009  * @new_mtu: new value for maximum frame size
5010  *
5011  * Returns 0 on success, negative on failure
5012  **/
5013 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
5014 {
5015         struct igb_adapter *adapter = netdev_priv(netdev);
5016         struct pci_dev *pdev = adapter->pdev;
5017         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
5018 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
5019         u32 rx_buffer_len, i;
5020 #endif
5021
5022         if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
5023                 dev_err(pci_dev_to_dev(pdev), "Invalid MTU setting\n");
5024                 return -EINVAL;
5025         }
5026
5027 #define MAX_STD_JUMBO_FRAME_SIZE 9238
5028         if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
5029                 dev_err(pci_dev_to_dev(pdev), "MTU > 9216 not supported.\n");
5030                 return -EINVAL;
5031         }
5032
5033         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
5034                 usleep_range(1000, 2000);
5035
5036         /* igb_down has a dependency on max_frame_size */
5037         adapter->max_frame_size = max_frame;
5038
5039 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
5040 #ifdef IGB_PER_PKT_TIMESTAMP
5041         if (adapter->hw.mac.type >= e1000_82580)
5042                 max_frame += IGB_TS_HDR_LEN;
5043
5044 #endif
5045         /*
5046          * RLPML prevents us from receiving a frame larger than max_frame so
5047          * it is safe to just set the rx_buffer_len to max_frame without the
5048          * risk of an skb over panic.
5049          */
5050         if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
5051                 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
5052         else
5053                 rx_buffer_len = max_frame;
5054
5055 #endif
5056         if (netif_running(netdev))
5057                 igb_down(adapter);
5058
5059         dev_info(pci_dev_to_dev(pdev), "changing MTU from %d to %d\n",
5060                 netdev->mtu, new_mtu);
5061         netdev->mtu = new_mtu;
5062
5063 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
5064         for (i = 0; i < adapter->num_rx_queues; i++)
5065                 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
5066
5067 #endif
5068         if (netif_running(netdev))
5069                 igb_up(adapter);
5070         else
5071                 igb_reset(adapter);
5072
5073         clear_bit(__IGB_RESETTING, &adapter->state);
5074
5075         return 0;
5076 }
5077
5078 /**
5079  * igb_update_stats - Update the board statistics counters
5080  * @adapter: board private structure
5081  **/
5082
5083 void igb_update_stats(struct igb_adapter *adapter)
5084 {
5085 #ifdef HAVE_NETDEV_STATS_IN_NETDEV
5086         struct net_device_stats *net_stats = &adapter->netdev->stats;
5087 #else
5088         struct net_device_stats *net_stats = &adapter->net_stats;
5089 #endif /* HAVE_NETDEV_STATS_IN_NETDEV */
5090         struct e1000_hw *hw = &adapter->hw;
5091 #ifdef HAVE_PCI_ERS
5092         struct pci_dev *pdev = adapter->pdev;
5093 #endif
5094         u32 reg, mpc;
5095         u16 phy_tmp;
5096         int i;
5097         u64 bytes, packets;
5098 #ifndef IGB_NO_LRO
5099         u32 flushed = 0, coal = 0, recycled = 0;
5100         struct igb_q_vector *q_vector;
5101 #endif
5102
5103 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
5104
5105         /*
5106          * Prevent stats update while adapter is being reset, or if the pci
5107          * connection is down.
5108          */
5109         if (adapter->link_speed == 0)
5110                 return;
5111 #ifdef HAVE_PCI_ERS
5112         if (pci_channel_offline(pdev))
5113                 return;
5114
5115 #endif
5116 #ifndef IGB_NO_LRO
5117         for (i = 0; i < adapter->num_q_vectors; i++) {
5118                 q_vector = adapter->q_vector[i];
5119                 if (!q_vector || !q_vector->lrolist)
5120                         continue;
5121                 flushed += q_vector->lrolist->stats.flushed;
5122                 coal += q_vector->lrolist->stats.coal;
5123                 recycled += q_vector->lrolist->stats.recycled;
5124         }
5125         adapter->lro_stats.flushed = flushed;
5126         adapter->lro_stats.coal = coal;
5127         adapter->lro_stats.recycled = recycled;
5128
5129 #endif
5130         bytes = 0;
5131         packets = 0;
5132         for (i = 0; i < adapter->num_rx_queues; i++) {
5133                 u32 rqdpc_tmp = E1000_READ_REG(hw, E1000_RQDPC(i)) & 0x0FFF;
5134                 struct igb_ring *ring = adapter->rx_ring[i];
5135                 ring->rx_stats.drops += rqdpc_tmp;
5136                 net_stats->rx_fifo_errors += rqdpc_tmp;
5137 #ifdef CONFIG_IGB_VMDQ_NETDEV
5138                 if (!ring->vmdq_netdev) {
5139                         bytes += ring->rx_stats.bytes;
5140                         packets += ring->rx_stats.packets;
5141                 }
5142 #else
5143                 bytes += ring->rx_stats.bytes;
5144                 packets += ring->rx_stats.packets;
5145 #endif
5146         }
5147
5148         net_stats->rx_bytes = bytes;
5149         net_stats->rx_packets = packets;
5150
5151         bytes = 0;
5152         packets = 0;
5153         for (i = 0; i < adapter->num_tx_queues; i++) {
5154                 struct igb_ring *ring = adapter->tx_ring[i];
5155 #ifdef CONFIG_IGB_VMDQ_NETDEV
5156                 if (!ring->vmdq_netdev) {
5157                         bytes += ring->tx_stats.bytes;
5158                         packets += ring->tx_stats.packets;
5159                 }
5160 #else
5161                 bytes += ring->tx_stats.bytes;
5162                 packets += ring->tx_stats.packets;
5163 #endif
5164         }
5165         net_stats->tx_bytes = bytes;
5166         net_stats->tx_packets = packets;
5167
5168         /* read stats registers */
5169         adapter->stats.crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
5170         adapter->stats.gprc += E1000_READ_REG(hw, E1000_GPRC);
5171         adapter->stats.gorc += E1000_READ_REG(hw, E1000_GORCL);
5172         E1000_READ_REG(hw, E1000_GORCH); /* clear GORCL */
5173         adapter->stats.bprc += E1000_READ_REG(hw, E1000_BPRC);
5174         adapter->stats.mprc += E1000_READ_REG(hw, E1000_MPRC);
5175         adapter->stats.roc += E1000_READ_REG(hw, E1000_ROC);
5176
5177         adapter->stats.prc64 += E1000_READ_REG(hw, E1000_PRC64);
5178         adapter->stats.prc127 += E1000_READ_REG(hw, E1000_PRC127);
5179         adapter->stats.prc255 += E1000_READ_REG(hw, E1000_PRC255);
5180         adapter->stats.prc511 += E1000_READ_REG(hw, E1000_PRC511);
5181         adapter->stats.prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
5182         adapter->stats.prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
5183         adapter->stats.symerrs += E1000_READ_REG(hw, E1000_SYMERRS);
5184         adapter->stats.sec += E1000_READ_REG(hw, E1000_SEC);
5185
5186         mpc = E1000_READ_REG(hw, E1000_MPC);
5187         adapter->stats.mpc += mpc;
5188         net_stats->rx_fifo_errors += mpc;
5189         adapter->stats.scc += E1000_READ_REG(hw, E1000_SCC);
5190         adapter->stats.ecol += E1000_READ_REG(hw, E1000_ECOL);
5191         adapter->stats.mcc += E1000_READ_REG(hw, E1000_MCC);
5192         adapter->stats.latecol += E1000_READ_REG(hw, E1000_LATECOL);
5193         adapter->stats.dc += E1000_READ_REG(hw, E1000_DC);
5194         adapter->stats.rlec += E1000_READ_REG(hw, E1000_RLEC);
5195         adapter->stats.xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
5196         adapter->stats.xontxc += E1000_READ_REG(hw, E1000_XONTXC);
5197         adapter->stats.xoffrxc += E1000_READ_REG(hw, E1000_XOFFRXC);
5198         adapter->stats.xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
5199         adapter->stats.fcruc += E1000_READ_REG(hw, E1000_FCRUC);
5200         adapter->stats.gptc += E1000_READ_REG(hw, E1000_GPTC);
5201         adapter->stats.gotc += E1000_READ_REG(hw, E1000_GOTCL);
5202         E1000_READ_REG(hw, E1000_GOTCH); /* clear GOTCL */
5203         adapter->stats.rnbc += E1000_READ_REG(hw, E1000_RNBC);
5204         adapter->stats.ruc += E1000_READ_REG(hw, E1000_RUC);
5205         adapter->stats.rfc += E1000_READ_REG(hw, E1000_RFC);
5206         adapter->stats.rjc += E1000_READ_REG(hw, E1000_RJC);
5207         adapter->stats.tor += E1000_READ_REG(hw, E1000_TORH);
5208         adapter->stats.tot += E1000_READ_REG(hw, E1000_TOTH);
5209         adapter->stats.tpr += E1000_READ_REG(hw, E1000_TPR);
5210
5211         adapter->stats.ptc64 += E1000_READ_REG(hw, E1000_PTC64);
5212         adapter->stats.ptc127 += E1000_READ_REG(hw, E1000_PTC127);
5213         adapter->stats.ptc255 += E1000_READ_REG(hw, E1000_PTC255);
5214         adapter->stats.ptc511 += E1000_READ_REG(hw, E1000_PTC511);
5215         adapter->stats.ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
5216         adapter->stats.ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
5217
5218         adapter->stats.mptc += E1000_READ_REG(hw, E1000_MPTC);
5219         adapter->stats.bptc += E1000_READ_REG(hw, E1000_BPTC);
5220
5221         adapter->stats.tpt += E1000_READ_REG(hw, E1000_TPT);
5222         adapter->stats.colc += E1000_READ_REG(hw, E1000_COLC);
5223
5224         adapter->stats.algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
5225         /* read internal phy sepecific stats */
5226         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
5227         if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
5228                 adapter->stats.rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
5229                 adapter->stats.tncrs += E1000_READ_REG(hw, E1000_TNCRS);
5230         }
5231
5232         adapter->stats.tsctc += E1000_READ_REG(hw, E1000_TSCTC);
5233         adapter->stats.tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
5234
5235         adapter->stats.iac += E1000_READ_REG(hw, E1000_IAC);
5236         adapter->stats.icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
5237         adapter->stats.icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
5238         adapter->stats.icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
5239         adapter->stats.ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
5240         adapter->stats.ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
5241         adapter->stats.ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
5242         adapter->stats.ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
5243         adapter->stats.icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
5244
5245         /* Fill out the OS statistics structure */
5246         net_stats->multicast = adapter->stats.mprc;
5247         net_stats->collisions = adapter->stats.colc;
5248
5249         /* Rx Errors */
5250
5251         /* RLEC on some newer hardware can be incorrect so build
5252          * our own version based on RUC and ROC */
5253         net_stats->rx_errors = adapter->stats.rxerrc +
5254                 adapter->stats.crcerrs + adapter->stats.algnerrc +
5255                 adapter->stats.ruc + adapter->stats.roc +
5256                 adapter->stats.cexterr;
5257         net_stats->rx_length_errors = adapter->stats.ruc +
5258                                       adapter->stats.roc;
5259         net_stats->rx_crc_errors = adapter->stats.crcerrs;
5260         net_stats->rx_frame_errors = adapter->stats.algnerrc;
5261         net_stats->rx_missed_errors = adapter->stats.mpc;
5262
5263         /* Tx Errors */
5264         net_stats->tx_errors = adapter->stats.ecol +
5265                                adapter->stats.latecol;
5266         net_stats->tx_aborted_errors = adapter->stats.ecol;
5267         net_stats->tx_window_errors = adapter->stats.latecol;
5268         net_stats->tx_carrier_errors = adapter->stats.tncrs;
5269
5270         /* Tx Dropped needs to be maintained elsewhere */
5271
5272         /* Phy Stats */
5273         if (hw->phy.media_type == e1000_media_type_copper) {
5274                 if ((adapter->link_speed == SPEED_1000) &&
5275                    (!e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
5276                         phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
5277                         adapter->phy_stats.idle_errors += phy_tmp;
5278                 }
5279         }
5280
5281         /* Management Stats */
5282         adapter->stats.mgptc += E1000_READ_REG(hw, E1000_MGTPTC);
5283         adapter->stats.mgprc += E1000_READ_REG(hw, E1000_MGTPRC);
5284         if (hw->mac.type > e1000_82580) {
5285                 adapter->stats.o2bgptc += E1000_READ_REG(hw, E1000_O2BGPTC);
5286                 adapter->stats.o2bspc += E1000_READ_REG(hw, E1000_O2BSPC);
5287                 adapter->stats.b2ospc += E1000_READ_REG(hw, E1000_B2OSPC);
5288                 adapter->stats.b2ogprc += E1000_READ_REG(hw, E1000_B2OGPRC);
5289         }
5290 }
5291
5292 static irqreturn_t igb_msix_other(int irq, void *data)
5293 {
5294         struct igb_adapter *adapter = data;
5295         struct e1000_hw *hw = &adapter->hw;
5296         u32 icr = E1000_READ_REG(hw, E1000_ICR);
5297         /* reading ICR causes bit 31 of EICR to be cleared */
5298
5299         if (icr & E1000_ICR_DRSTA)
5300                 schedule_work(&adapter->reset_task);
5301
5302         if (icr & E1000_ICR_DOUTSYNC) {
5303                 /* HW is reporting DMA is out of sync */
5304                 adapter->stats.doosync++;
5305                 /* The DMA Out of Sync is also indication of a spoof event
5306                  * in IOV mode. Check the Wrong VM Behavior register to
5307                  * see if it is really a spoof event. */
5308                 igb_check_wvbr(adapter);
5309         }
5310
5311         /* Check for a mailbox event */
5312         if (icr & E1000_ICR_VMMB)
5313                 igb_msg_task(adapter);
5314
5315         if (icr & E1000_ICR_LSC) {
5316                 hw->mac.get_link_status = 1;
5317                 /* guard against interrupt when we're going down */
5318                 if (!test_bit(__IGB_DOWN, &adapter->state))
5319                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
5320         }
5321
5322         /* Check for MDD event */
5323         if (icr & E1000_ICR_MDDET)
5324                 igb_process_mdd_event(adapter);
5325
5326         E1000_WRITE_REG(hw, E1000_EIMS, adapter->eims_other);
5327
5328         return IRQ_HANDLED;
5329 }
5330
5331 static void igb_write_itr(struct igb_q_vector *q_vector)
5332 {
5333         struct igb_adapter *adapter = q_vector->adapter;
5334         u32 itr_val = q_vector->itr_val & 0x7FFC;
5335
5336         if (!q_vector->set_itr)
5337                 return;
5338
5339         if (!itr_val)
5340                 itr_val = 0x4;
5341
5342         if (adapter->hw.mac.type == e1000_82575)
5343                 itr_val |= itr_val << 16;
5344         else
5345                 itr_val |= E1000_EITR_CNT_IGNR;
5346
5347         writel(itr_val, q_vector->itr_register);
5348         q_vector->set_itr = 0;
5349 }
5350
5351 static irqreturn_t igb_msix_ring(int irq, void *data)
5352 {
5353         struct igb_q_vector *q_vector = data;
5354
5355         /* Write the ITR value calculated from the previous interrupt. */
5356         igb_write_itr(q_vector);
5357
5358         napi_schedule(&q_vector->napi);
5359
5360         return IRQ_HANDLED;
5361 }
5362
5363 #ifdef IGB_DCA
5364 static void igb_update_dca(struct igb_q_vector *q_vector)
5365 {
5366         struct igb_adapter *adapter = q_vector->adapter;
5367         struct e1000_hw *hw = &adapter->hw;
5368         int cpu = get_cpu();
5369
5370         if (q_vector->cpu == cpu)
5371                 goto out_no_update;
5372
5373         if (q_vector->tx.ring) {
5374                 int q = q_vector->tx.ring->reg_idx;
5375                 u32 dca_txctrl = E1000_READ_REG(hw, E1000_DCA_TXCTRL(q));
5376                 if (hw->mac.type == e1000_82575) {
5377                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
5378                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
5379                 } else {
5380                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
5381                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
5382                                       E1000_DCA_TXCTRL_CPUID_SHIFT_82576;
5383                 }
5384                 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
5385                 E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(q), dca_txctrl);
5386         }
5387         if (q_vector->rx.ring) {
5388                 int q = q_vector->rx.ring->reg_idx;
5389                 u32 dca_rxctrl = E1000_READ_REG(hw, E1000_DCA_RXCTRL(q));
5390                 if (hw->mac.type == e1000_82575) {
5391                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
5392                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
5393                 } else {
5394                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
5395                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
5396                                       E1000_DCA_RXCTRL_CPUID_SHIFT_82576;
5397                 }
5398                 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
5399                 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
5400                 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
5401                 E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(q), dca_rxctrl);
5402         }
5403         q_vector->cpu = cpu;
5404 out_no_update:
5405         put_cpu();
5406 }
5407
5408 static void igb_setup_dca(struct igb_adapter *adapter)
5409 {
5410         struct e1000_hw *hw = &adapter->hw;
5411         int i;
5412
5413         if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
5414                 return;
5415
5416         /* Always use CB2 mode, difference is masked in the CB driver. */
5417         E1000_WRITE_REG(hw, E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
5418
5419         for (i = 0; i < adapter->num_q_vectors; i++) {
5420                 adapter->q_vector[i]->cpu = -1;
5421                 igb_update_dca(adapter->q_vector[i]);
5422         }
5423 }
5424
5425 static int __igb_notify_dca(struct device *dev, void *data)
5426 {
5427         struct net_device *netdev = dev_get_drvdata(dev);
5428         struct igb_adapter *adapter = netdev_priv(netdev);
5429         struct pci_dev *pdev = adapter->pdev;
5430         struct e1000_hw *hw = &adapter->hw;
5431         unsigned long event = *(unsigned long *)data;
5432
5433         switch (event) {
5434         case DCA_PROVIDER_ADD:
5435                 /* if already enabled, don't do it again */
5436                 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
5437                         break;
5438                 if (dca_add_requester(dev) == E1000_SUCCESS) {
5439                         adapter->flags |= IGB_FLAG_DCA_ENABLED;
5440                         dev_info(pci_dev_to_dev(pdev), "DCA enabled\n");
5441                         igb_setup_dca(adapter);
5442                         break;
5443                 }
5444                 /* Fall Through since DCA is disabled. */
5445         case DCA_PROVIDER_REMOVE:
5446                 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
5447                         /* without this a class_device is left
5448                          * hanging around in the sysfs model */
5449                         dca_remove_requester(dev);
5450                         dev_info(pci_dev_to_dev(pdev), "DCA disabled\n");
5451                         adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
5452                         E1000_WRITE_REG(hw, E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_DISABLE);
5453                 }
5454                 break;
5455         }
5456
5457         return E1000_SUCCESS;
5458 }
5459
5460 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
5461                           void *p)
5462 {
5463         int ret_val;
5464
5465         ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
5466                                          __igb_notify_dca);
5467
5468         return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
5469 }
5470 #endif /* IGB_DCA */
5471
5472 static int igb_vf_configure(struct igb_adapter *adapter, int vf)
5473 {
5474         unsigned char mac_addr[ETH_ALEN];
5475 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
5476         struct pci_dev *pdev = adapter->pdev;
5477         struct e1000_hw *hw = &adapter->hw;
5478         struct pci_dev *pvfdev;
5479         unsigned int device_id;
5480         u16 thisvf_devfn;
5481 #endif
5482
5483         random_ether_addr(mac_addr);
5484         igb_set_vf_mac(adapter, vf, mac_addr);
5485
5486 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
5487         switch (adapter->hw.mac.type) {
5488         case e1000_82576:
5489                 device_id = IGB_82576_VF_DEV_ID;
5490                 /* VF Stride for 82576 is 2 */
5491                 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 1)) |
5492                         (pdev->devfn & 1);
5493                 break;
5494         case e1000_i350:
5495                 device_id = IGB_I350_VF_DEV_ID;
5496                 /* VF Stride for I350 is 4 */
5497                 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 2)) |
5498                                 (pdev->devfn & 3);
5499                 break;
5500         default:
5501                 device_id = 0;
5502                 thisvf_devfn = 0;
5503                 break;
5504         }
5505
5506         pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5507         while (pvfdev) {
5508                 if (pvfdev->devfn == thisvf_devfn)
5509                         break;
5510                 pvfdev = pci_get_device(hw->vendor_id,
5511                                         device_id, pvfdev);
5512         }
5513
5514         if (pvfdev)
5515                 adapter->vf_data[vf].vfdev = pvfdev;
5516         else
5517                 dev_err(&pdev->dev,
5518                         "Couldn't find pci dev ptr for VF %4.4x\n",
5519                         thisvf_devfn);
5520         return pvfdev != NULL;
5521 #else
5522         return true;
5523 #endif
5524 }
5525
5526 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
5527 static int igb_find_enabled_vfs(struct igb_adapter *adapter)
5528 {
5529         struct e1000_hw *hw = &adapter->hw;
5530         struct pci_dev *pdev = adapter->pdev;
5531         struct pci_dev *pvfdev;
5532         u16 vf_devfn = 0;
5533         u16 vf_stride;
5534         unsigned int device_id;
5535         int vfs_found = 0;
5536
5537         switch (adapter->hw.mac.type) {
5538         case e1000_82576:
5539                 device_id = IGB_82576_VF_DEV_ID;
5540                 /* VF Stride for 82576 is 2 */
5541                 vf_stride = 2;
5542                 break;
5543         case e1000_i350:
5544                 device_id = IGB_I350_VF_DEV_ID;
5545                 /* VF Stride for I350 is 4 */
5546                 vf_stride = 4;
5547                 break;
5548         default:
5549                 device_id = 0;
5550                 vf_stride = 0;
5551                 break;
5552         }
5553
5554         vf_devfn = pdev->devfn + 0x80;
5555         pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5556         while (pvfdev) {
5557                 if (pvfdev->devfn == vf_devfn)
5558                         vfs_found++;
5559                 vf_devfn += vf_stride;
5560                 pvfdev = pci_get_device(hw->vendor_id,
5561                                         device_id, pvfdev);
5562         }
5563
5564         return vfs_found;
5565 }
5566 #endif
5567
5568 static int igb_check_vf_assignment(struct igb_adapter *adapter)
5569 {
5570 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
5571         int i;
5572         for (i = 0; i < adapter->vfs_allocated_count; i++) {
5573                 if (adapter->vf_data[i].vfdev) {
5574                         if (adapter->vf_data[i].vfdev->dev_flags &
5575                             PCI_DEV_FLAGS_ASSIGNED)
5576                                 return true;
5577                 }
5578         }
5579 #endif
5580         return false;
5581 }
5582
5583 static void igb_ping_all_vfs(struct igb_adapter *adapter)
5584 {
5585         struct e1000_hw *hw = &adapter->hw;
5586         u32 ping;
5587         int i;
5588
5589         for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
5590                 ping = E1000_PF_CONTROL_MSG;
5591                 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
5592                         ping |= E1000_VT_MSGTYPE_CTS;
5593                 e1000_write_mbx(hw, &ping, 1, i);
5594         }
5595 }
5596
5597 /**
5598  *  igb_mta_set_ - Set multicast filter table address
5599  *  @adapter: pointer to the adapter structure
5600  *  @hash_value: determines the MTA register and bit to set
5601  *
5602  *  The multicast table address is a register array of 32-bit registers.
5603  *  The hash_value is used to determine what register the bit is in, the
5604  *  current value is read, the new bit is OR'd in and the new value is
5605  *  written back into the register.
5606  **/
5607 void igb_mta_set(struct igb_adapter *adapter, u32 hash_value)
5608 {
5609         struct e1000_hw *hw = &adapter->hw;
5610         u32 hash_bit, hash_reg, mta;
5611
5612         /*
5613          * The MTA is a register array of 32-bit registers. It is
5614          * treated like an array of (32*mta_reg_count) bits.  We want to
5615          * set bit BitArray[hash_value]. So we figure out what register
5616          * the bit is in, read it, OR in the new bit, then write
5617          * back the new value.  The (hw->mac.mta_reg_count - 1) serves as a
5618          * mask to bits 31:5 of the hash value which gives us the
5619          * register we're modifying.  The hash bit within that register
5620          * is determined by the lower 5 bits of the hash value.
5621          */
5622         hash_reg = (hash_value >> 5) & (hw->mac.mta_reg_count - 1);
5623         hash_bit = hash_value & 0x1F;
5624
5625         mta = E1000_READ_REG_ARRAY(hw, E1000_MTA, hash_reg);
5626
5627         mta |= (1 << hash_bit);
5628
5629         E1000_WRITE_REG_ARRAY(hw, E1000_MTA, hash_reg, mta);
5630         E1000_WRITE_FLUSH(hw);
5631 }
5632
5633 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5634 {
5635
5636         struct e1000_hw *hw = &adapter->hw;
5637         u32 vmolr = E1000_READ_REG(hw, E1000_VMOLR(vf));
5638         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5639
5640         vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
5641                             IGB_VF_FLAG_MULTI_PROMISC);
5642         vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5643
5644 #ifdef IGB_ENABLE_VF_PROMISC
5645         if (*msgbuf & E1000_VF_SET_PROMISC_UNICAST) {
5646                 vmolr |= E1000_VMOLR_ROPE;
5647                 vf_data->flags |= IGB_VF_FLAG_UNI_PROMISC;
5648                 *msgbuf &= ~E1000_VF_SET_PROMISC_UNICAST;
5649         }
5650 #endif
5651         if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
5652                 vmolr |= E1000_VMOLR_MPME;
5653                 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
5654                 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
5655         } else {
5656                 /*
5657                  * if we have hashes and we are clearing a multicast promisc
5658                  * flag we need to write the hashes to the MTA as this step
5659                  * was previously skipped
5660                  */
5661                 if (vf_data->num_vf_mc_hashes > 30) {
5662                         vmolr |= E1000_VMOLR_MPME;
5663                 } else if (vf_data->num_vf_mc_hashes) {
5664                         int j;
5665                         vmolr |= E1000_VMOLR_ROMPE;
5666                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5667                                 igb_mta_set(adapter, vf_data->vf_mc_hashes[j]);
5668                 }
5669         }
5670
5671         E1000_WRITE_REG(hw, E1000_VMOLR(vf), vmolr);
5672
5673         /* there are flags left unprocessed, likely not supported */
5674         if (*msgbuf & E1000_VT_MSGINFO_MASK)
5675                 return -EINVAL;
5676
5677         return 0;
5678
5679 }
5680
5681 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
5682                                   u32 *msgbuf, u32 vf)
5683 {
5684         int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5685         u16 *hash_list = (u16 *)&msgbuf[1];
5686         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5687         int i;
5688
5689         /* salt away the number of multicast addresses assigned
5690          * to this VF for later use to restore when the PF multi cast
5691          * list changes
5692          */
5693         vf_data->num_vf_mc_hashes = n;
5694
5695         /* only up to 30 hash values supported */
5696         if (n > 30)
5697                 n = 30;
5698
5699         /* store the hashes for later use */
5700         for (i = 0; i < n; i++)
5701                 vf_data->vf_mc_hashes[i] = hash_list[i];
5702
5703         /* Flush and reset the mta with the new values */
5704         igb_set_rx_mode(adapter->netdev);
5705
5706         return 0;
5707 }
5708
5709 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
5710 {
5711         struct e1000_hw *hw = &adapter->hw;
5712         struct vf_data_storage *vf_data;
5713         int i, j;
5714
5715         for (i = 0; i < adapter->vfs_allocated_count; i++) {
5716                 u32 vmolr = E1000_READ_REG(hw, E1000_VMOLR(i));
5717                 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5718
5719                 vf_data = &adapter->vf_data[i];
5720
5721                 if ((vf_data->num_vf_mc_hashes > 30) ||
5722                     (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5723                         vmolr |= E1000_VMOLR_MPME;
5724                 } else if (vf_data->num_vf_mc_hashes) {
5725                         vmolr |= E1000_VMOLR_ROMPE;
5726                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5727                                 igb_mta_set(adapter, vf_data->vf_mc_hashes[j]);
5728                 }
5729                 E1000_WRITE_REG(hw, E1000_VMOLR(i), vmolr);
5730         }
5731 }
5732
5733 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5734 {
5735         struct e1000_hw *hw = &adapter->hw;
5736         u32 pool_mask, reg, vid;
5737         u16 vlan_default;
5738         int i;
5739
5740         pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5741
5742         /* Find the vlan filter for this id */
5743         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5744                 reg = E1000_READ_REG(hw, E1000_VLVF(i));
5745
5746                 /* remove the vf from the pool */
5747                 reg &= ~pool_mask;
5748
5749                 /* if pool is empty then remove entry from vfta */
5750                 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5751                     (reg & E1000_VLVF_VLANID_ENABLE)) {
5752                         reg = 0;
5753                         vid = reg & E1000_VLVF_VLANID_MASK;
5754                         igb_vfta_set(adapter, vid, FALSE);
5755                 }
5756
5757                 E1000_WRITE_REG(hw, E1000_VLVF(i), reg);
5758         }
5759
5760         adapter->vf_data[vf].vlans_enabled = 0;
5761
5762         vlan_default = adapter->vf_data[vf].default_vf_vlan_id;
5763         if (vlan_default)
5764                 igb_vlvf_set(adapter, vlan_default, true, vf);
5765 }
5766
5767 s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5768 {
5769         struct e1000_hw *hw = &adapter->hw;
5770         u32 reg, i;
5771
5772         /* The vlvf table only exists on 82576 hardware and newer */
5773         if (hw->mac.type < e1000_82576)
5774                 return -1;
5775
5776         /* we only need to do this if VMDq is enabled */
5777         if (!adapter->vmdq_pools)
5778                 return -1;
5779
5780         /* Find the vlan filter for this id */
5781         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5782                 reg = E1000_READ_REG(hw, E1000_VLVF(i));
5783                 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5784                     vid == (reg & E1000_VLVF_VLANID_MASK))
5785                         break;
5786         }
5787
5788         if (add) {
5789                 if (i == E1000_VLVF_ARRAY_SIZE) {
5790                         /* Did not find a matching VLAN ID entry that was
5791                          * enabled.  Search for a free filter entry, i.e.
5792                          * one without the enable bit set
5793                          */
5794                         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5795                                 reg = E1000_READ_REG(hw, E1000_VLVF(i));
5796                                 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5797                                         break;
5798                         }
5799                 }
5800                 if (i < E1000_VLVF_ARRAY_SIZE) {
5801                         /* Found an enabled/available entry */
5802                         reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5803
5804                         /* if !enabled we need to set this up in vfta */
5805                         if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
5806                                 /* add VID to filter table */
5807                                 igb_vfta_set(adapter, vid, TRUE);
5808                                 reg |= E1000_VLVF_VLANID_ENABLE;
5809                         }
5810                         reg &= ~E1000_VLVF_VLANID_MASK;
5811                         reg |= vid;
5812                         E1000_WRITE_REG(hw, E1000_VLVF(i), reg);
5813
5814                         /* do not modify RLPML for PF devices */
5815                         if (vf >= adapter->vfs_allocated_count)
5816                                 return E1000_SUCCESS;
5817
5818                         if (!adapter->vf_data[vf].vlans_enabled) {
5819                                 u32 size;
5820                                 reg = E1000_READ_REG(hw, E1000_VMOLR(vf));
5821                                 size = reg & E1000_VMOLR_RLPML_MASK;
5822                                 size += 4;
5823                                 reg &= ~E1000_VMOLR_RLPML_MASK;
5824                                 reg |= size;
5825                                 E1000_WRITE_REG(hw, E1000_VMOLR(vf), reg);
5826                         }
5827
5828                         adapter->vf_data[vf].vlans_enabled++;
5829                 }
5830         } else {
5831                 if (i < E1000_VLVF_ARRAY_SIZE) {
5832                         /* remove vf from the pool */
5833                         reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5834                         /* if pool is empty then remove entry from vfta */
5835                         if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5836                                 reg = 0;
5837                                 igb_vfta_set(adapter, vid, FALSE);
5838                         }
5839                         E1000_WRITE_REG(hw, E1000_VLVF(i), reg);
5840
5841                         /* do not modify RLPML for PF devices */
5842                         if (vf >= adapter->vfs_allocated_count)
5843                                 return E1000_SUCCESS;
5844
5845                         adapter->vf_data[vf].vlans_enabled--;
5846                         if (!adapter->vf_data[vf].vlans_enabled) {
5847                                 u32 size;
5848                                 reg = E1000_READ_REG(hw, E1000_VMOLR(vf));
5849                                 size = reg & E1000_VMOLR_RLPML_MASK;
5850                                 size -= 4;
5851                                 reg &= ~E1000_VMOLR_RLPML_MASK;
5852                                 reg |= size;
5853                                 E1000_WRITE_REG(hw, E1000_VMOLR(vf), reg);
5854                         }
5855                 }
5856         }
5857         return E1000_SUCCESS;
5858 }
5859
5860 #ifdef IFLA_VF_MAX
5861 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5862 {
5863         struct e1000_hw *hw = &adapter->hw;
5864
5865         if (vid)
5866                 E1000_WRITE_REG(hw, E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5867         else
5868                 E1000_WRITE_REG(hw, E1000_VMVIR(vf), 0);
5869 }
5870
5871 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5872                                int vf, u16 vlan, u8 qos)
5873 {
5874         int err = 0;
5875         struct igb_adapter *adapter = netdev_priv(netdev);
5876
5877         /* VLAN IDs accepted range 0-4094 */
5878         if ((vf >= adapter->vfs_allocated_count) || (vlan > VLAN_VID_MASK-1) || (qos > 7))
5879                 return -EINVAL;
5880         if (vlan || qos) {
5881                 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5882                 if (err)
5883                         goto out;
5884                 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5885                 igb_set_vmolr(adapter, vf, !vlan);
5886                 adapter->vf_data[vf].pf_vlan = vlan;
5887                 adapter->vf_data[vf].pf_qos = qos;
5888                 igb_set_vf_vlan_strip(adapter, vf, true);
5889                 dev_info(&adapter->pdev->dev,
5890                          "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5891                 if (test_bit(__IGB_DOWN, &adapter->state)) {
5892                         dev_warn(&adapter->pdev->dev,
5893                                  "The VF VLAN has been set,"
5894                                  " but the PF device is not up.\n");
5895                         dev_warn(&adapter->pdev->dev,
5896                                  "Bring the PF device up before"
5897                                  " attempting to use the VF device.\n");
5898                 }
5899         } else {
5900                 if (adapter->vf_data[vf].pf_vlan)
5901                         dev_info(&adapter->pdev->dev,
5902                                  "Clearing VLAN on VF %d\n", vf);
5903                 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5904                                    false, vf);
5905                 igb_set_vmvir(adapter, vlan, vf);
5906                 igb_set_vmolr(adapter, vf, true);
5907                 igb_set_vf_vlan_strip(adapter, vf, false);
5908                 adapter->vf_data[vf].pf_vlan = 0;
5909                 adapter->vf_data[vf].pf_qos = 0;
5910        }
5911 out:
5912        return err;
5913 }
5914 #endif
5915
5916 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5917 {
5918         int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5919         int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5920
5921         if (vid)
5922                 igb_set_vf_vlan_strip(adapter, vf, true);
5923         else
5924                 igb_set_vf_vlan_strip(adapter, vf, false);
5925
5926         return igb_vlvf_set(adapter, vid, add, vf);
5927 }
5928
5929 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
5930 {
5931         struct e1000_hw *hw = &adapter->hw;
5932
5933         /* clear flags except flag that the PF has set the MAC */
5934         adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
5935         adapter->vf_data[vf].last_nack = jiffies;
5936
5937         /* reset offloads to defaults */
5938         igb_set_vmolr(adapter, vf, true);
5939
5940         /* reset vlans for device */
5941         igb_clear_vf_vfta(adapter, vf);
5942 #ifdef IFLA_VF_MAX
5943         if (adapter->vf_data[vf].pf_vlan)
5944                 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5945                                     adapter->vf_data[vf].pf_vlan,
5946                                     adapter->vf_data[vf].pf_qos);
5947         else
5948                 igb_clear_vf_vfta(adapter, vf);
5949 #endif
5950
5951         /* reset multicast table array for vf */
5952         adapter->vf_data[vf].num_vf_mc_hashes = 0;
5953
5954         /* Flush and reset the mta with the new values */
5955         igb_set_rx_mode(adapter->netdev);
5956
5957         /*
5958          * Reset the VFs TDWBAL and TDWBAH registers which are not
5959          * cleared by a VFLR
5960          */
5961         E1000_WRITE_REG(hw, E1000_TDWBAH(vf), 0);
5962         E1000_WRITE_REG(hw, E1000_TDWBAL(vf), 0);
5963         if (hw->mac.type == e1000_82576) {
5964                 E1000_WRITE_REG(hw, E1000_TDWBAH(IGB_MAX_VF_FUNCTIONS + vf), 0);
5965                 E1000_WRITE_REG(hw, E1000_TDWBAL(IGB_MAX_VF_FUNCTIONS + vf), 0);
5966         }
5967 }
5968
5969 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5970 {
5971         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5972
5973         /* generate a new mac address as we were hotplug removed/added */
5974         if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5975                 random_ether_addr(vf_mac);
5976
5977         /* process remaining reset events */
5978         igb_vf_reset(adapter, vf);
5979 }
5980
5981 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
5982 {
5983         struct e1000_hw *hw = &adapter->hw;
5984         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5985         u32 reg, msgbuf[3];
5986         u8 *addr = (u8 *)(&msgbuf[1]);
5987
5988         /* process all the same items cleared in a function level reset */
5989         igb_vf_reset(adapter, vf);
5990
5991         /* set vf mac address */
5992         igb_del_mac_filter(adapter, vf_mac, vf);
5993         igb_add_mac_filter(adapter, vf_mac, vf);
5994
5995         /* enable transmit and receive for vf */
5996         reg = E1000_READ_REG(hw, E1000_VFTE);
5997         E1000_WRITE_REG(hw, E1000_VFTE, reg | (1 << vf));
5998         reg = E1000_READ_REG(hw, E1000_VFRE);
5999         E1000_WRITE_REG(hw, E1000_VFRE, reg | (1 << vf));
6000
6001         adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
6002
6003         /* reply to reset with ack and vf mac address */
6004         msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
6005         memcpy(addr, vf_mac, 6);
6006         e1000_write_mbx(hw, msgbuf, 3, vf);
6007 }
6008
6009 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
6010 {
6011         /*
6012          * The VF MAC Address is stored in a packed array of bytes
6013          * starting at the second 32 bit word of the msg array
6014          */
6015         unsigned char *addr = (unsigned char *)&msg[1];
6016         int err = -1;
6017
6018         if (is_valid_ether_addr(addr))
6019                 err = igb_set_vf_mac(adapter, vf, addr);
6020
6021         return err;
6022 }
6023
6024 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
6025 {
6026         struct e1000_hw *hw = &adapter->hw;
6027         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6028         u32 msg = E1000_VT_MSGTYPE_NACK;
6029
6030         /* if device isn't clear to send it shouldn't be reading either */
6031         if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
6032             time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
6033                 e1000_write_mbx(hw, &msg, 1, vf);
6034                 vf_data->last_nack = jiffies;
6035         }
6036 }
6037
6038 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
6039 {
6040         struct pci_dev *pdev = adapter->pdev;
6041         u32 msgbuf[E1000_VFMAILBOX_SIZE];
6042         struct e1000_hw *hw = &adapter->hw;
6043         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6044         s32 retval;
6045
6046         retval = e1000_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
6047
6048         if (retval) {
6049                 dev_err(pci_dev_to_dev(pdev), "Error receiving message from VF\n");
6050                 return;
6051         }
6052
6053         /* this is a message we already processed, do nothing */
6054         if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
6055                 return;
6056
6057         /*
6058          * until the vf completes a reset it should not be
6059          * allowed to start any configuration.
6060          */
6061
6062         if (msgbuf[0] == E1000_VF_RESET) {
6063                 igb_vf_reset_msg(adapter, vf);
6064                 return;
6065         }
6066
6067         if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
6068                 msgbuf[0] = E1000_VT_MSGTYPE_NACK;
6069                 if (time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
6070                         e1000_write_mbx(hw, msgbuf, 1, vf);
6071                         vf_data->last_nack = jiffies;
6072                 }
6073                 return;
6074         }
6075
6076         switch ((msgbuf[0] & 0xFFFF)) {
6077         case E1000_VF_SET_MAC_ADDR:
6078                 retval = -EINVAL;
6079 #ifndef IGB_DISABLE_VF_MAC_SET
6080                 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
6081                         retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
6082                 else
6083                         DPRINTK(DRV, INFO,
6084                                 "VF %d attempted to override administratively "
6085                                 "set MAC address\nReload the VF driver to "
6086                                 "resume operations\n", vf);
6087 #endif
6088                 break;
6089         case E1000_VF_SET_PROMISC:
6090                 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
6091                 break;
6092         case E1000_VF_SET_MULTICAST:
6093                 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
6094                 break;
6095         case E1000_VF_SET_LPE:
6096                 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
6097                 break;
6098         case E1000_VF_SET_VLAN:
6099                 retval = -1;
6100 #ifdef IFLA_VF_MAX
6101                 if (vf_data->pf_vlan)
6102                         DPRINTK(DRV, INFO,
6103                                 "VF %d attempted to override administratively "
6104                                 "set VLAN tag\nReload the VF driver to "
6105                                 "resume operations\n", vf);
6106                 else
6107 #endif
6108                         retval = igb_set_vf_vlan(adapter, msgbuf, vf);
6109                 break;
6110         default:
6111                 dev_err(pci_dev_to_dev(pdev), "Unhandled Msg %08x\n", msgbuf[0]);
6112                 retval = -E1000_ERR_MBX;
6113                 break;
6114         }
6115
6116         /* notify the VF of the results of what it sent us */
6117         if (retval)
6118                 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
6119         else
6120                 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
6121
6122         msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
6123
6124         e1000_write_mbx(hw, msgbuf, 1, vf);
6125 }
6126
6127 static void igb_msg_task(struct igb_adapter *adapter)
6128 {
6129         struct e1000_hw *hw = &adapter->hw;
6130         u32 vf;
6131
6132         for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
6133                 /* process any reset requests */
6134                 if (!e1000_check_for_rst(hw, vf))
6135                         igb_vf_reset_event(adapter, vf);
6136
6137                 /* process any messages pending */
6138                 if (!e1000_check_for_msg(hw, vf))
6139                         igb_rcv_msg_from_vf(adapter, vf);
6140
6141                 /* process any acks */
6142                 if (!e1000_check_for_ack(hw, vf))
6143                         igb_rcv_ack_from_vf(adapter, vf);
6144         }
6145 }
6146
6147 /**
6148  *  igb_set_uta - Set unicast filter table address
6149  *  @adapter: board private structure
6150  *
6151  *  The unicast table address is a register array of 32-bit registers.
6152  *  The table is meant to be used in a way similar to how the MTA is used
6153  *  however due to certain limitations in the hardware it is necessary to
6154  *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
6155  *  enable bit to allow vlan tag stripping when promiscuous mode is enabled
6156  **/
6157 static void igb_set_uta(struct igb_adapter *adapter)
6158 {
6159         struct e1000_hw *hw = &adapter->hw;
6160         int i;
6161
6162         /* The UTA table only exists on 82576 hardware and newer */
6163         if (hw->mac.type < e1000_82576)
6164                 return;
6165
6166         /* we only need to do this if VMDq is enabled */
6167         if (!adapter->vmdq_pools)
6168                 return;
6169
6170         for (i = 0; i < hw->mac.uta_reg_count; i++)
6171                 E1000_WRITE_REG_ARRAY(hw, E1000_UTA, i, ~0);
6172 }
6173
6174 /**
6175  * igb_intr_msi - Interrupt Handler
6176  * @irq: interrupt number
6177  * @data: pointer to a network interface device structure
6178  **/
6179 static irqreturn_t igb_intr_msi(int irq, void *data)
6180 {
6181         struct igb_adapter *adapter = data;
6182         struct igb_q_vector *q_vector = adapter->q_vector[0];
6183         struct e1000_hw *hw = &adapter->hw;
6184         /* read ICR disables interrupts using IAM */
6185         u32 icr = E1000_READ_REG(hw, E1000_ICR);
6186
6187         igb_write_itr(q_vector);
6188
6189         if (icr & E1000_ICR_DRSTA)
6190                 schedule_work(&adapter->reset_task);
6191
6192         if (icr & E1000_ICR_DOUTSYNC) {
6193                 /* HW is reporting DMA is out of sync */
6194                 adapter->stats.doosync++;
6195         }
6196
6197         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
6198                 hw->mac.get_link_status = 1;
6199                 if (!test_bit(__IGB_DOWN, &adapter->state))
6200                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
6201         }
6202
6203         napi_schedule(&q_vector->napi);
6204
6205         return IRQ_HANDLED;
6206 }
6207
6208 /**
6209  * igb_intr - Legacy Interrupt Handler
6210  * @irq: interrupt number
6211  * @data: pointer to a network interface device structure
6212  **/
6213 static irqreturn_t igb_intr(int irq, void *data)
6214 {
6215         struct igb_adapter *adapter = data;
6216         struct igb_q_vector *q_vector = adapter->q_vector[0];
6217         struct e1000_hw *hw = &adapter->hw;
6218         /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
6219          * need for the IMC write */
6220         u32 icr = E1000_READ_REG(hw, E1000_ICR);
6221
6222         /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
6223          * not set, then the adapter didn't send an interrupt */
6224         if (!(icr & E1000_ICR_INT_ASSERTED))
6225                 return IRQ_NONE;
6226
6227         igb_write_itr(q_vector);
6228
6229         if (icr & E1000_ICR_DRSTA)
6230                 schedule_work(&adapter->reset_task);
6231
6232         if (icr & E1000_ICR_DOUTSYNC) {
6233                 /* HW is reporting DMA is out of sync */
6234                 adapter->stats.doosync++;
6235         }
6236
6237         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
6238                 hw->mac.get_link_status = 1;
6239                 /* guard against interrupt when we're going down */
6240                 if (!test_bit(__IGB_DOWN, &adapter->state))
6241                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
6242         }
6243
6244         napi_schedule(&q_vector->napi);
6245
6246         return IRQ_HANDLED;
6247 }
6248
6249 void igb_ring_irq_enable(struct igb_q_vector *q_vector)
6250 {
6251         struct igb_adapter *adapter = q_vector->adapter;
6252         struct e1000_hw *hw = &adapter->hw;
6253
6254         if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
6255             (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
6256                 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
6257                         igb_set_itr(q_vector);
6258                 else
6259                         igb_update_ring_itr(q_vector);
6260         }
6261
6262         if (!test_bit(__IGB_DOWN, &adapter->state)) {
6263                 if (adapter->msix_entries)
6264                         E1000_WRITE_REG(hw, E1000_EIMS, q_vector->eims_value);
6265                 else
6266                         igb_irq_enable(adapter);
6267         }
6268 }
6269
6270 /**
6271  * igb_poll - NAPI Rx polling callback
6272  * @napi: napi polling structure
6273  * @budget: count of how many packets we should handle
6274  **/
6275 static int igb_poll(struct napi_struct *napi, int budget)
6276 {
6277         struct igb_q_vector *q_vector = container_of(napi, struct igb_q_vector, napi);
6278         bool clean_complete = true;
6279
6280 #ifdef IGB_DCA
6281         if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
6282                 igb_update_dca(q_vector);
6283 #endif
6284         if (q_vector->tx.ring)
6285                 clean_complete = igb_clean_tx_irq(q_vector);
6286
6287         if (q_vector->rx.ring)
6288                 clean_complete &= igb_clean_rx_irq(q_vector, budget);
6289
6290 #ifndef HAVE_NETDEV_NAPI_LIST
6291         /* if netdev is disabled we need to stop polling */
6292         if (!netif_running(q_vector->adapter->netdev))
6293                 clean_complete = true;
6294
6295 #endif
6296         /* If all work not completed, return budget and keep polling */
6297         if (!clean_complete)
6298                 return budget;
6299
6300         /* If not enough Rx work done, exit the polling mode */
6301         napi_complete(napi);
6302         igb_ring_irq_enable(q_vector);
6303
6304         return 0;
6305 }
6306
6307 #ifdef HAVE_HW_TIME_STAMP
6308 /**
6309  * igb_systim_to_hwtstamp - convert system time value to hw timestamp
6310  * @adapter: board private structure
6311  * @shhwtstamps: timestamp structure to update
6312  * @regval: unsigned 64bit system time value.
6313  *
6314  * We need to convert the system time value stored in the RX/TXSTMP registers
6315  * into a hwtstamp which can be used by the upper level timestamping functions
6316  */
6317 static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
6318                                    struct skb_shared_hwtstamps *shhwtstamps,
6319                                    u64 regval)
6320 {
6321         u64 ns;
6322
6323         /*
6324          * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
6325          * 24 to match clock shift we setup earlier.
6326          */
6327         if (adapter->hw.mac.type >= e1000_82580)
6328                 regval <<= IGB_82580_TSYNC_SHIFT;
6329
6330         ns = timecounter_cyc2time(&adapter->clock, regval);
6331
6332         /*
6333          * force a timecompare_update here (even if less than a second
6334          * has passed) in order to prevent the case when ptpd or other
6335          * software jumps the clock offset. othwerise there is a small
6336          * window when the timestamp would be based on previous skew
6337          * and invalid results would be pushed to the network stack.
6338          */
6339         timecompare_update(&adapter->compare, 0);
6340         memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
6341         shhwtstamps->hwtstamp = ns_to_ktime(ns);
6342         shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
6343 }
6344
6345 /**
6346  * igb_tx_hwtstamp - utility function which checks for TX time stamp
6347  * @q_vector: pointer to q_vector containing needed info
6348  * @buffer: pointer to igb_tx_buffer structure
6349  *
6350  * If we were asked to do hardware stamping and such a time stamp is
6351  * available, then it must have been for this skb here because we only
6352  * allow only one such packet into the queue.
6353  */
6354 static void igb_tx_hwtstamp(struct igb_q_vector *q_vector,
6355                             struct igb_tx_buffer *buffer_info)
6356 {
6357         struct igb_adapter *adapter = q_vector->adapter;
6358         struct e1000_hw *hw = &adapter->hw;
6359         struct skb_shared_hwtstamps shhwtstamps;
6360         u64 regval;
6361
6362         /* if skb does not support hw timestamp or TX stamp not valid exit */
6363         if (likely(!(buffer_info->tx_flags & IGB_TX_FLAGS_TSTAMP)) ||
6364             !(E1000_READ_REG(hw, E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
6365                 return;
6366
6367         regval = E1000_READ_REG(hw, E1000_TXSTMPL);
6368         regval |= (u64)E1000_READ_REG(hw, E1000_TXSTMPH) << 32;
6369
6370         igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
6371         skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
6372 }
6373
6374 #endif
6375 /**
6376  * igb_clean_tx_irq - Reclaim resources after transmit completes
6377  * @q_vector: pointer to q_vector containing needed info
6378  * returns TRUE if ring is completely cleaned
6379  **/
6380 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
6381 {
6382         struct igb_adapter *adapter = q_vector->adapter;
6383         struct igb_ring *tx_ring = q_vector->tx.ring;
6384         struct igb_tx_buffer *tx_buffer;
6385         union e1000_adv_tx_desc *tx_desc, *eop_desc;
6386         unsigned int total_bytes = 0, total_packets = 0;
6387         unsigned int budget = q_vector->tx.work_limit;
6388         unsigned int i = tx_ring->next_to_clean;
6389
6390         if (test_bit(__IGB_DOWN, &adapter->state))
6391                 return true;
6392
6393         tx_buffer = &tx_ring->tx_buffer_info[i];
6394         tx_desc = IGB_TX_DESC(tx_ring, i);
6395         i -= tx_ring->count;
6396
6397         for (; budget; budget--) {
6398                 eop_desc = tx_buffer->next_to_watch;
6399
6400                 /* prevent any other reads prior to eop_desc */
6401                 rmb();
6402
6403                 /* if next_to_watch is not set then there is no work pending */
6404                 if (!eop_desc)
6405                         break;
6406
6407                 /* if DD is not set pending work has not been completed */
6408                 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
6409                         break;
6410
6411                 /* clear next_to_watch to prevent false hangs */
6412                 tx_buffer->next_to_watch = NULL;
6413
6414                 /* update the statistics for this packet */
6415                 total_bytes += tx_buffer->bytecount;
6416                 total_packets += tx_buffer->gso_segs;
6417
6418 #ifdef HAVE_HW_TIME_STAMP
6419                 /* retrieve hardware timestamp */
6420                 igb_tx_hwtstamp(q_vector, tx_buffer);
6421
6422 #endif
6423                 /* free the skb */
6424                 dev_kfree_skb_any(tx_buffer->skb);
6425
6426                 /* unmap skb header data */
6427                 dma_unmap_single(tx_ring->dev,
6428                                  dma_unmap_addr(tx_buffer, dma),
6429                                  dma_unmap_len(tx_buffer, len),
6430                                  DMA_TO_DEVICE);
6431
6432                 /* clear tx_buffer data */
6433                 tx_buffer->skb = NULL;
6434                 dma_unmap_len_set(tx_buffer, len, 0);
6435
6436                 /* clear last DMA location and unmap remaining buffers */
6437                 while (tx_desc != eop_desc) {
6438                         tx_buffer++;
6439                         tx_desc++;
6440                         i++;
6441                         if (unlikely(!i)) {
6442                                 i -= tx_ring->count;
6443                                 tx_buffer = tx_ring->tx_buffer_info;
6444                                 tx_desc = IGB_TX_DESC(tx_ring, 0);
6445                         }
6446
6447                         /* unmap any remaining paged data */
6448                         if (dma_unmap_len(tx_buffer, len)) {
6449                                 dma_unmap_page(tx_ring->dev,
6450                                                dma_unmap_addr(tx_buffer, dma),
6451                                                dma_unmap_len(tx_buffer, len),
6452                                                DMA_TO_DEVICE);
6453                                 dma_unmap_len_set(tx_buffer, len, 0);
6454                         }
6455                 }
6456
6457                 /* move us one more past the eop_desc for start of next pkt */
6458                 tx_buffer++;
6459                 tx_desc++;
6460                 i++;
6461                 if (unlikely(!i)) {
6462                         i -= tx_ring->count;
6463                         tx_buffer = tx_ring->tx_buffer_info;
6464                         tx_desc = IGB_TX_DESC(tx_ring, 0);
6465                 }
6466         }
6467
6468 #ifdef CONFIG_BQL
6469         netdev_tx_completed_queue(txring_txq(tx_ring),
6470                                   total_packets, total_bytes);
6471 #endif /* CONFIG_BQL */
6472
6473         i += tx_ring->count;
6474         tx_ring->next_to_clean = i;
6475         tx_ring->tx_stats.bytes += total_bytes;
6476         tx_ring->tx_stats.packets += total_packets;
6477         q_vector->tx.total_bytes += total_bytes;
6478         q_vector->tx.total_packets += total_packets;
6479
6480         if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
6481                 struct e1000_hw *hw = &adapter->hw;
6482
6483                 eop_desc = tx_buffer->next_to_watch;
6484
6485                 /* Detect a transmit hang in hardware, this serializes the
6486                  * check with the clearing of time_stamp and movement of i */
6487                 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
6488                 if (eop_desc &&
6489                     time_after(jiffies, tx_buffer->time_stamp +
6490                                (adapter->tx_timeout_factor * HZ))
6491                     && !(E1000_READ_REG(hw, E1000_STATUS) &
6492                          E1000_STATUS_TXOFF)) {
6493
6494                         /* detected Tx unit hang */
6495                         dev_err(tx_ring->dev,
6496                                 "Detected Tx Unit Hang\n"
6497                                 "  Tx Queue             <%d>\n"
6498                                 "  TDH                  <%x>\n"
6499                                 "  TDT                  <%x>\n"
6500                                 "  next_to_use          <%x>\n"
6501                                 "  next_to_clean        <%x>\n"
6502                                 "buffer_info[next_to_clean]\n"
6503                                 "  time_stamp           <%lx>\n"
6504                                 "  next_to_watch        <%p>\n"
6505                                 "  jiffies              <%lx>\n"
6506                                 "  desc.status          <%x>\n",
6507                                 tx_ring->queue_index,
6508                                 E1000_READ_REG(hw, E1000_TDH(tx_ring->reg_idx)),
6509                                 readl(tx_ring->tail),
6510                                 tx_ring->next_to_use,
6511                                 tx_ring->next_to_clean,
6512                                 tx_buffer->time_stamp,
6513                                 eop_desc,
6514                                 jiffies,
6515                                 eop_desc->wb.status);
6516                         if (netif_is_multiqueue(netdev_ring(tx_ring)))
6517                                 netif_stop_subqueue(netdev_ring(tx_ring),
6518                                                     ring_queue_index(tx_ring));
6519                         else
6520                                 netif_stop_queue(netdev_ring(tx_ring));
6521
6522                         /* we are about to reset, no point in enabling stuff */
6523                         return true;
6524                 }
6525         }
6526
6527         if (unlikely(total_packets &&
6528                      netif_carrier_ok(netdev_ring(tx_ring)) &&
6529                      igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
6530                 /* Make sure that anybody stopping the queue after this
6531                  * sees the new next_to_clean.
6532                  */
6533                 smp_mb();
6534                 if (netif_is_multiqueue(netdev_ring(tx_ring))) {
6535                         if (__netif_subqueue_stopped(netdev_ring(tx_ring),
6536                                                      ring_queue_index(tx_ring)) &&
6537                             !(test_bit(__IGB_DOWN, &adapter->state))) {
6538                                 netif_wake_subqueue(netdev_ring(tx_ring),
6539                                                     ring_queue_index(tx_ring));
6540                                 tx_ring->tx_stats.restart_queue++;
6541                         }
6542                 } else {
6543                         if (netif_queue_stopped(netdev_ring(tx_ring)) &&
6544                             !(test_bit(__IGB_DOWN, &adapter->state))) {
6545                                 netif_wake_queue(netdev_ring(tx_ring));
6546                                 tx_ring->tx_stats.restart_queue++;
6547                         }
6548                 }
6549         }
6550
6551         return !!budget;
6552 }
6553
6554 #ifdef HAVE_VLAN_RX_REGISTER
6555 /**
6556  * igb_receive_skb - helper function to handle rx indications
6557  * @q_vector: structure containing interrupt and ring information
6558  * @skb: packet to send up
6559  **/
6560 static void igb_receive_skb(struct igb_q_vector *q_vector,
6561                             struct sk_buff *skb)
6562 {
6563         struct vlan_group **vlgrp = netdev_priv(skb->dev);
6564
6565         if (IGB_CB(skb)->vid) {
6566                 if (*vlgrp) {
6567                         vlan_gro_receive(&q_vector->napi, *vlgrp,
6568                                          IGB_CB(skb)->vid, skb);
6569                 } else {
6570                         dev_kfree_skb_any(skb);
6571                 }
6572         } else {
6573                 napi_gro_receive(&q_vector->napi, skb);
6574         }
6575 }
6576
6577 #endif /* HAVE_VLAN_RX_REGISTER */
6578 static inline void igb_rx_checksum(struct igb_ring *ring,
6579                                    union e1000_adv_rx_desc *rx_desc,
6580                                    struct sk_buff *skb)
6581 {
6582         skb_checksum_none_assert(skb);
6583
6584         /* Ignore Checksum bit is set */
6585         if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
6586                 return;
6587
6588         /* Rx checksum disabled via ethtool */
6589 #ifdef HAVE_NDO_SET_FEATURES
6590         if (!(netdev_ring(ring)->features & NETIF_F_RXCSUM))
6591 #else
6592         if (!test_bit(IGB_RING_FLAG_RX_CSUM, &ring->flags))
6593 #endif
6594                 return;
6595
6596         /* TCP/UDP checksum error bit is set */
6597         if (igb_test_staterr(rx_desc,
6598                              E1000_RXDEXT_STATERR_TCPE |
6599                              E1000_RXDEXT_STATERR_IPE)) {
6600                 /*
6601                  * work around errata with sctp packets where the TCPE aka
6602                  * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
6603                  * packets, (aka let the stack check the crc32c)
6604                  */
6605                 if (!((skb->len == 60) &&
6606                       test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags)))
6607                         ring->rx_stats.csum_err++;
6608
6609                 /* let the stack verify checksum errors */
6610                 return;
6611         }
6612         /* It must be a TCP or UDP packet with a valid checksum */
6613         if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
6614                                       E1000_RXD_STAT_UDPCS))
6615                 skb->ip_summed = CHECKSUM_UNNECESSARY;
6616 }
6617
6618 #ifdef NETIF_F_RXHASH
6619 static inline void igb_rx_hash(struct igb_ring *ring,
6620                                union e1000_adv_rx_desc *rx_desc,
6621                                struct sk_buff *skb)
6622 {
6623         if (netdev_ring(ring)->features & NETIF_F_RXHASH)
6624                 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
6625 }
6626
6627 #endif
6628 #ifdef HAVE_HW_TIME_STAMP
6629 static void igb_rx_hwtstamp(struct igb_q_vector *q_vector,
6630                             union e1000_adv_rx_desc *rx_desc,
6631                             struct sk_buff *skb)
6632 {
6633         struct igb_adapter *adapter = q_vector->adapter;
6634         struct e1000_hw *hw = &adapter->hw;
6635         u64 regval;
6636
6637         if (!igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP |
6638                                        E1000_RXDADV_STAT_TS))
6639                 return;
6640
6641         /*
6642          * If this bit is set, then the RX registers contain the time stamp. No
6643          * other packet will be time stamped until we read these registers, so
6644          * read the registers to make them available again. Because only one
6645          * packet can be time stamped at a time, we know that the register
6646          * values must belong to this one here and therefore we don't need to
6647          * compare any of the additional attributes stored for it.
6648          *
6649          * If nothing went wrong, then it should have a skb_shared_tx that we
6650          * can turn into a skb_shared_hwtstamps.
6651          */
6652         if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
6653                 u32 *stamp = (u32 *)skb->data;
6654                 regval = le32_to_cpu(*(stamp + 2));
6655                 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
6656                 skb_pull(skb, IGB_TS_HDR_LEN);
6657         } else {
6658                 if(!(E1000_READ_REG(hw, E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
6659                         return;
6660
6661                 regval = E1000_READ_REG(hw, E1000_RXSTMPL);
6662                 regval |= (u64)E1000_READ_REG(hw, E1000_RXSTMPH) << 32;
6663         }
6664
6665         igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
6666 }
6667 #endif
6668 static void igb_rx_vlan(struct igb_ring *ring,
6669                         union e1000_adv_rx_desc *rx_desc,
6670                         struct sk_buff *skb)
6671 {
6672         if (igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
6673                 u16 vid = 0;
6674                 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
6675                     test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags))
6676                         vid = be16_to_cpu(rx_desc->wb.upper.vlan);
6677                 else
6678                         vid = le16_to_cpu(rx_desc->wb.upper.vlan);
6679 #ifdef HAVE_VLAN_RX_REGISTER
6680                 IGB_CB(skb)->vid = vid;
6681         } else {
6682                 IGB_CB(skb)->vid = 0;
6683 #else
6684
6685 #ifdef HAVE_VLAN_PROTOCOL
6686                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
6687 #else
6688                 __vlan_hwaccel_put_tag(skb, vid);
6689 #endif
6690
6691
6692 #endif
6693         }
6694 }
6695
6696 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
6697 static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
6698 {
6699         /* HW will not DMA in data larger than the given buffer, even if it
6700          * parses the (NFS, of course) header to be larger.  In that case, it
6701          * fills the header buffer and spills the rest into the page.
6702          */
6703         u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info) &
6704                    E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
6705         if (hlen > IGB_RX_HDR_LEN)
6706                 hlen = IGB_RX_HDR_LEN;
6707         return hlen;
6708 }
6709
6710 #endif
6711 #ifndef IGB_NO_LRO
6712 /**
6713  * igb_merge_active_tail - merge active tail into lro skb
6714  * @tail: pointer to active tail in frag_list
6715  *
6716  * This function merges the length and data of an active tail into the
6717  * skb containing the frag_list.  It resets the tail's pointer to the head,
6718  * but it leaves the heads pointer to tail intact.
6719  **/
6720 static inline struct sk_buff *igb_merge_active_tail(struct sk_buff *tail)
6721 {
6722         struct sk_buff *head = IGB_CB(tail)->head;
6723
6724         if (!head)
6725                 return tail;
6726
6727         head->len += tail->len;
6728         head->data_len += tail->len;
6729         head->truesize += tail->len;
6730
6731         IGB_CB(tail)->head = NULL;
6732
6733         return head;
6734 }
6735
6736 /**
6737  * igb_add_active_tail - adds an active tail into the skb frag_list
6738  * @head: pointer to the start of the skb
6739  * @tail: pointer to active tail to add to frag_list
6740  *
6741  * This function adds an active tail to the end of the frag list.  This tail
6742  * will still be receiving data so we cannot yet ad it's stats to the main
6743  * skb.  That is done via igb_merge_active_tail.
6744  **/
6745 static inline void igb_add_active_tail(struct sk_buff *head, struct sk_buff *tail)
6746 {
6747         struct sk_buff *old_tail = IGB_CB(head)->tail;
6748
6749         if (old_tail) {
6750                 igb_merge_active_tail(old_tail);
6751                 old_tail->next = tail;
6752         } else {
6753                 skb_shinfo(head)->frag_list = tail;
6754         }
6755
6756         IGB_CB(tail)->head = head;
6757         IGB_CB(head)->tail = tail;
6758
6759         IGB_CB(head)->append_cnt++;
6760 }
6761
6762 /**
6763  * igb_close_active_frag_list - cleanup pointers on a frag_list skb
6764  * @head: pointer to head of an active frag list
6765  *
6766  * This function will clear the frag_tail_tracker pointer on an active
6767  * frag_list and returns true if the pointer was actually set
6768  **/
6769 static inline bool igb_close_active_frag_list(struct sk_buff *head)
6770 {
6771         struct sk_buff *tail = IGB_CB(head)->tail;
6772
6773         if (!tail)
6774                 return false;
6775
6776         igb_merge_active_tail(tail);
6777
6778         IGB_CB(head)->tail = NULL;
6779
6780         return true;
6781 }
6782
6783 /**
6784  * igb_can_lro - returns true if packet is TCP/IPV4 and LRO is enabled
6785  * @adapter: board private structure
6786  * @rx_desc: pointer to the rx descriptor
6787  * @skb: pointer to the skb to be merged
6788  *
6789  **/
6790 static inline bool igb_can_lro(struct igb_ring *rx_ring,
6791                                union e1000_adv_rx_desc *rx_desc,
6792                                struct sk_buff *skb)
6793 {
6794         struct iphdr *iph = (struct iphdr *)skb->data;
6795         __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
6796
6797         /* verify LRO is enabled */
6798         if (!(netdev_ring(rx_ring)->features & NETIF_F_LRO))
6799                 return false;
6800
6801         /* verify hardware indicates this is IPv4/TCP */
6802         if((!(pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_TCP)) ||
6803             !(pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_IPV4))))
6804                 return false;
6805
6806         /* verify the header is large enough for us to read IP/TCP fields */
6807         if (!pskb_may_pull(skb, sizeof(struct igb_lrohdr)))
6808                 return false;
6809
6810         /* verify there are no VLANs on packet */
6811         if (skb->protocol != __constant_htons(ETH_P_IP))
6812                 return false;
6813
6814         /* ensure we are version 4 with no options */
6815         if (*(u8 *)iph != 0x45)
6816                 return false;
6817
6818         /* .. and the packet is not fragmented */
6819         if (iph->frag_off & htons(IP_MF | IP_OFFSET))
6820                 return false;
6821
6822         /* .. and that next header is TCP */
6823         if (iph->protocol != IPPROTO_TCP)
6824                 return false;
6825
6826         return true;
6827 }
6828
6829 static inline struct igb_lrohdr *igb_lro_hdr(struct sk_buff *skb)
6830 {
6831         return (struct igb_lrohdr *)skb->data;
6832 }
6833
6834 /**
6835  * igb_lro_flush - Indicate packets to upper layer.
6836  *
6837  * Update IP and TCP header part of head skb if more than one
6838  * skb's chained and indicate packets to upper layer.
6839  **/
6840 static void igb_lro_flush(struct igb_q_vector *q_vector,
6841                           struct sk_buff *skb)
6842 {
6843         struct igb_lro_list *lrolist = q_vector->lrolist;
6844
6845         __skb_unlink(skb, &lrolist->active);
6846
6847         if (IGB_CB(skb)->append_cnt) {
6848                 struct igb_lrohdr *lroh = igb_lro_hdr(skb);
6849
6850                 /* close any active lro contexts */
6851                 igb_close_active_frag_list(skb);
6852
6853                 /* incorporate ip header and re-calculate checksum */
6854                 lroh->iph.tot_len = ntohs(skb->len);
6855                 lroh->iph.check = 0;
6856
6857                 /* header length is 5 since we know no options exist */
6858                 lroh->iph.check = ip_fast_csum((u8 *)lroh, 5);
6859
6860                 /* clear TCP checksum to indicate we are an LRO frame */
6861                 lroh->th.check = 0;
6862
6863                 /* incorporate latest timestamp into the tcp header */
6864                 if (IGB_CB(skb)->tsecr) {
6865                         lroh->ts[2] = IGB_CB(skb)->tsecr;
6866                         lroh->ts[1] = htonl(IGB_CB(skb)->tsval);
6867                 }
6868 #ifdef NETIF_F_TSO
6869
6870                 skb_shinfo(skb)->gso_size = IGB_CB(skb)->mss;
6871 #endif
6872         }
6873
6874 #ifdef HAVE_VLAN_RX_REGISTER
6875         igb_receive_skb(q_vector, skb);
6876 #else
6877         napi_gro_receive(&q_vector->napi, skb);
6878 #endif
6879         lrolist->stats.flushed++;
6880 }
6881
6882 static void igb_lro_flush_all(struct igb_q_vector *q_vector)
6883 {
6884         struct igb_lro_list *lrolist = q_vector->lrolist;
6885         struct sk_buff *skb, *tmp;
6886
6887         skb_queue_reverse_walk_safe(&lrolist->active, skb, tmp)
6888                 igb_lro_flush(q_vector, skb);
6889 }
6890
6891 /*
6892  * igb_lro_header_ok - Main LRO function.
6893  **/
6894 static void igb_lro_header_ok(struct sk_buff *skb)
6895 {
6896         struct igb_lrohdr *lroh = igb_lro_hdr(skb);
6897         u16 opt_bytes, data_len;
6898
6899         IGB_CB(skb)->tail = NULL;
6900         IGB_CB(skb)->tsecr = 0;
6901         IGB_CB(skb)->append_cnt = 0;
6902         IGB_CB(skb)->mss = 0;
6903
6904         /* ensure that the checksum is valid */
6905         if (skb->ip_summed != CHECKSUM_UNNECESSARY)
6906                 return;
6907
6908         /* If we see CE codepoint in IP header, packet is not mergeable */
6909         if (INET_ECN_is_ce(ipv4_get_dsfield(&lroh->iph)))
6910                 return;
6911
6912         /* ensure no bits set besides ack or psh */
6913         if (lroh->th.fin || lroh->th.syn || lroh->th.rst ||
6914             lroh->th.urg || lroh->th.ece || lroh->th.cwr ||
6915             !lroh->th.ack)
6916                 return;
6917
6918         /* store the total packet length */
6919         data_len = ntohs(lroh->iph.tot_len);
6920
6921         /* remove any padding from the end of the skb */
6922         __pskb_trim(skb, data_len);
6923
6924         /* remove header length from data length */
6925         data_len -= sizeof(struct igb_lrohdr);
6926
6927         /*
6928          * check for timestamps. Since the only option we handle are timestamps,
6929          * we only have to handle the simple case of aligned timestamps
6930          */
6931         opt_bytes = (lroh->th.doff << 2) - sizeof(struct tcphdr);
6932         if (opt_bytes != 0) {
6933                 if ((opt_bytes != TCPOLEN_TSTAMP_ALIGNED) ||
6934                     !pskb_may_pull(skb, sizeof(struct igb_lrohdr) +
6935                                         TCPOLEN_TSTAMP_ALIGNED) ||
6936                     (lroh->ts[0] != htonl((TCPOPT_NOP << 24) |
6937                                              (TCPOPT_NOP << 16) |
6938                                              (TCPOPT_TIMESTAMP << 8) |
6939                                               TCPOLEN_TIMESTAMP)) ||
6940                     (lroh->ts[2] == 0)) {
6941                         return;
6942                 }
6943
6944                 IGB_CB(skb)->tsval = ntohl(lroh->ts[1]);
6945                 IGB_CB(skb)->tsecr = lroh->ts[2];
6946
6947                 data_len -= TCPOLEN_TSTAMP_ALIGNED;
6948         }
6949
6950         /* record data_len as mss for the packet */
6951         IGB_CB(skb)->mss = data_len;
6952         IGB_CB(skb)->next_seq = ntohl(lroh->th.seq);
6953 }
6954
6955 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
6956 static bool igb_merge_frags(struct sk_buff *lro_skb, struct sk_buff *new_skb)
6957 {
6958         struct sk_buff *tail;
6959         struct skb_shared_info *tail_info;
6960         struct skb_shared_info *new_skb_info;
6961         u16 data_len;
6962
6963         /* header must be empty to pull frags into current skb */
6964         if (skb_headlen(new_skb))
6965                 return false;
6966
6967         if (IGB_CB(lro_skb)->tail)
6968                 tail = IGB_CB(lro_skb)->tail;
6969         else
6970                 tail = lro_skb;
6971
6972         tail_info = skb_shinfo(tail);
6973         new_skb_info = skb_shinfo(new_skb);
6974
6975         /* make sure we have room in frags list */
6976         if (new_skb_info->nr_frags >= (MAX_SKB_FRAGS - tail_info->nr_frags))
6977                 return false;
6978
6979         /* bump append count */
6980         IGB_CB(lro_skb)->append_cnt++;
6981
6982         /* copy frags into the last skb */
6983         memcpy(tail_info->frags + tail_info->nr_frags,
6984                new_skb_info->frags,
6985                new_skb_info->nr_frags * sizeof(skb_frag_t));
6986
6987         /* copy size data over */
6988         tail_info->nr_frags += new_skb_info->nr_frags;
6989         data_len = IGB_CB(new_skb)->mss;
6990         tail->len += data_len;
6991         tail->data_len += data_len;
6992         tail->truesize += data_len;
6993
6994         /* wipe record of data from new_skb */
6995         new_skb_info->nr_frags = 0;
6996         new_skb->len = new_skb->data_len = 0;
6997         new_skb->truesize -= data_len;
6998         new_skb->data = new_skb->head + NET_SKB_PAD + NET_IP_ALIGN;
6999         skb_reset_tail_pointer(new_skb);
7000         new_skb->protocol = 0;
7001         new_skb->ip_summed = CHECKSUM_NONE;
7002 #ifdef HAVE_VLAN_RX_REGISTER
7003         IGB_CB(new_skb)->vid = 0;
7004 #else
7005         new_skb->vlan_tci = 0;
7006 #endif
7007
7008         return true;
7009 }
7010
7011 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7012 /**
7013  * igb_lro_queue - if able, queue skb into lro chain
7014  * @q_vector: structure containing interrupt and ring information
7015  * @new_skb: pointer to current skb being checked
7016  *
7017  * Checks whether the skb given is eligible for LRO and if that's
7018  * fine chains it to the existing lro_skb based on flowid. If an LRO for
7019  * the flow doesn't exist create one.
7020  **/
7021 static struct sk_buff *igb_lro_queue(struct igb_q_vector *q_vector,
7022                                        struct sk_buff *new_skb)
7023 {
7024         struct sk_buff *lro_skb;
7025         struct igb_lro_list *lrolist = q_vector->lrolist;
7026         struct igb_lrohdr *lroh = igb_lro_hdr(new_skb);
7027         __be32 saddr = lroh->iph.saddr;
7028         __be32 daddr = lroh->iph.daddr;
7029         __be32 tcp_ports = *(__be32 *)&lroh->th;
7030         u16 data_len;
7031 #ifdef HAVE_VLAN_RX_REGISTER
7032         u16 vid = IGB_CB(new_skb)->vid;
7033 #else
7034         u16 vid = new_skb->vlan_tci;
7035 #endif
7036
7037         igb_lro_header_ok(new_skb);
7038
7039         /*
7040          * we have a packet that might be eligible for LRO,
7041          * so see if it matches anything we might expect
7042          */
7043         skb_queue_walk(&lrolist->active, lro_skb) {
7044                 if (*(__be32 *)&igb_lro_hdr(lro_skb)->th != tcp_ports ||
7045                     igb_lro_hdr(lro_skb)->iph.saddr != saddr ||
7046                     igb_lro_hdr(lro_skb)->iph.daddr != daddr)
7047                         continue;
7048
7049 #ifdef HAVE_VLAN_RX_REGISTER
7050                 if (IGB_CB(lro_skb)->vid != vid)
7051 #else
7052                 if (lro_skb->vlan_tci != vid)
7053 #endif
7054                         continue;
7055
7056                 /* out of order packet */
7057                 if (IGB_CB(lro_skb)->next_seq != IGB_CB(new_skb)->next_seq) {
7058                         igb_lro_flush(q_vector, lro_skb);
7059                         IGB_CB(new_skb)->mss = 0;
7060                         break;
7061                 }
7062
7063                 /* TCP timestamp options have changed */
7064                 if (!IGB_CB(lro_skb)->tsecr != !IGB_CB(new_skb)->tsecr) {
7065                         igb_lro_flush(q_vector, lro_skb);
7066                         break;
7067                 }
7068
7069                 /* make sure timestamp values are increasing */
7070                 if (IGB_CB(lro_skb)->tsecr &&
7071                     IGB_CB(lro_skb)->tsval > IGB_CB(new_skb)->tsval) {
7072                         igb_lro_flush(q_vector, lro_skb);
7073                         IGB_CB(new_skb)->mss = 0;
7074                         break;
7075                 }
7076
7077                 data_len = IGB_CB(new_skb)->mss;
7078
7079                 /*
7080                  * malformed header, no tcp data, resultant packet would
7081                  * be too large, or new skb is larger than our current mss.
7082                  */
7083                 if (data_len == 0 ||
7084                     data_len > IGB_CB(lro_skb)->mss ||
7085                     data_len > IGB_CB(lro_skb)->free) {
7086                         igb_lro_flush(q_vector, lro_skb);
7087                         break;
7088                 }
7089
7090                 /* ack sequence numbers or window size has changed */
7091                 if (igb_lro_hdr(lro_skb)->th.ack_seq != lroh->th.ack_seq ||
7092                     igb_lro_hdr(lro_skb)->th.window != lroh->th.window) {
7093                         igb_lro_flush(q_vector, lro_skb);
7094                         break;
7095                 }
7096
7097                 /* Remove IP and TCP header*/
7098                 skb_pull(new_skb, new_skb->len - data_len);
7099
7100                 /* update timestamp and timestamp echo response */
7101                 IGB_CB(lro_skb)->tsval = IGB_CB(new_skb)->tsval;
7102                 IGB_CB(lro_skb)->tsecr = IGB_CB(new_skb)->tsecr;
7103
7104                 /* update sequence and free space */
7105                 IGB_CB(lro_skb)->next_seq += data_len;
7106                 IGB_CB(lro_skb)->free -= data_len;
7107
7108 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7109                 /* if header is empty pull pages into current skb */
7110                 if (igb_merge_frags(lro_skb, new_skb)) {
7111                         lrolist->stats.recycled++;
7112                 } else {
7113 #endif
7114                         /* chain this new skb in frag_list */
7115                         igb_add_active_tail(lro_skb, new_skb);
7116                         new_skb = NULL;
7117 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7118                 }
7119 #endif
7120
7121                 if ((data_len < IGB_CB(lro_skb)->mss) || lroh->th.psh) {
7122                         igb_lro_hdr(lro_skb)->th.psh |= lroh->th.psh;
7123                         igb_lro_flush(q_vector, lro_skb);
7124                 }
7125
7126                 lrolist->stats.coal++;
7127                 return new_skb;
7128         }
7129
7130         if (IGB_CB(new_skb)->mss && !lroh->th.psh) {
7131                 /* if we are at capacity flush the tail */
7132                 if (skb_queue_len(&lrolist->active) >= IGB_LRO_MAX) {
7133                         lro_skb = skb_peek_tail(&lrolist->active);
7134                         if (lro_skb)
7135                                 igb_lro_flush(q_vector, lro_skb);
7136                 }
7137
7138                 /* update sequence and free space */
7139                 IGB_CB(new_skb)->next_seq += IGB_CB(new_skb)->mss;
7140                 IGB_CB(new_skb)->free = 65521 - new_skb->len;
7141
7142                 /* .. and insert at the front of the active list */
7143                 __skb_queue_head(&lrolist->active, new_skb);
7144
7145                 lrolist->stats.coal++;
7146                 return NULL;
7147         }
7148
7149         /* packet not handled by any of the above, pass it to the stack */
7150 #ifdef HAVE_VLAN_RX_REGISTER
7151         igb_receive_skb(q_vector, new_skb);
7152 #else
7153         napi_gro_receive(&q_vector->napi, new_skb);
7154 #endif
7155         return NULL;
7156 }
7157
7158 #endif /* IGB_NO_LRO */
7159 static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
7160 {
7161         struct igb_ring *rx_ring = q_vector->rx.ring;
7162         union e1000_adv_rx_desc *rx_desc;
7163 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7164         const int current_node = numa_node_id();
7165 #endif
7166         unsigned int total_bytes = 0, total_packets = 0;
7167         u16 cleaned_count = igb_desc_unused(rx_ring);
7168         u16 i = rx_ring->next_to_clean;
7169
7170         rx_desc = IGB_RX_DESC(rx_ring, i);
7171
7172         while (igb_test_staterr(rx_desc, E1000_RXD_STAT_DD)) {
7173                 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
7174                 struct sk_buff *skb = buffer_info->skb;
7175                 union e1000_adv_rx_desc *next_rxd;
7176
7177                 buffer_info->skb = NULL;
7178                 prefetch(skb->data);
7179
7180                 i++;
7181                 if (i == rx_ring->count)
7182                         i = 0;
7183
7184                 next_rxd = IGB_RX_DESC(rx_ring, i);
7185                 prefetch(next_rxd);
7186
7187                 /*
7188                  * This memory barrier is needed to keep us from reading
7189                  * any other fields out of the rx_desc until we know the
7190                  * RXD_STAT_DD bit is set
7191                  */
7192                 rmb();
7193
7194 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7195                 __skb_put(skb, le16_to_cpu(rx_desc->wb.upper.length));
7196                 dma_unmap_single(rx_ring->dev, buffer_info->dma,
7197                                  rx_ring->rx_buffer_len,
7198                                  DMA_FROM_DEVICE);
7199                 buffer_info->dma = 0;
7200
7201 #else
7202                 if (!skb_is_nonlinear(skb)) {
7203                         __skb_put(skb, igb_get_hlen(rx_desc));
7204                         dma_unmap_single(rx_ring->dev, buffer_info->dma,
7205                                          IGB_RX_HDR_LEN,
7206                                          DMA_FROM_DEVICE);
7207                         buffer_info->dma = 0;
7208                 }
7209
7210                 if (rx_desc->wb.upper.length) {
7211                         u16 length = le16_to_cpu(rx_desc->wb.upper.length);
7212
7213                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
7214                                            buffer_info->page,
7215                                            buffer_info->page_offset,
7216                                            length);
7217
7218                         skb->len += length;
7219                         skb->data_len += length;
7220                         skb->truesize += length;
7221
7222                         if ((page_count(buffer_info->page) != 1) ||
7223                             (page_to_nid(buffer_info->page) != current_node))
7224                                 buffer_info->page = NULL;
7225                         else
7226                                 get_page(buffer_info->page);
7227
7228                         dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
7229                                        PAGE_SIZE / 2, DMA_FROM_DEVICE);
7230                         buffer_info->page_dma = 0;
7231                 }
7232
7233                 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)) {
7234                         struct igb_rx_buffer *next_buffer;
7235                         next_buffer = &rx_ring->rx_buffer_info[i];
7236                         buffer_info->skb = next_buffer->skb;
7237                         buffer_info->dma = next_buffer->dma;
7238                         next_buffer->skb = skb;
7239                         next_buffer->dma = 0;
7240                         goto next_desc;
7241                 }
7242
7243 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7244                 if (igb_test_staterr(rx_desc,
7245                                      E1000_RXDEXT_ERR_FRAME_ERR_MASK)) {
7246                         dev_kfree_skb_any(skb);
7247                         goto next_desc;
7248                 }
7249
7250 #ifdef HAVE_HW_TIME_STAMP
7251                 igb_rx_hwtstamp(q_vector, rx_desc, skb);
7252 #endif
7253 #ifdef NETIF_F_RXHASH
7254                 igb_rx_hash(rx_ring, rx_desc, skb);
7255 #endif
7256                 igb_rx_checksum(rx_ring, rx_desc, skb);
7257                 igb_rx_vlan(rx_ring, rx_desc, skb);
7258
7259                 total_bytes += skb->len;
7260                 total_packets++;
7261
7262                 skb->protocol = eth_type_trans(skb, netdev_ring(rx_ring));
7263
7264 #ifndef IGB_NO_LRO
7265                 if (igb_can_lro(rx_ring, rx_desc, skb))
7266                         buffer_info->skb = igb_lro_queue(q_vector, skb);
7267                 else
7268 #endif
7269 #ifdef HAVE_VLAN_RX_REGISTER
7270                         igb_receive_skb(q_vector, skb);
7271 #else
7272                         napi_gro_receive(&q_vector->napi, skb);
7273 #endif
7274
7275 #ifndef NETIF_F_GRO
7276                 netdev_ring(rx_ring)->last_rx = jiffies;
7277
7278 #endif
7279                 budget--;
7280 next_desc:
7281                 cleaned_count++;
7282
7283                 if (!budget)
7284                         break;
7285
7286                 /* return some buffers to hardware, one at a time is too slow */
7287                 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
7288                         igb_alloc_rx_buffers(rx_ring, cleaned_count);
7289                         cleaned_count = 0;
7290                 }
7291
7292                 /* use prefetched values */
7293                 rx_desc = next_rxd;
7294         }
7295
7296         rx_ring->next_to_clean = i;
7297         rx_ring->rx_stats.packets += total_packets;
7298         rx_ring->rx_stats.bytes += total_bytes;
7299         q_vector->rx.total_packets += total_packets;
7300         q_vector->rx.total_bytes += total_bytes;
7301
7302         if (cleaned_count)
7303                 igb_alloc_rx_buffers(rx_ring, cleaned_count);
7304
7305 #ifndef IGB_NO_LRO
7306         if (netdev_ring(rx_ring)->features & NETIF_F_LRO)
7307                 igb_lro_flush_all(q_vector);
7308
7309 #endif /* IGB_NO_LRO */
7310         return !!budget;
7311 }
7312
7313 static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
7314                                  struct igb_rx_buffer *bi)
7315 {
7316         struct sk_buff *skb = bi->skb;
7317         dma_addr_t dma = bi->dma;
7318
7319         if (dma)
7320                 return true;
7321
7322         if (likely(!skb)) {
7323 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7324                 skb = netdev_alloc_skb_ip_align(netdev_ring(rx_ring),
7325                                                 rx_ring->rx_buffer_len);
7326 #else
7327                 skb = netdev_alloc_skb_ip_align(netdev_ring(rx_ring),
7328                                                 IGB_RX_HDR_LEN);
7329 #endif
7330                 bi->skb = skb;
7331                 if (!skb) {
7332                         rx_ring->rx_stats.alloc_failed++;
7333                         return false;
7334                 }
7335
7336                 /* initialize skb for ring */
7337                 skb_record_rx_queue(skb, ring_queue_index(rx_ring));
7338         }
7339
7340 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7341         dma = dma_map_single(rx_ring->dev, skb->data,
7342                              rx_ring->rx_buffer_len, DMA_FROM_DEVICE);
7343 #else
7344         dma = dma_map_single(rx_ring->dev, skb->data,
7345                              IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
7346 #endif
7347
7348         if (dma_mapping_error(rx_ring->dev, dma)) {
7349                 rx_ring->rx_stats.alloc_failed++;
7350                 return false;
7351         }
7352
7353         bi->dma = dma;
7354         return true;
7355 }
7356
7357 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7358 static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
7359                                   struct igb_rx_buffer *bi)
7360 {
7361         struct page *page = bi->page;
7362         dma_addr_t page_dma = bi->page_dma;
7363         unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
7364
7365         if (page_dma)
7366                 return true;
7367
7368         if (!page) {
7369                 page = alloc_page(GFP_ATOMIC | __GFP_COLD);
7370                 bi->page = page;
7371                 if (unlikely(!page)) {
7372                         rx_ring->rx_stats.alloc_failed++;
7373                         return false;
7374                 }
7375         }
7376
7377         page_dma = dma_map_page(rx_ring->dev, page,
7378                                 page_offset, PAGE_SIZE / 2,
7379                                 DMA_FROM_DEVICE);
7380
7381         if (dma_mapping_error(rx_ring->dev, page_dma)) {
7382                 rx_ring->rx_stats.alloc_failed++;
7383                 return false;
7384         }
7385
7386         bi->page_dma = page_dma;
7387         bi->page_offset = page_offset;
7388         return true;
7389 }
7390
7391 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7392 /**
7393  * igb_alloc_rx_buffers - Replace used receive buffers; packet split
7394  * @adapter: address of board private structure
7395  **/
7396 void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
7397 {
7398         union e1000_adv_rx_desc *rx_desc;
7399         struct igb_rx_buffer *bi;
7400         u16 i = rx_ring->next_to_use;
7401
7402         rx_desc = IGB_RX_DESC(rx_ring, i);
7403         bi = &rx_ring->rx_buffer_info[i];
7404         i -= rx_ring->count;
7405
7406         while (cleaned_count--) {
7407                 if (!igb_alloc_mapped_skb(rx_ring, bi))
7408                         break;
7409
7410                 /* Refresh the desc even if buffer_addrs didn't change
7411                  * because each write-back erases this info. */
7412 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7413                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
7414 #else
7415                 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
7416
7417                 if (!igb_alloc_mapped_page(rx_ring, bi))
7418                         break;
7419
7420                 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
7421
7422 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7423                 rx_desc++;
7424                 bi++;
7425                 i++;
7426                 if (unlikely(!i)) {
7427                         rx_desc = IGB_RX_DESC(rx_ring, 0);
7428                         bi = rx_ring->rx_buffer_info;
7429                         i -= rx_ring->count;
7430                 }
7431
7432                 /* clear the hdr_addr for the next_to_use descriptor */
7433                 rx_desc->read.hdr_addr = 0;
7434         }
7435
7436         i += rx_ring->count;
7437
7438         if (rx_ring->next_to_use != i) {
7439                 rx_ring->next_to_use = i;
7440
7441                 /* Force memory writes to complete before letting h/w
7442                  * know there are new descriptors to fetch.  (Only
7443                  * applicable for weak-ordered memory model archs,
7444                  * such as IA-64). */
7445                 wmb();
7446                 writel(i, rx_ring->tail);
7447         }
7448 }
7449
7450 #ifdef SIOCGMIIPHY
7451 /**
7452  * igb_mii_ioctl -
7453  * @netdev:
7454  * @ifreq:
7455  * @cmd:
7456  **/
7457 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
7458 {
7459         struct igb_adapter *adapter = netdev_priv(netdev);
7460         struct mii_ioctl_data *data = if_mii(ifr);
7461
7462         if (adapter->hw.phy.media_type != e1000_media_type_copper)
7463                 return -EOPNOTSUPP;
7464
7465         switch (cmd) {
7466         case SIOCGMIIPHY:
7467                 data->phy_id = adapter->hw.phy.addr;
7468                 break;
7469         case SIOCGMIIREG:
7470                 if (!capable(CAP_NET_ADMIN))
7471                         return -EPERM;
7472                 if (e1000_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
7473                                    &data->val_out))
7474                         return -EIO;
7475                 break;
7476         case SIOCSMIIREG:
7477         default:
7478                 return -EOPNOTSUPP;
7479         }
7480         return E1000_SUCCESS;
7481 }
7482
7483 #endif
7484 #ifdef HAVE_HW_TIME_STAMP
7485 /**
7486  * igb_hwtstamp_ioctl - control hardware time stamping
7487  * @netdev:
7488  * @ifreq:
7489  * @cmd:
7490  *
7491  * Outgoing time stamping can be enabled and disabled. Play nice and
7492  * disable it when requested, although it shouldn't case any overhead
7493  * when no packet needs it. At most one packet in the queue may be
7494  * marked for time stamping, otherwise it would be impossible to tell
7495  * for sure to which packet the hardware time stamp belongs.
7496  *
7497  * Incoming time stamping has to be configured via the hardware
7498  * filters. Not all combinations are supported, in particular event
7499  * type has to be specified. Matching the kind of event packet is
7500  * not supported, with the exception of "all V2 events regardless of
7501  * level 2 or 4".
7502  *
7503  **/
7504 static int igb_hwtstamp_ioctl(struct net_device *netdev,
7505                               struct ifreq *ifr, int cmd)
7506 {
7507         struct igb_adapter *adapter = netdev_priv(netdev);
7508         struct e1000_hw *hw = &adapter->hw;
7509         struct hwtstamp_config config;
7510         u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
7511         u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
7512         u32 tsync_rx_cfg = 0;
7513         bool is_l4 = false;
7514         bool is_l2 = false;
7515         u32 regval;
7516
7517         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
7518                 return -EFAULT;
7519
7520         /* reserved for future extensions */
7521         if (config.flags)
7522                 return -EINVAL;
7523
7524         switch (config.tx_type) {
7525         case HWTSTAMP_TX_OFF:
7526                 tsync_tx_ctl = 0;
7527         case HWTSTAMP_TX_ON:
7528                 break;
7529         default:
7530                 return -ERANGE;
7531         }
7532
7533         switch (config.rx_filter) {
7534         case HWTSTAMP_FILTER_NONE:
7535                 tsync_rx_ctl = 0;
7536                 break;
7537         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
7538         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
7539         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
7540         case HWTSTAMP_FILTER_ALL:
7541                 /*
7542                  * register TSYNCRXCFG must be set, therefore it is not
7543                  * possible to time stamp both Sync and Delay_Req messages
7544                  * => fall back to time stamping all packets
7545                  */
7546                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
7547                 config.rx_filter = HWTSTAMP_FILTER_ALL;
7548                 break;
7549         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
7550                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
7551                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
7552                 is_l4 = true;
7553                 break;
7554         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
7555                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
7556                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
7557                 is_l4 = true;
7558                 break;
7559         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
7560         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
7561                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
7562                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
7563                 is_l2 = true;
7564                 is_l4 = true;
7565                 config.rx_filter = HWTSTAMP_FILTER_SOME;
7566                 break;
7567         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
7568         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
7569                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
7570                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
7571                 is_l2 = true;
7572                 is_l4 = true;
7573                 config.rx_filter = HWTSTAMP_FILTER_SOME;
7574                 break;
7575         case HWTSTAMP_FILTER_PTP_V2_EVENT:
7576         case HWTSTAMP_FILTER_PTP_V2_SYNC:
7577         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
7578                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
7579                 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
7580                 is_l2 = true;
7581                 is_l4 = true;
7582                 break;
7583         default:
7584                 return -ERANGE;
7585         }
7586
7587         if (hw->mac.type == e1000_82575) {
7588                 if (tsync_rx_ctl | tsync_tx_ctl)
7589                         return -EINVAL;
7590                 return 0;
7591         }
7592
7593 #ifdef IGB_PER_PKT_TIMESTAMP
7594         /*
7595          * Per-packet timestamping only works if all packets are
7596          * timestamped, so enable timestamping in all packets as
7597          * long as one rx filter was configured.
7598          */
7599         if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) {
7600                 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
7601                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
7602         }
7603 #endif
7604
7605         /* enable/disable TX */
7606         regval = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
7607         regval &= ~E1000_TSYNCTXCTL_ENABLED;
7608         regval |= tsync_tx_ctl;
7609         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, regval);
7610
7611         /* enable/disable RX */
7612         regval = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
7613         regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
7614         regval |= tsync_rx_ctl;
7615         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, regval);
7616
7617         /* define which PTP packets are time stamped */
7618         E1000_WRITE_REG(hw, E1000_TSYNCRXCFG, tsync_rx_cfg);
7619
7620         /* define ethertype filter for timestamped packets */
7621         if (is_l2)
7622                 E1000_WRITE_REG(hw, E1000_ETQF(3),
7623                                 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
7624                                  E1000_ETQF_1588 | /* enable timestamping */
7625                                  ETH_P_1588));     /* 1588 eth protocol type */
7626         else
7627                 E1000_WRITE_REG(hw, E1000_ETQF(3), 0);
7628
7629 #define PTP_PORT 319
7630         /* L4 Queue Filter[3]: filter by destination port and protocol */
7631         if (is_l4) {
7632                 u32 ftqf = (IPPROTO_UDP /* UDP */
7633                         | E1000_FTQF_VF_BP /* VF not compared */
7634                         | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
7635                         | E1000_FTQF_MASK); /* mask all inputs */
7636                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
7637
7638                 E1000_WRITE_REG(hw, E1000_IMIR(3), htons(PTP_PORT));
7639                 E1000_WRITE_REG(hw, E1000_IMIREXT(3),
7640                                 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
7641                 if (hw->mac.type == e1000_82576) {
7642                         /* enable source port check */
7643                         E1000_WRITE_REG(hw, E1000_SPQF(3), htons(PTP_PORT));
7644                         ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
7645                 }
7646                 E1000_WRITE_REG(hw, E1000_FTQF(3), ftqf);
7647         } else {
7648                 E1000_WRITE_REG(hw, E1000_FTQF(3), E1000_FTQF_MASK);
7649         }
7650         E1000_WRITE_FLUSH(hw);
7651
7652         adapter->hwtstamp_config = config;
7653
7654         /* clear TX/RX time stamp registers, just to be sure */
7655         regval = E1000_READ_REG(hw, E1000_TXSTMPH);
7656         regval = E1000_READ_REG(hw, E1000_RXSTMPH);
7657
7658         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
7659                 -EFAULT : 0;
7660 }
7661
7662 #endif
7663 /**
7664  * igb_ioctl -
7665  * @netdev:
7666  * @ifreq:
7667  * @cmd:
7668  **/
7669 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
7670 {
7671         switch (cmd) {
7672 #ifdef SIOCGMIIPHY
7673         case SIOCGMIIPHY:
7674         case SIOCGMIIREG:
7675         case SIOCSMIIREG:
7676                 return igb_mii_ioctl(netdev, ifr, cmd);
7677 #endif
7678 #ifdef HAVE_HW_TIME_STAMP
7679         case SIOCSHWTSTAMP:
7680                 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
7681 #endif
7682 #ifdef ETHTOOL_OPS_COMPAT
7683         case SIOCETHTOOL:
7684                 return ethtool_ioctl(ifr);
7685 #endif
7686         default:
7687                 return -EOPNOTSUPP;
7688         }
7689 }
7690
7691 s32 e1000_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
7692 {
7693         struct igb_adapter *adapter = hw->back;
7694         u16 cap_offset;
7695
7696         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
7697         if (!cap_offset)
7698                 return -E1000_ERR_CONFIG;
7699
7700         pci_read_config_word(adapter->pdev, cap_offset + reg, value);
7701
7702         return E1000_SUCCESS;
7703 }
7704
7705 s32 e1000_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
7706 {
7707         struct igb_adapter *adapter = hw->back;
7708         u16 cap_offset;
7709
7710         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
7711         if (!cap_offset)
7712                 return -E1000_ERR_CONFIG;
7713
7714         pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
7715
7716         return E1000_SUCCESS;
7717 }
7718
7719 #ifdef HAVE_VLAN_RX_REGISTER
7720 static void igb_vlan_mode(struct net_device *netdev, struct vlan_group *vlgrp)
7721 #else
7722 void igb_vlan_mode(struct net_device *netdev, u32 features)
7723 #endif
7724 {
7725         struct igb_adapter *adapter = netdev_priv(netdev);
7726         struct e1000_hw *hw = &adapter->hw;
7727         u32 ctrl, rctl;
7728         int i;
7729 #ifdef HAVE_VLAN_RX_REGISTER
7730         bool enable = !!vlgrp;
7731
7732         igb_irq_disable(adapter);
7733
7734         adapter->vlgrp = vlgrp;
7735
7736         if (!test_bit(__IGB_DOWN, &adapter->state))
7737                 igb_irq_enable(adapter);
7738 #else
7739         bool enable = !!(features & NETIF_F_HW_VLAN_RX);
7740 #endif
7741
7742         if (enable) {
7743                 /* enable VLAN tag insert/strip */
7744                 ctrl = E1000_READ_REG(hw, E1000_CTRL);
7745                 ctrl |= E1000_CTRL_VME;
7746                 E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
7747
7748                 /* Disable CFI check */
7749                 rctl = E1000_READ_REG(hw, E1000_RCTL);
7750                 rctl &= ~E1000_RCTL_CFIEN;
7751                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
7752         } else {
7753                 /* disable VLAN tag insert/strip */
7754                 ctrl = E1000_READ_REG(hw, E1000_CTRL);
7755                 ctrl &= ~E1000_CTRL_VME;
7756                 E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
7757         }
7758
7759 #ifndef CONFIG_IGB_VMDQ_NETDEV
7760         for (i = 0; i < adapter->vmdq_pools; i++) {
7761                 igb_set_vf_vlan_strip(adapter,
7762                                       adapter->vfs_allocated_count + i,
7763                                       enable);
7764         }
7765
7766 #else
7767         igb_set_vf_vlan_strip(adapter,
7768                               adapter->vfs_allocated_count,
7769                               enable);
7770
7771         for (i = 1; i < adapter->vmdq_pools; i++) {
7772 #ifdef HAVE_VLAN_RX_REGISTER
7773                 struct igb_vmdq_adapter *vadapter;
7774                 vadapter = netdev_priv(adapter->vmdq_netdev[i-1]);
7775                 enable = !!vadapter->vlgrp;
7776 #else
7777                 struct net_device *vnetdev;
7778                 vnetdev = adapter->vmdq_netdev[i-1];
7779                 enable = !!(vnetdev->features & NETIF_F_HW_VLAN_RX);
7780 #endif
7781                 igb_set_vf_vlan_strip(adapter,
7782                                       adapter->vfs_allocated_count + i,
7783                                       enable);
7784         }
7785
7786 #endif
7787         igb_rlpml_set(adapter);
7788 }
7789
7790 #ifdef HAVE_VLAN_PROTOCOL
7791 static int igb_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
7792 #elif defined HAVE_INT_NDO_VLAN_RX_ADD_VID
7793 static int igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
7794 #else
7795 static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
7796 #endif
7797 {
7798         struct igb_adapter *adapter = netdev_priv(netdev);
7799         int pf_id = adapter->vfs_allocated_count;
7800
7801         /* attempt to add filter to vlvf array */
7802         igb_vlvf_set(adapter, vid, TRUE, pf_id);
7803
7804         /* add the filter since PF can receive vlans w/o entry in vlvf */
7805         igb_vfta_set(adapter, vid, TRUE);
7806 #ifndef HAVE_NETDEV_VLAN_FEATURES
7807
7808         /* Copy feature flags from netdev to the vlan netdev for this vid.
7809          * This allows things like TSO to bubble down to our vlan device.
7810          * There is no need to update netdev for vlan 0 (DCB), since it
7811          * wouldn't has v_netdev.
7812          */
7813         if (adapter->vlgrp) {
7814                 struct vlan_group *vlgrp = adapter->vlgrp;
7815                 struct net_device *v_netdev = vlan_group_get_device(vlgrp, vid);
7816                 if (v_netdev) {
7817                         v_netdev->features |= netdev->features;
7818                         vlan_group_set_device(vlgrp, vid, v_netdev);
7819                 }
7820         }
7821 #endif
7822 #ifndef HAVE_VLAN_RX_REGISTER
7823
7824         set_bit(vid, adapter->active_vlans);
7825 #endif
7826 #ifdef HAVE_INT_NDO_VLAN_RX_ADD_VID
7827         return 0;
7828 #endif
7829 }
7830
7831 #ifdef HAVE_VLAN_PROTOCOL
7832 static int igb_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
7833 #elif defined HAVE_INT_NDO_VLAN_RX_ADD_VID
7834 static int igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
7835 #else
7836 static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
7837 #endif
7838 {
7839         struct igb_adapter *adapter = netdev_priv(netdev);
7840         int pf_id = adapter->vfs_allocated_count;
7841         s32 err;
7842
7843 #ifdef HAVE_VLAN_RX_REGISTER
7844         igb_irq_disable(adapter);
7845
7846         vlan_group_set_device(adapter->vlgrp, vid, NULL);
7847
7848         if (!test_bit(__IGB_DOWN, &adapter->state))
7849                 igb_irq_enable(adapter);
7850
7851 #endif /* HAVE_VLAN_RX_REGISTER */
7852         /* remove vlan from VLVF table array */
7853         err = igb_vlvf_set(adapter, vid, FALSE, pf_id);
7854
7855         /* if vid was not present in VLVF just remove it from table */
7856         if (err)
7857                 igb_vfta_set(adapter, vid, FALSE);
7858 #ifndef HAVE_VLAN_RX_REGISTER
7859
7860         clear_bit(vid, adapter->active_vlans);
7861 #endif
7862 #ifdef HAVE_INT_NDO_VLAN_RX_ADD_VID
7863         return 0;
7864 #endif
7865 }
7866
7867 static void igb_restore_vlan(struct igb_adapter *adapter)
7868 {
7869 #ifdef HAVE_VLAN_RX_REGISTER
7870         igb_vlan_mode(adapter->netdev, adapter->vlgrp);
7871
7872         if (adapter->vlgrp) {
7873                 u16 vid;
7874                 for (vid = 0; vid < VLAN_N_VID; vid++) {
7875                         if (!vlan_group_get_device(adapter->vlgrp, vid))
7876                                 continue;
7877                         igb_vlan_rx_add_vid(adapter->netdev, vid);
7878                 }
7879         }
7880 #else
7881         u16 vid;
7882
7883         igb_vlan_mode(adapter->netdev, adapter->netdev->features);
7884
7885         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
7886 #ifdef HAVE_VLAN_PROTOCOL
7887                 igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
7888 #else
7889                 igb_vlan_rx_add_vid(adapter->netdev, vid);
7890 #endif
7891
7892 #endif
7893 }
7894
7895 int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
7896 {
7897         struct pci_dev *pdev = adapter->pdev;
7898         struct e1000_mac_info *mac = &adapter->hw.mac;
7899
7900         mac->autoneg = 0;
7901
7902         /* Fiber NIC's only allow 1000 gbps Full duplex */
7903         if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes ) &&
7904                 spddplx != (SPEED_1000 + DUPLEX_FULL)) {
7905                 dev_err(pci_dev_to_dev(pdev),
7906                         "Unsupported Speed/Duplex configuration\n");
7907                 return -EINVAL;
7908         }
7909
7910         switch (spddplx) {
7911         case SPEED_10 + DUPLEX_HALF:
7912                 mac->forced_speed_duplex = ADVERTISE_10_HALF;
7913                 break;
7914         case SPEED_10 + DUPLEX_FULL:
7915                 mac->forced_speed_duplex = ADVERTISE_10_FULL;
7916                 break;
7917         case SPEED_100 + DUPLEX_HALF:
7918                 mac->forced_speed_duplex = ADVERTISE_100_HALF;
7919                 break;
7920         case SPEED_100 + DUPLEX_FULL:
7921                 mac->forced_speed_duplex = ADVERTISE_100_FULL;
7922                 break;
7923         case SPEED_1000 + DUPLEX_FULL:
7924                 mac->autoneg = 1;
7925                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
7926                 break;
7927         case SPEED_1000 + DUPLEX_HALF: /* not supported */
7928         default:
7929                 dev_err(pci_dev_to_dev(pdev), "Unsupported Speed/Duplex configuration\n");
7930                 return -EINVAL;
7931         }
7932         return 0;
7933 }
7934
7935 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
7936                           bool runtime)
7937 {
7938         struct net_device *netdev = pci_get_drvdata(pdev);
7939         struct igb_adapter *adapter = netdev_priv(netdev);
7940         struct e1000_hw *hw = &adapter->hw;
7941         u32 ctrl, rctl, status;
7942         u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
7943 #ifdef CONFIG_PM
7944         int retval = 0;
7945 #endif
7946
7947         netif_device_detach(netdev);
7948
7949         if (netif_running(netdev))
7950                 __igb_close(netdev, true);
7951
7952         igb_clear_interrupt_scheme(adapter);
7953
7954 #ifdef CONFIG_PM
7955         retval = pci_save_state(pdev);
7956         if (retval)
7957                 return retval;
7958 #endif
7959
7960         status = E1000_READ_REG(hw, E1000_STATUS);
7961         if (status & E1000_STATUS_LU)
7962                 wufc &= ~E1000_WUFC_LNKC;
7963
7964         if (wufc) {
7965                 igb_setup_rctl(adapter);
7966                 igb_set_rx_mode(netdev);
7967
7968                 /* turn on all-multi mode if wake on multicast is enabled */
7969                 if (wufc & E1000_WUFC_MC) {
7970                         rctl = E1000_READ_REG(hw, E1000_RCTL);
7971                         rctl |= E1000_RCTL_MPE;
7972                         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
7973                 }
7974
7975                 ctrl = E1000_READ_REG(hw, E1000_CTRL);
7976                 /* phy power management enable */
7977                 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
7978                 ctrl |= E1000_CTRL_ADVD3WUC;
7979                 E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
7980
7981                 /* Allow time for pending master requests to run */
7982                 e1000_disable_pcie_master(hw);
7983
7984                 E1000_WRITE_REG(hw, E1000_WUC, E1000_WUC_PME_EN);
7985                 E1000_WRITE_REG(hw, E1000_WUFC, wufc);
7986         } else {
7987                 E1000_WRITE_REG(hw, E1000_WUC, 0);
7988                 E1000_WRITE_REG(hw, E1000_WUFC, 0);
7989         }
7990
7991         *enable_wake = wufc || adapter->en_mng_pt;
7992         if (!*enable_wake)
7993                 igb_power_down_link(adapter);
7994         else
7995                 igb_power_up_link(adapter);
7996
7997         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
7998          * would have already happened in close and is redundant. */
7999         igb_release_hw_control(adapter);
8000
8001         pci_disable_device(pdev);
8002
8003         return 0;
8004 }
8005
8006 #ifdef CONFIG_PM
8007 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
8008 static int igb_suspend(struct device *dev)
8009 {
8010         int retval;
8011         bool wake;
8012         struct pci_dev *pdev = to_pci_dev(dev);
8013
8014         retval = __igb_shutdown(pdev, &wake, 0);
8015         if (retval)
8016                 return retval;
8017
8018         if (wake) {
8019                 pci_prepare_to_sleep(pdev);
8020         } else {
8021                 pci_wake_from_d3(pdev, false);
8022                 pci_set_power_state(pdev, PCI_D3hot);
8023         }
8024
8025         return 0;
8026 }
8027
8028 static int igb_resume(struct device *dev)
8029 {
8030         struct pci_dev *pdev = to_pci_dev(dev);
8031         struct net_device *netdev = pci_get_drvdata(pdev);
8032         struct igb_adapter *adapter = netdev_priv(netdev);
8033         struct e1000_hw *hw = &adapter->hw;
8034         u32 err;
8035
8036         pci_set_power_state(pdev, PCI_D0);
8037         pci_restore_state(pdev);
8038         pci_save_state(pdev);
8039
8040         err = pci_enable_device_mem(pdev);
8041         if (err) {
8042                 dev_err(pci_dev_to_dev(pdev),
8043                         "igb: Cannot enable PCI device from suspend\n");
8044                 return err;
8045         }
8046         pci_set_master(pdev);
8047
8048         pci_enable_wake(pdev, PCI_D3hot, 0);
8049         pci_enable_wake(pdev, PCI_D3cold, 0);
8050
8051 #ifdef CONFIG_PM_RUNTIME
8052         if (!rtnl_is_locked()) {
8053                 /*
8054                  * shut up ASSERT_RTNL() warning in
8055                  * netif_set_real_num_tx/rx_queues.
8056                  */
8057                 rtnl_lock();
8058                 err = igb_init_interrupt_scheme(adapter);
8059                 rtnl_unlock();
8060         } else {
8061                 err = igb_init_interrupt_scheme(adapter);
8062         }
8063         if (err) {
8064 #else
8065         if (igb_init_interrupt_scheme(adapter)) {
8066 #endif /* CONFIG_PM_RUNTIME */
8067                 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for queues\n");
8068                 return -ENOMEM;
8069         }
8070
8071         igb_reset(adapter);
8072
8073         /* let the f/w know that the h/w is now under the control of the
8074          * driver. */
8075         igb_get_hw_control(adapter);
8076
8077         E1000_WRITE_REG(hw, E1000_WUS, ~0);
8078
8079         if (netdev->flags & IFF_UP) {
8080                 err = __igb_open(netdev, true);
8081                 if (err)
8082                         return err;
8083         }
8084
8085         netif_device_attach(netdev);
8086
8087         return 0;
8088 }
8089
8090 #ifdef CONFIG_PM_RUNTIME
8091 static int igb_runtime_idle(struct device *dev)
8092 {
8093         struct pci_dev *pdev = to_pci_dev(dev);
8094         struct net_device *netdev = pci_get_drvdata(pdev);
8095         struct igb_adapter *adapter = netdev_priv(netdev);
8096
8097         if (!igb_has_link(adapter))
8098                 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
8099
8100         return -EBUSY;
8101 }
8102
8103 static int igb_runtime_suspend(struct device *dev)
8104 {
8105         struct pci_dev *pdev = to_pci_dev(dev);
8106         int retval;
8107         bool wake;
8108
8109         retval = __igb_shutdown(pdev, &wake, 1);
8110         if (retval)
8111                 return retval;
8112
8113         if (wake) {
8114                 pci_prepare_to_sleep(pdev);
8115         } else {
8116                 pci_wake_from_d3(pdev, false);
8117                 pci_set_power_state(pdev, PCI_D3hot);
8118         }
8119
8120         return 0;
8121 }
8122
8123 static int igb_runtime_resume(struct device *dev)
8124 {
8125         return igb_resume(dev);
8126 }
8127 #endif /* CONFIG_PM_RUNTIME */
8128 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
8129 #endif /* CONFIG_PM */
8130
8131 #ifdef USE_REBOOT_NOTIFIER
8132 /* only want to do this for 2.4 kernels? */
8133 static int igb_notify_reboot(struct notifier_block *nb, unsigned long event,
8134                              void *p)
8135 {
8136         struct pci_dev *pdev = NULL;
8137         bool wake;
8138
8139         switch (event) {
8140         case SYS_DOWN:
8141         case SYS_HALT:
8142         case SYS_POWER_OFF:
8143                 while ((pdev = pci_find_device(PCI_ANY_ID, PCI_ANY_ID, pdev))) {
8144                         if (pci_dev_driver(pdev) == &igb_driver) {
8145                                 __igb_shutdown(pdev, &wake, 0);
8146                                 if (event == SYS_POWER_OFF) {
8147                                         pci_wake_from_d3(pdev, wake);
8148                                         pci_set_power_state(pdev, PCI_D3hot);
8149                                 }
8150                         }
8151                 }
8152         }
8153         return NOTIFY_DONE;
8154 }
8155 #else
8156 static void igb_shutdown(struct pci_dev *pdev)
8157 {
8158         bool wake = false;
8159
8160         __igb_shutdown(pdev, &wake, 0);
8161
8162         if (system_state == SYSTEM_POWER_OFF) {
8163                 pci_wake_from_d3(pdev, wake);
8164                 pci_set_power_state(pdev, PCI_D3hot);
8165         }
8166 }
8167 #endif /* USE_REBOOT_NOTIFIER */
8168
8169 #ifdef CONFIG_NET_POLL_CONTROLLER
8170 /*
8171  * Polling 'interrupt' - used by things like netconsole to send skbs
8172  * without having to re-enable interrupts. It's not called while
8173  * the interrupt routine is executing.
8174  */
8175 static void igb_netpoll(struct net_device *netdev)
8176 {
8177         struct igb_adapter *adapter = netdev_priv(netdev);
8178         struct e1000_hw *hw = &adapter->hw;
8179         struct igb_q_vector *q_vector;
8180         int i;
8181
8182         for (i = 0; i < adapter->num_q_vectors; i++) {
8183                 q_vector = adapter->q_vector[i];
8184                 if (adapter->msix_entries)
8185                         E1000_WRITE_REG(hw, E1000_EIMC, q_vector->eims_value);
8186                 else
8187                         igb_irq_disable(adapter);
8188                 napi_schedule(&q_vector->napi);
8189         }
8190 }
8191 #endif /* CONFIG_NET_POLL_CONTROLLER */
8192
8193 #ifdef HAVE_PCI_ERS
8194 #define E1000_DEV_ID_82576_VF 0x10CA
8195 /**
8196  * igb_io_error_detected - called when PCI error is detected
8197  * @pdev: Pointer to PCI device
8198  * @state: The current pci connection state
8199  *
8200  * This function is called after a PCI bus error affecting
8201  * this device has been detected.
8202  */
8203 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
8204                                               pci_channel_state_t state)
8205 {
8206         struct net_device *netdev = pci_get_drvdata(pdev);
8207         struct igb_adapter *adapter = netdev_priv(netdev);
8208
8209 #ifdef CONFIG_PCI_IOV__UNUSED
8210         struct pci_dev *bdev, *vfdev;
8211         u32 dw0, dw1, dw2, dw3;
8212         int vf, pos;
8213         u16 req_id, pf_func;
8214
8215         if (!(adapter->flags & IGB_FLAG_DETECT_BAD_DMA))
8216                 goto skip_bad_vf_detection;
8217
8218         bdev = pdev->bus->self;
8219         while (bdev && (bdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT))
8220                 bdev = bdev->bus->self;
8221
8222         if (!bdev)
8223                 goto skip_bad_vf_detection;
8224
8225         pos = pci_find_ext_capability(bdev, PCI_EXT_CAP_ID_ERR);
8226         if (!pos)
8227                 goto skip_bad_vf_detection;
8228
8229         pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG, &dw0);
8230         pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 4, &dw1);
8231         pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 8, &dw2);
8232         pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 12, &dw3);
8233
8234         req_id = dw1 >> 16;
8235         /* On the 82576 if bit 7 of the requestor ID is set then it's a VF */
8236         if (!(req_id & 0x0080))
8237                 goto skip_bad_vf_detection;
8238
8239         pf_func = req_id & 0x01;
8240         if ((pf_func & 1) == (pdev->devfn & 1)) {
8241
8242                 vf = (req_id & 0x7F) >> 1;
8243                 dev_err(pci_dev_to_dev(pdev),
8244                         "VF %d has caused a PCIe error\n", vf);
8245                 dev_err(pci_dev_to_dev(pdev),
8246                         "TLP: dw0: %8.8x\tdw1: %8.8x\tdw2: "
8247                         "%8.8x\tdw3: %8.8x\n",
8248                         dw0, dw1, dw2, dw3);
8249
8250                 /* Find the pci device of the offending VF */
8251                 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL,
8252                                        E1000_DEV_ID_82576_VF, NULL);
8253                 while (vfdev) {
8254                         if (vfdev->devfn == (req_id & 0xFF))
8255                                 break;
8256                         vfdev = pci_get_device(PCI_VENDOR_ID_INTEL,
8257                                                E1000_DEV_ID_82576_VF, vfdev);
8258                 }
8259                 /*
8260                  * There's a slim chance the VF could have been hot plugged,
8261                  * so if it is no longer present we don't need to issue the
8262                  * VFLR.  Just clean up the AER in that case.
8263                  */
8264                 if (vfdev) {
8265                         dev_err(pci_dev_to_dev(pdev),
8266                                 "Issuing VFLR to VF %d\n", vf);
8267                         pci_write_config_dword(vfdev, 0xA8, 0x00008000);
8268                 }
8269
8270                 pci_cleanup_aer_uncorrect_error_status(pdev);
8271         }
8272
8273         /*
8274          * Even though the error may have occurred on the other port
8275          * we still need to increment the vf error reference count for
8276          * both ports because the I/O resume function will be called
8277          * for both of them.
8278          */
8279         adapter->vferr_refcount++;
8280
8281         return PCI_ERS_RESULT_RECOVERED;
8282
8283 skip_bad_vf_detection:
8284 #endif /* CONFIG_PCI_IOV */
8285
8286         netif_device_detach(netdev);
8287
8288         if (state == pci_channel_io_perm_failure)
8289                 return PCI_ERS_RESULT_DISCONNECT;
8290
8291         if (netif_running(netdev))
8292                 igb_down(adapter);
8293         pci_disable_device(pdev);
8294
8295         /* Request a slot slot reset. */
8296         return PCI_ERS_RESULT_NEED_RESET;
8297 }
8298
8299 /**
8300  * igb_io_slot_reset - called after the pci bus has been reset.
8301  * @pdev: Pointer to PCI device
8302  *
8303  * Restart the card from scratch, as if from a cold-boot. Implementation
8304  * resembles the first-half of the igb_resume routine.
8305  */
8306 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
8307 {
8308         struct net_device *netdev = pci_get_drvdata(pdev);
8309         struct igb_adapter *adapter = netdev_priv(netdev);
8310         struct e1000_hw *hw = &adapter->hw;
8311         pci_ers_result_t result;
8312
8313         if (pci_enable_device_mem(pdev)) {
8314                 dev_err(pci_dev_to_dev(pdev),
8315                         "Cannot re-enable PCI device after reset.\n");
8316                 result = PCI_ERS_RESULT_DISCONNECT;
8317         } else {
8318                 pci_set_master(pdev);
8319                 pci_restore_state(pdev);
8320                 pci_save_state(pdev);
8321
8322                 pci_enable_wake(pdev, PCI_D3hot, 0);
8323                 pci_enable_wake(pdev, PCI_D3cold, 0);
8324
8325                 schedule_work(&adapter->reset_task);
8326                 E1000_WRITE_REG(hw, E1000_WUS, ~0);
8327                 result = PCI_ERS_RESULT_RECOVERED;
8328         }
8329
8330         pci_cleanup_aer_uncorrect_error_status(pdev);
8331
8332         return result;
8333 }
8334
8335 /**
8336  * igb_io_resume - called when traffic can start flowing again.
8337  * @pdev: Pointer to PCI device
8338  *
8339  * This callback is called when the error recovery driver tells us that
8340  * its OK to resume normal operation. Implementation resembles the
8341  * second-half of the igb_resume routine.
8342  */
8343 static void igb_io_resume(struct pci_dev *pdev)
8344 {
8345         struct net_device *netdev = pci_get_drvdata(pdev);
8346         struct igb_adapter *adapter = netdev_priv(netdev);
8347
8348         if (adapter->vferr_refcount) {
8349                 dev_info(pci_dev_to_dev(pdev), "Resuming after VF err\n");
8350                 adapter->vferr_refcount--;
8351                 return;
8352         }
8353
8354         if (netif_running(netdev)) {
8355                 if (igb_up(adapter)) {
8356                         dev_err(pci_dev_to_dev(pdev), "igb_up failed after reset\n");
8357                         return;
8358                 }
8359         }
8360
8361         netif_device_attach(netdev);
8362
8363         /* let the f/w know that the h/w is now under the control of the
8364          * driver. */
8365         igb_get_hw_control(adapter);
8366 }
8367
8368 #endif /* HAVE_PCI_ERS */
8369
8370 int igb_add_mac_filter(struct igb_adapter *adapter, u8 *addr, u16 queue)
8371 {
8372         struct e1000_hw *hw = &adapter->hw;
8373         int i;
8374
8375         if (is_zero_ether_addr(addr))
8376                 return 0;
8377
8378         for (i = 0; i < hw->mac.rar_entry_count; i++) {
8379                 if (adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE)
8380                         continue;
8381                 adapter->mac_table[i].state = (IGB_MAC_STATE_MODIFIED |
8382                                                    IGB_MAC_STATE_IN_USE);
8383                 memcpy(adapter->mac_table[i].addr, addr, ETH_ALEN);
8384                 adapter->mac_table[i].queue = queue;
8385                 igb_sync_mac_table(adapter);
8386                 return 0;
8387         }
8388         return -ENOMEM;
8389 }
8390 int igb_del_mac_filter(struct igb_adapter *adapter, u8* addr, u16 queue)
8391 {
8392         /* search table for addr, if found, set to 0 and sync */
8393         int i;
8394         struct e1000_hw *hw = &adapter->hw;
8395
8396         if (is_zero_ether_addr(addr))
8397                 return 0;
8398         for (i = 0; i < hw->mac.rar_entry_count; i++) {
8399                 if (!compare_ether_addr(addr, adapter->mac_table[i].addr) &&
8400                     adapter->mac_table[i].queue == queue) {
8401                         adapter->mac_table[i].state = IGB_MAC_STATE_MODIFIED;
8402                         memset(adapter->mac_table[i].addr, 0, ETH_ALEN);
8403                         adapter->mac_table[i].queue = 0;
8404                         igb_sync_mac_table(adapter);
8405                         return 0;
8406                 }
8407         }
8408         return -ENOMEM;
8409 }
8410 static int igb_set_vf_mac(struct igb_adapter *adapter,
8411                           int vf, unsigned char *mac_addr)
8412 {
8413         igb_del_mac_filter(adapter, adapter->vf_data[vf].vf_mac_addresses, vf);
8414         memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
8415
8416         igb_add_mac_filter(adapter, mac_addr, vf);
8417
8418         return 0;
8419 }
8420
8421 #ifdef IFLA_VF_MAX
8422 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
8423 {
8424         struct igb_adapter *adapter = netdev_priv(netdev);
8425         if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
8426                 return -EINVAL;
8427         adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
8428         dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
8429         dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
8430                                       " change effective.\n");
8431         if (test_bit(__IGB_DOWN, &adapter->state)) {
8432                 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
8433                          " but the PF device is not up.\n");
8434                 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
8435                          " attempting to use the VF device.\n");
8436         }
8437         return igb_set_vf_mac(adapter, vf, mac);
8438 }
8439
8440 static int igb_link_mbps(int internal_link_speed)
8441 {
8442         switch (internal_link_speed) {
8443         case SPEED_100:
8444                 return 100;
8445         case SPEED_1000:
8446                 return 1000;
8447         default:
8448                 return 0;
8449         }
8450 }
8451
8452 static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
8453                         int link_speed)
8454 {
8455         int rf_dec, rf_int;
8456         u32 bcnrc_val;
8457
8458         if (tx_rate != 0) {
8459                 /* Calculate the rate factor values to set */
8460                 rf_int = link_speed / tx_rate;
8461                 rf_dec = (link_speed - (rf_int * tx_rate));
8462                 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
8463
8464                 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
8465                 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
8466                                 E1000_RTTBCNRC_RF_INT_MASK);
8467                 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
8468         } else {
8469                 bcnrc_val = 0;
8470         }
8471
8472         E1000_WRITE_REG(hw, E1000_RTTDQSEL, vf); /* vf X uses queue X */
8473         /*
8474          * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
8475          * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
8476          */
8477         E1000_WRITE_REG(hw, E1000_RTTBCNRM(0), 0x14);
8478         E1000_WRITE_REG(hw, E1000_RTTBCNRC, bcnrc_val);
8479 }
8480
8481 static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
8482 {
8483         int actual_link_speed, i;
8484         bool reset_rate = false;
8485
8486         /* VF TX rate limit was not set */
8487         if ((adapter->vf_rate_link_speed == 0) ||
8488                 (adapter->hw.mac.type != e1000_82576))
8489                 return;
8490
8491         actual_link_speed = igb_link_mbps(adapter->link_speed);
8492         if (actual_link_speed != adapter->vf_rate_link_speed) {
8493                 reset_rate = true;
8494                 adapter->vf_rate_link_speed = 0;
8495                 dev_info(&adapter->pdev->dev,
8496                 "Link speed has been changed. VF Transmit rate is disabled\n");
8497         }
8498
8499         for (i = 0; i < adapter->vfs_allocated_count; i++) {
8500                 if (reset_rate)
8501                         adapter->vf_data[i].tx_rate = 0;
8502
8503                 igb_set_vf_rate_limit(&adapter->hw, i,
8504                         adapter->vf_data[i].tx_rate, actual_link_speed);
8505         }
8506 }
8507
8508 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
8509 {
8510         struct igb_adapter *adapter = netdev_priv(netdev);
8511         struct e1000_hw *hw = &adapter->hw;
8512         int actual_link_speed;
8513
8514         if (hw->mac.type != e1000_82576)
8515                 return -EOPNOTSUPP;
8516
8517         actual_link_speed = igb_link_mbps(adapter->link_speed);
8518         if ((vf >= adapter->vfs_allocated_count) ||
8519                 (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) ||
8520                 (tx_rate < 0) || (tx_rate > actual_link_speed))
8521                 return -EINVAL;
8522
8523         adapter->vf_rate_link_speed = actual_link_speed;
8524         adapter->vf_data[vf].tx_rate = (u16)tx_rate;
8525         igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
8526
8527         return 0;
8528 }
8529
8530 static int igb_ndo_get_vf_config(struct net_device *netdev,
8531                                  int vf, struct ifla_vf_info *ivi)
8532 {
8533         struct igb_adapter *adapter = netdev_priv(netdev);
8534         if (vf >= adapter->vfs_allocated_count)
8535                 return -EINVAL;
8536         ivi->vf = vf;
8537         memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
8538         ivi->tx_rate = adapter->vf_data[vf].tx_rate;
8539         ivi->vlan = adapter->vf_data[vf].pf_vlan;
8540         ivi->qos = adapter->vf_data[vf].pf_qos;
8541         return 0;
8542 }
8543 #endif
8544 static void igb_vmm_control(struct igb_adapter *adapter)
8545 {
8546         struct e1000_hw *hw = &adapter->hw;
8547         u32 reg;
8548
8549         switch (hw->mac.type) {
8550         case e1000_82575:
8551         default:
8552                 /* replication is not supported for 82575 */
8553                 return;
8554         case e1000_82576:
8555                 /* notify HW that the MAC is adding vlan tags */
8556                 reg = E1000_READ_REG(hw, E1000_DTXCTL);
8557                 reg |= (E1000_DTXCTL_VLAN_ADDED |
8558                         E1000_DTXCTL_SPOOF_INT);
8559                 E1000_WRITE_REG(hw, E1000_DTXCTL, reg);
8560         case e1000_82580:
8561                 /* enable replication vlan tag stripping */
8562                 reg = E1000_READ_REG(hw, E1000_RPLOLR);
8563                 reg |= E1000_RPLOLR_STRVLAN;
8564                 E1000_WRITE_REG(hw, E1000_RPLOLR, reg);
8565         case e1000_i350:
8566                 /* none of the above registers are supported by i350 */
8567                 break;
8568         }
8569
8570         /* Enable Malicious Driver Detection */
8571         if ((hw->mac.type == e1000_i350) && (adapter->vfs_allocated_count) &&
8572             (adapter->mdd))
8573                 igb_enable_mdd(adapter);
8574
8575         /* enable replication and loopback support */
8576         e1000_vmdq_set_loopback_pf(hw, adapter->vfs_allocated_count ||
8577                                    adapter->vmdq_pools);
8578
8579         e1000_vmdq_set_anti_spoofing_pf(hw, adapter->vfs_allocated_count ||
8580                                         adapter->vmdq_pools,
8581                                         adapter->vfs_allocated_count);
8582         e1000_vmdq_set_replication_pf(hw, adapter->vfs_allocated_count ||
8583                                       adapter->vmdq_pools);
8584 }
8585
8586 static void igb_init_fw(struct igb_adapter *adapter)
8587 {
8588         struct e1000_fw_drv_info fw_cmd;
8589         struct e1000_hw *hw = &adapter->hw;
8590         int i;
8591         u16 mask;
8592
8593         mask = E1000_SWFW_PHY0_SM;
8594
8595         if (!hw->mac.ops.acquire_swfw_sync(hw, mask)) {
8596                 for (i = 0; i <= FW_MAX_RETRIES; i++) {
8597                         E1000_WRITE_REG(hw, E1000_FWSTS, E1000_FWSTS_FWRI);
8598                         fw_cmd.hdr.cmd = FW_CMD_DRV_INFO;
8599                         fw_cmd.hdr.buf_len = FW_CMD_DRV_INFO_LEN;
8600                         fw_cmd.hdr.cmd_or_resp.cmd_resv = FW_CMD_RESERVED;
8601                         fw_cmd.port_num = hw->bus.func;
8602                         fw_cmd.drv_version = FW_FAMILY_DRV_VER;
8603                         fw_cmd.hdr.checksum = 0;
8604                         fw_cmd.hdr.checksum = e1000_calculate_checksum((u8 *)&fw_cmd,
8605                                                                    (FW_HDR_LEN +
8606                                                                     fw_cmd.hdr.buf_len));
8607                          e1000_host_interface_command(hw, (u8*)&fw_cmd,
8608                                                      sizeof(fw_cmd));
8609                         if (fw_cmd.hdr.cmd_or_resp.ret_status == FW_STATUS_SUCCESS)
8610                                 break;
8611                 }
8612         } else
8613                 dev_warn(pci_dev_to_dev(adapter->pdev),
8614                          "Unable to get semaphore, firmware init failed.\n");
8615         hw->mac.ops.release_swfw_sync(hw, mask);
8616 }
8617
8618 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
8619 {
8620         struct e1000_hw *hw = &adapter->hw;
8621         u32 dmac_thr;
8622         u16 hwm;
8623
8624         if (hw->mac.type > e1000_82580) {
8625                 if (adapter->dmac != IGB_DMAC_DISABLE) {
8626                         u32 reg;
8627
8628                         /* force threshold to 0.  */
8629                         E1000_WRITE_REG(hw, E1000_DMCTXTH, 0);
8630
8631                         /*
8632                          * DMA Coalescing high water mark needs to be greater
8633                          * than the Rx threshold. Set hwm to PBA - max frame
8634                          * size in 16B units, capping it at PBA - 6KB.
8635                          */
8636                         hwm = 64 * pba - adapter->max_frame_size / 16;
8637                         if (hwm < 64 * (pba - 6))
8638                                 hwm = 64 * (pba - 6);
8639                         reg = E1000_READ_REG(hw, E1000_FCRTC);
8640                         reg &= ~E1000_FCRTC_RTH_COAL_MASK;
8641                         reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
8642                                 & E1000_FCRTC_RTH_COAL_MASK);
8643                         E1000_WRITE_REG(hw, E1000_FCRTC, reg);
8644
8645                         /*
8646                          * Set the DMA Coalescing Rx threshold to PBA - 2 * max
8647                          * frame size, capping it at PBA - 10KB.
8648                          */
8649                         dmac_thr = pba - adapter->max_frame_size / 512;
8650                         if (dmac_thr < pba - 10)
8651                                 dmac_thr = pba - 10;
8652                         reg = E1000_READ_REG(hw, E1000_DMACR);
8653                         reg &= ~E1000_DMACR_DMACTHR_MASK;
8654                         reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
8655                                 & E1000_DMACR_DMACTHR_MASK);
8656
8657                         /* transition to L0x or L1 if available..*/
8658                         reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
8659
8660                         /* watchdog timer= msec values in 32usec intervals */
8661                         reg |= ((adapter->dmac) >> 5);
8662                         E1000_WRITE_REG(hw, E1000_DMACR, reg);
8663
8664                         /* no lower threshold to disable coalescing(smart fifb)-UTRESH=0*/
8665                         E1000_WRITE_REG(hw, E1000_DMCRTRH, 0);
8666
8667                         /*
8668                          * This sets the time to wait before requesting transition to
8669                          * low power state to number of usecs needed to receive 1 512
8670                          * byte frame at gigabit line rate
8671                          */
8672                         reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
8673
8674                         E1000_WRITE_REG(hw, E1000_DMCTLX, reg);
8675
8676                         /* free space in tx packet buffer to wake from DMA coal */
8677                         E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
8678                                 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
8679
8680                         /* make low power state decision controlled by DMA coal */
8681                         reg = E1000_READ_REG(hw, E1000_PCIEMISC);
8682                         reg &= ~E1000_PCIEMISC_LX_DECISION;
8683                         E1000_WRITE_REG(hw, E1000_PCIEMISC, reg);
8684                 } /* endif adapter->dmac is not disabled */
8685         } else if (hw->mac.type == e1000_82580) {
8686                 u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
8687                 E1000_WRITE_REG(hw, E1000_PCIEMISC,
8688                                 reg & ~E1000_PCIEMISC_LX_DECISION);
8689                 E1000_WRITE_REG(hw, E1000_DMACR, 0);
8690         }
8691 }
8692
8693 /* igb_main.c */
8694
8695
8696 /**
8697  * igb_probe - Device Initialization Routine
8698  * @pdev: PCI device information struct
8699  * @ent: entry in igb_pci_tbl
8700  *
8701  * Returns 0 on success, negative on failure
8702  *
8703  * igb_probe initializes an adapter identified by a pci_dev structure.
8704  * The OS initialization, configuring of the adapter private structure,
8705  * and a hardware reset occur.
8706  **/
8707 int igb_kni_probe(struct pci_dev *pdev,
8708                                struct net_device **lad_dev)
8709 {
8710         struct net_device *netdev;
8711         struct igb_adapter *adapter;
8712         struct e1000_hw *hw;
8713         u16 eeprom_data = 0;
8714         u8 pba_str[E1000_PBANUM_LENGTH];
8715         s32 ret_val;
8716         static int global_quad_port_a; /* global quad port a indication */
8717         int i, err, pci_using_dac = 0;
8718         static int cards_found;
8719
8720         err = pci_enable_device_mem(pdev);
8721         if (err)
8722                 return err;
8723
8724 #ifdef NO_KNI
8725         pci_using_dac = 0;
8726         err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
8727         if (!err) {
8728                 err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
8729                 if (!err)
8730                         pci_using_dac = 1;
8731         } else {
8732                 err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
8733                 if (err) {
8734                         err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
8735                         if (err) {
8736                                 IGB_ERR("No usable DMA configuration, "
8737                                         "aborting\n");
8738                                 goto err_dma;
8739                         }
8740                 }
8741         }
8742
8743 #ifndef HAVE_ASPM_QUIRKS
8744         /* 82575 requires that the pci-e link partner disable the L0s state */
8745         switch (pdev->device) {
8746         case E1000_DEV_ID_82575EB_COPPER:
8747         case E1000_DEV_ID_82575EB_FIBER_SERDES:
8748         case E1000_DEV_ID_82575GB_QUAD_COPPER:
8749                 pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S);
8750         default:
8751                 break;
8752         }
8753
8754 #endif /* HAVE_ASPM_QUIRKS */
8755         err = pci_request_selected_regions(pdev,
8756                                            pci_select_bars(pdev,
8757                                                            IORESOURCE_MEM),
8758                                            igb_driver_name);
8759         if (err)
8760                 goto err_pci_reg;
8761
8762         pci_enable_pcie_error_reporting(pdev);
8763
8764         pci_set_master(pdev);
8765 #endif /* NO_KNI */
8766         err = -ENOMEM;
8767 #ifdef HAVE_TX_MQ
8768         netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
8769                                    IGB_MAX_TX_QUEUES);
8770 #else
8771         netdev = alloc_etherdev(sizeof(struct igb_adapter));
8772 #endif /* HAVE_TX_MQ */
8773         if (!netdev)
8774                 goto err_alloc_etherdev;
8775
8776
8777         SET_MODULE_OWNER(netdev);
8778         SET_NETDEV_DEV(netdev, &pdev->dev);
8779
8780         //pci_set_drvdata(pdev, netdev);
8781
8782         adapter = netdev_priv(netdev);
8783         adapter->netdev = netdev;
8784         adapter->pdev = pdev;
8785         hw = &adapter->hw;
8786         hw->back = adapter;
8787         adapter->port_num = hw->bus.func;
8788         adapter->msg_enable = (1 << debug) - 1;
8789
8790 #ifdef HAVE_PCI_ERS
8791         err = pci_save_state(pdev);
8792         if (err)
8793                 goto err_ioremap;
8794 #endif
8795         err = -EIO;
8796         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
8797                               pci_resource_len(pdev, 0));
8798         if (!hw->hw_addr)
8799                 goto err_ioremap;
8800
8801 #ifdef HAVE_NET_DEVICE_OPS
8802         netdev->netdev_ops = &igb_netdev_ops;
8803 #else /* HAVE_NET_DEVICE_OPS */
8804         netdev->open = &igb_open;
8805         netdev->stop = &igb_close;
8806         netdev->get_stats = &igb_get_stats;
8807 #ifdef HAVE_SET_RX_MODE
8808         netdev->set_rx_mode = &igb_set_rx_mode;
8809 #endif
8810         netdev->set_multicast_list = &igb_set_rx_mode;
8811         netdev->set_mac_address = &igb_set_mac;
8812         netdev->change_mtu = &igb_change_mtu;
8813         netdev->do_ioctl = &igb_ioctl;
8814 #ifdef HAVE_TX_TIMEOUT
8815         netdev->tx_timeout = &igb_tx_timeout;
8816 #endif
8817         netdev->vlan_rx_register = igb_vlan_mode;
8818         netdev->vlan_rx_add_vid = igb_vlan_rx_add_vid;
8819         netdev->vlan_rx_kill_vid = igb_vlan_rx_kill_vid;
8820 #ifdef CONFIG_NET_POLL_CONTROLLER
8821         netdev->poll_controller = igb_netpoll;
8822 #endif
8823         netdev->hard_start_xmit = &igb_xmit_frame;
8824 #endif /* HAVE_NET_DEVICE_OPS */
8825         igb_set_ethtool_ops(netdev);
8826 #ifdef HAVE_TX_TIMEOUT
8827         netdev->watchdog_timeo = 5 * HZ;
8828 #endif
8829
8830         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
8831
8832         adapter->bd_number = cards_found;
8833
8834         /* setup the private structure */
8835         err = igb_sw_init(adapter);
8836         if (err)
8837                 goto err_sw_init;
8838
8839         e1000_get_bus_info(hw);
8840
8841         hw->phy.autoneg_wait_to_complete = FALSE;
8842         hw->mac.adaptive_ifs = FALSE;
8843
8844         /* Copper options */
8845         if (hw->phy.media_type == e1000_media_type_copper) {
8846 #ifdef ETH_TP_MDI_X
8847                 hw->phy.mdix = ETH_TP_MDI_INVALID;
8848 #else
8849                 hw->phy.mdix = AUTO_ALL_MODES;
8850 #endif /* ETH_TP_MDI_X */
8851                 hw->phy.disable_polarity_correction = FALSE;
8852                 hw->phy.ms_type = e1000_ms_hw_default;
8853         }
8854
8855         if (e1000_check_reset_block(hw))
8856                 dev_info(pci_dev_to_dev(pdev),
8857                         "PHY reset is blocked due to SOL/IDER session.\n");
8858
8859         /*
8860          * features is initialized to 0 in allocation, it might have bits
8861          * set by igb_sw_init so we should use an or instead of an
8862          * assignment.
8863          */
8864         netdev->features |= NETIF_F_SG |
8865                             NETIF_F_IP_CSUM |
8866 #ifdef NETIF_F_IPV6_CSUM
8867                             NETIF_F_IPV6_CSUM |
8868 #endif
8869 #ifdef NETIF_F_TSO
8870                             NETIF_F_TSO |
8871 #ifdef NETIF_F_TSO6
8872                             NETIF_F_TSO6 |
8873 #endif
8874 #endif /* NETIF_F_TSO */
8875 #ifdef NETIF_F_RXHASH
8876                             NETIF_F_RXHASH |
8877 #endif
8878 #ifdef HAVE_NDO_SET_FEATURES
8879                             NETIF_F_RXCSUM |
8880 #endif
8881                             NETIF_F_HW_VLAN_RX |
8882                             NETIF_F_HW_VLAN_TX;
8883
8884 #ifdef HAVE_NDO_SET_FEATURES
8885         /* copy netdev features into list of user selectable features */
8886         netdev->hw_features |= netdev->features;
8887 #ifndef IGB_NO_LRO
8888
8889         /* give us the option of enabling LRO later */
8890         netdev->hw_features |= NETIF_F_LRO;
8891 #endif
8892 #else
8893 #ifdef NETIF_F_GRO
8894
8895         /* this is only needed on kernels prior to 2.6.39 */
8896         netdev->features |= NETIF_F_GRO;
8897 #endif
8898 #endif
8899
8900         /* set this bit last since it cannot be part of hw_features */
8901         netdev->features |= NETIF_F_HW_VLAN_FILTER;
8902
8903 #ifdef HAVE_NETDEV_VLAN_FEATURES
8904         netdev->vlan_features |= NETIF_F_TSO |
8905                                  NETIF_F_TSO6 |
8906                                  NETIF_F_IP_CSUM |
8907                                  NETIF_F_IPV6_CSUM |
8908                                  NETIF_F_SG;
8909
8910 #endif
8911         if (pci_using_dac)
8912                 netdev->features |= NETIF_F_HIGHDMA;
8913
8914         if (hw->mac.type >= e1000_82576)
8915                 netdev->features |= NETIF_F_SCTP_CSUM;
8916
8917 #ifdef NO_KNI
8918         adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
8919
8920         /* before reading the NVM, reset the controller to put the device in a
8921          * known good starting state */
8922         e1000_reset_hw(hw);
8923 #endif
8924
8925         /* make sure the NVM is good */
8926         if (e1000_validate_nvm_checksum(hw) < 0) {
8927                 dev_err(pci_dev_to_dev(pdev), "The NVM Checksum Is Not"
8928                         " Valid\n");
8929                 err = -EIO;
8930                 goto err_eeprom;
8931         }
8932
8933         /* copy the MAC address out of the NVM */
8934         if (e1000_read_mac_addr(hw))
8935                 dev_err(pci_dev_to_dev(pdev), "NVM Read Error\n");
8936         memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
8937 #ifdef ETHTOOL_GPERMADDR
8938         memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
8939
8940         if (!is_valid_ether_addr(netdev->perm_addr)) {
8941 #else
8942         if (!is_valid_ether_addr(netdev->dev_addr)) {
8943 #endif
8944                 dev_err(pci_dev_to_dev(pdev), "Invalid MAC Address\n");
8945                 err = -EIO;
8946                 goto err_eeprom;
8947         }
8948
8949         memcpy(&adapter->mac_table[0].addr, hw->mac.addr, netdev->addr_len);
8950         adapter->mac_table[0].queue = adapter->vfs_allocated_count;
8951         adapter->mac_table[0].state = (IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE);
8952         igb_rar_set(adapter, 0);
8953
8954         /* get firmware version for ethtool -i */
8955         e1000_read_nvm(&adapter->hw, 5, 1, &adapter->fw_version);
8956 #ifdef NO_KNI
8957         setup_timer(&adapter->watchdog_timer, &igb_watchdog,
8958                     (unsigned long) adapter);
8959         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
8960                 setup_timer(&adapter->dma_err_timer, &igb_dma_err_timer,
8961                             (unsigned long) adapter);
8962         setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
8963                     (unsigned long) adapter);
8964
8965         INIT_WORK(&adapter->reset_task, igb_reset_task);
8966         INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
8967         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
8968                 INIT_WORK(&adapter->dma_err_task, igb_dma_err_task);
8969 #endif
8970
8971         /* Initialize link properties that are user-changeable */
8972         adapter->fc_autoneg = true;
8973         hw->mac.autoneg = true;
8974         hw->phy.autoneg_advertised = 0x2f;
8975
8976         hw->fc.requested_mode = e1000_fc_default;
8977         hw->fc.current_mode = e1000_fc_default;
8978
8979         e1000_validate_mdi_setting(hw);
8980
8981         /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
8982          * enable the ACPI Magic Packet filter
8983          */
8984
8985         if (hw->bus.func == 0)
8986                 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
8987         else if (hw->mac.type >= e1000_82580)
8988                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
8989                                  NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
8990                                  &eeprom_data);
8991         else if (hw->bus.func == 1)
8992                 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
8993
8994         if (eeprom_data & IGB_EEPROM_APME)
8995                 adapter->eeprom_wol |= E1000_WUFC_MAG;
8996
8997         /* now that we have the eeprom settings, apply the special cases where
8998          * the eeprom may be wrong or the board simply won't support wake on
8999          * lan on a particular port */
9000         switch (pdev->device) {
9001         case E1000_DEV_ID_82575GB_QUAD_COPPER:
9002                 adapter->eeprom_wol = 0;
9003                 break;
9004         case E1000_DEV_ID_82575EB_FIBER_SERDES:
9005         case E1000_DEV_ID_82576_FIBER:
9006         case E1000_DEV_ID_82576_SERDES:
9007                 /* Wake events only supported on port A for dual fiber
9008                  * regardless of eeprom setting */
9009                 if (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_FUNC_1)
9010                         adapter->eeprom_wol = 0;
9011                 break;
9012         case E1000_DEV_ID_82576_QUAD_COPPER:
9013         case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
9014                 /* if quad port adapter, disable WoL on all but port A */
9015                 if (global_quad_port_a != 0)
9016                         adapter->eeprom_wol = 0;
9017                 else
9018                         adapter->flags |= IGB_FLAG_QUAD_PORT_A;
9019                 /* Reset for multiple quad port adapters */
9020                 if (++global_quad_port_a == 4)
9021                         global_quad_port_a = 0;
9022                 break;
9023         }
9024
9025         /* initialize the wol settings based on the eeprom settings */
9026         adapter->wol = adapter->eeprom_wol;
9027 #ifdef NO_KNI
9028         device_set_wakeup_enable(pci_dev_to_dev(adapter->pdev), adapter->wol);
9029
9030         /* reset the hardware with the new settings */
9031         igb_reset(adapter);
9032
9033         /* let the f/w know that the h/w is now under the control of the
9034          * driver. */
9035         igb_get_hw_control(adapter);
9036
9037         strncpy(netdev->name, "eth%d", IFNAMSIZ);
9038         err = register_netdev(netdev);
9039         if (err)
9040                 goto err_register;
9041
9042 #ifdef CONFIG_IGB_VMDQ_NETDEV
9043         err = igb_init_vmdq_netdevs(adapter);
9044         if (err)
9045                 goto err_register;
9046 #endif
9047         /* carrier off reporting is important to ethtool even BEFORE open */
9048         netif_carrier_off(netdev);
9049
9050 #ifdef IGB_DCA
9051         if (dca_add_requester(&pdev->dev) == E1000_SUCCESS) {
9052                 adapter->flags |= IGB_FLAG_DCA_ENABLED;
9053                 dev_info(pci_dev_to_dev(pdev), "DCA enabled\n");
9054                 igb_setup_dca(adapter);
9055         }
9056
9057 #endif
9058 #ifdef HAVE_HW_TIME_STAMP
9059         /* do hw tstamp init after resetting */
9060         igb_init_hw_timer(adapter);
9061
9062 #endif
9063
9064 #endif /* NO_KNI */
9065         dev_info(pci_dev_to_dev(pdev), "Intel(R) Gigabit Ethernet Network Connection\n");
9066         /* print bus type/speed/width info */
9067         dev_info(pci_dev_to_dev(pdev), "%s: (PCIe:%s:%s) ",
9068                  netdev->name,
9069                  ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5GT/s" :
9070                   (hw->bus.speed == e1000_bus_speed_5000) ? "5.0GT/s" :
9071                                                             "unknown"),
9072                  ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4\n" :
9073                   (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2\n" :
9074                   (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1\n" :
9075                    "unknown"));
9076         dev_info(pci_dev_to_dev(pdev), "%s: MAC: ", netdev->name);
9077         for (i = 0; i < 6; i++)
9078                 printk("%2.2x%c", netdev->dev_addr[i], i == 5 ? '\n' : ':');
9079
9080         ret_val = e1000_read_pba_string(hw, pba_str, E1000_PBANUM_LENGTH);
9081         if (ret_val)
9082                 strncpy(pba_str, "Unknown", sizeof(pba_str) - 1);
9083         dev_info(pci_dev_to_dev(pdev), "%s: PBA No: %s\n", netdev->name,
9084                  pba_str);
9085
9086         /* Initialize the thermal sensor on i350 devices. */
9087         if (hw->mac.type == e1000_i350 && hw->bus.func == 0) {
9088                 u16 ets_word;
9089
9090                 /*
9091                  * Read the NVM to determine if this i350 device supports an
9092                  * external thermal sensor.
9093                  */
9094                 e1000_read_nvm(hw, NVM_ETS_CFG, 1, &ets_word);
9095                 if (ets_word != 0x0000 && ets_word != 0xFFFF)
9096                         adapter->ets = true;
9097                 else
9098                         adapter->ets = false;
9099 #ifdef NO_KNI
9100 #ifdef IGB_SYSFS
9101                 igb_sysfs_init(adapter);
9102 #else
9103 #ifdef IGB_PROCFS
9104                 igb_procfs_init(adapter);
9105 #endif /* IGB_PROCFS */
9106 #endif /* IGB_SYSFS */
9107 #endif /* NO_KNI */
9108         } else {
9109                 adapter->ets = false;
9110         }
9111
9112         switch (hw->mac.type) {
9113         case e1000_i350:
9114                 /* Enable EEE for internal copper PHY devices */
9115                 if (hw->phy.media_type == e1000_media_type_copper)
9116                         e1000_set_eee_i350(hw);
9117
9118                 /* send driver version info to firmware */
9119                 igb_init_fw(adapter);
9120                 break;
9121         default:
9122                 break;
9123         }
9124 #ifndef IGB_NO_LRO
9125         if (netdev->features & NETIF_F_LRO)
9126                 dev_info(pci_dev_to_dev(pdev), "Internal LRO is enabled \n");
9127         else
9128                 dev_info(pci_dev_to_dev(pdev), "LRO is disabled \n");
9129 #endif
9130         dev_info(pci_dev_to_dev(pdev),
9131                  "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
9132                  adapter->msix_entries ? "MSI-X" :
9133                  (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9134                  adapter->num_rx_queues, adapter->num_tx_queues);
9135
9136         cards_found++;
9137         *lad_dev = netdev;
9138
9139         pm_runtime_put_noidle(&pdev->dev);
9140         return 0;
9141
9142 //err_register:
9143         //igb_release_hw_control(adapter);
9144 err_eeprom:
9145         //if (!e1000_check_reset_block(hw))
9146         //      e1000_phy_hw_reset(hw);
9147
9148         if (hw->flash_address)
9149                 iounmap(hw->flash_address);
9150 err_sw_init:
9151         //igb_clear_interrupt_scheme(adapter);
9152         //igb_reset_sriov_capability(adapter);
9153         iounmap(hw->hw_addr);
9154 err_ioremap:
9155         free_netdev(netdev);
9156 err_alloc_etherdev:
9157         //pci_release_selected_regions(pdev,
9158         //                             pci_select_bars(pdev, IORESOURCE_MEM));
9159 //err_pci_reg:
9160 //err_dma:
9161         pci_disable_device(pdev);
9162         return err;
9163 }
9164
9165
9166 void igb_kni_remove(struct pci_dev *pdev)
9167 {
9168         pci_disable_device(pdev);
9169 }
9170