lib/librte_eal/linuxapp/kni/ethtool/igb/igb_main.c

   1 /*******************************************************************************
   2
   3   Intel(R) Gigabit Ethernet Linux driver
   4   Copyright(c) 2007-2012 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28 #include <linux/module.h>
  29 #include <linux/types.h>
  30 #include <linux/init.h>
  31 #include <linux/vmalloc.h>
  32 #include <linux/pagemap.h>
  33 #include <linux/netdevice.h>
  34 #include <linux/tcp.h>
  35 #ifdef NETIF_F_TSO
  36 #include <net/checksum.h>
  37 #ifdef NETIF_F_TSO6
  38 #include <linux/ipv6.h>
  39 #include <net/ip6_checksum.h>
  40 #endif
  41 #endif
  42 #ifdef SIOCGMIIPHY
  43 #include <linux/mii.h>
  44 #endif
  45 #ifdef SIOCETHTOOL
  46 #include <linux/ethtool.h>
  47 #endif
  48 #include <linux/if_vlan.h>
  49 #ifdef CONFIG_PM_RUNTIME
  50 #include <linux/pm_runtime.h>
  51 #endif /* CONFIG_PM_RUNTIME */
  52
  53 #include "igb.h"
  54 #include "igb_vmdq.h"
  55
  56 #include <linux/uio_driver.h>
  57
  58 #define DRV_DEBUG
  59 #define DRV_HW_PERF
  60 #define VERSION_SUFFIX
  61
  62 #define MAJ 3
  63 #define MIN 4
  64 #define BUILD 8
  65 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." __stringify(BUILD) VERSION_SUFFIX DRV_DEBUG DRV_HW_PERF
  66
  67 char igb_driver_name[] = "igb";
  68 char igb_driver_version[] = DRV_VERSION;
  69 static const char igb_driver_string[] =
  70                                 "Intel(R) Gigabit Ethernet Network Driver";
  71 static const char igb_copyright[] = "Copyright (c) 2007-2011 Intel Corporation.";
  72
  73 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
  74         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER) },
  75         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER) },
  76         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES) },
  77         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII) },
  78         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) },
  79         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) },
  80         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) },
  81         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER) },
  82         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER) },
  83         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER) },
  84         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES) },
  85         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII) },
  86         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL) },
  87         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII) },
  88         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES) },
  89         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE) },
  90         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP) },
  91         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576) },
  92         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS) },
  93         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES) },
  94         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER) },
  95         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES) },
  96         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD) },
  97         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2) },
  98         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER) },
  99         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER) },
 100         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES) },
 101         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER) },
 102         /* required last entry */
 103         {0, }
 104 };
 105
 106 //MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
 107 static void igb_set_sriov_capability(struct igb_adapter *adapter) __attribute__((__unused__));
 108 void igb_reset(struct igb_adapter *);
 109 static int igb_setup_all_tx_resources(struct igb_adapter *);
 110 static int igb_setup_all_rx_resources(struct igb_adapter *);
 111 static void igb_free_all_tx_resources(struct igb_adapter *);
 112 static void igb_free_all_rx_resources(struct igb_adapter *);
 113 static void igb_setup_mrqc(struct igb_adapter *);
 114 void igb_update_stats(struct igb_adapter *);
 115 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
 116 static void __devexit igb_remove(struct pci_dev *pdev);
 117 #ifdef HAVE_HW_TIME_STAMP
 118 static void igb_init_hw_timer(struct igb_adapter *adapter);
 119 #endif
 120 static int igb_sw_init(struct igb_adapter *);
 121 static int igb_open(struct net_device *);
 122 static int igb_close(struct net_device *);
 123 static void igb_configure_tx(struct igb_adapter *);
 124 static void igb_configure_rx(struct igb_adapter *);
 125 static void igb_clean_all_tx_rings(struct igb_adapter *);
 126 static void igb_clean_all_rx_rings(struct igb_adapter *);
 127 static void igb_clean_tx_ring(struct igb_ring *);
 128 static void igb_set_rx_mode(struct net_device *);
 129 static void igb_update_phy_info(unsigned long);
 130 static void igb_watchdog(unsigned long);
 131 static void igb_watchdog_task(struct work_struct *);
 132 static void igb_dma_err_task(struct work_struct *);
 133 static void igb_dma_err_timer(unsigned long data);
 134 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
 135 static struct net_device_stats *igb_get_stats(struct net_device *);
 136 static int igb_change_mtu(struct net_device *, int);
 137 void igb_full_sync_mac_table(struct igb_adapter *adapter);
 138 static int igb_set_mac(struct net_device *, void *);
 139 static void igb_set_uta(struct igb_adapter *adapter);
 140 static irqreturn_t igb_intr(int irq, void *);
 141 static irqreturn_t igb_intr_msi(int irq, void *);
 142 static irqreturn_t igb_msix_other(int irq, void *);
 143 static irqreturn_t igb_msix_ring(int irq, void *);
 144 #ifdef IGB_DCA
 145 static void igb_update_dca(struct igb_q_vector *);
 146 static void igb_setup_dca(struct igb_adapter *);
 147 #endif /* IGB_DCA */
 148 static int igb_poll(struct napi_struct *, int);
 149 static bool igb_clean_tx_irq(struct igb_q_vector *);
 150 static bool igb_clean_rx_irq(struct igb_q_vector *, int);
 151 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
 152 static void igb_tx_timeout(struct net_device *);
 153 static void igb_reset_task(struct work_struct *);
 154 #ifdef HAVE_VLAN_RX_REGISTER
 155 static void igb_vlan_mode(struct net_device *, struct vlan_group *);
 156 #endif
 157 #ifdef HAVE_VLAN_PROTOCOL
 158 static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16);
 159 static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16);
 160 #elif defined HAVE_INT_NDO_VLAN_RX_ADD_VID
 161 static int igb_vlan_rx_add_vid(struct net_device *, u16);
 162 static int igb_vlan_rx_kill_vid(struct net_device *, u16);
 163 #else
 164 static void igb_vlan_rx_add_vid(struct net_device *, u16);
 165 static void igb_vlan_rx_kill_vid(struct net_device *, u16);
 166 #endif
 167 static void igb_restore_vlan(struct igb_adapter *);
 168 void igb_rar_set(struct igb_adapter *adapter, u32 index);
 169 static void igb_ping_all_vfs(struct igb_adapter *);
 170 static void igb_msg_task(struct igb_adapter *);
 171 static void igb_vmm_control(struct igb_adapter *);
 172 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
 173 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
 174 static void igb_process_mdd_event(struct igb_adapter *);
 175 #ifdef IFLA_VF_MAX
 176 static int igb_ndo_set_vf_mac( struct net_device *netdev, int vf, u8 *mac);
 177 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
 178                                int vf, u16 vlan, u8 qos);
 179 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
 180 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
 181                                  struct ifla_vf_info *ivi);
 182 static void igb_check_vf_rate_limit(struct igb_adapter *);
 183 #endif
 184 static int igb_vf_configure(struct igb_adapter *adapter, int vf);
 185 static int igb_check_vf_assignment(struct igb_adapter *adapter);
 186 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
 187 static int igb_find_enabled_vfs(struct igb_adapter *adapter);
 188 #endif
 189 #ifdef CONFIG_PM
 190 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
 191 static int igb_suspend(struct device *dev);
 192 static int igb_resume(struct device *dev);
 193 #ifdef CONFIG_PM_RUNTIME
 194 static int igb_runtime_suspend(struct device *dev);
 195 static int igb_runtime_resume(struct device *dev);
 196 static int igb_runtime_idle(struct device *dev);
 197 #endif /* CONFIG_PM_RUNTIME */
 198 static const struct dev_pm_ops igb_pm_ops = {
 199 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,34)
 200         .suspend = igb_suspend,
 201         .resume = igb_resume,
 202         .freeze = igb_suspend,
 203         .thaw = igb_resume,
 204         .poweroff = igb_suspend,
 205         .restore = igb_resume,
 206 #ifdef CONFIG_PM_RUNTIME
 207         .runtime_suspend = igb_runtime_suspend,
 208         .runtime_resume = igb_runtime_resume,
 209         .runtime_idle = igb_runtime_idle,
 210 #endif
 211 #else /* Linux >= 2.6.34 */
 212         SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
 213 #ifdef CONFIG_PM_RUNTIME
 214         SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
 215                         igb_runtime_idle)
 216 #endif /* CONFIG_PM_RUNTIME */
 217 #endif /* Linux version */
 218 };
 219 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
 220 #endif /* CONFIG_PM */
 221 #ifndef USE_REBOOT_NOTIFIER
 222 static void igb_shutdown(struct pci_dev *);
 223 #else
 224 static int igb_notify_reboot(struct notifier_block *, unsigned long, void *);
 225 static struct notifier_block igb_notifier_reboot = {
 226         .notifier_call  = igb_notify_reboot,
 227         .next           = NULL,
 228         .priority       = 0
 229 };
 230 #endif
 231 #ifdef IGB_DCA
 232 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
 233 static struct notifier_block dca_notifier = {
 234         .notifier_call  = igb_notify_dca,
 235         .next           = NULL,
 236         .priority       = 0
 237 };
 238 #endif
 239 #ifdef CONFIG_NET_POLL_CONTROLLER
 240 /* for netdump / net console */
 241 static void igb_netpoll(struct net_device *);
 242 #endif
 243
 244 #ifdef HAVE_PCI_ERS
 245 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
 246                      pci_channel_state_t);
 247 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
 248 static void igb_io_resume(struct pci_dev *);
 249
 250 static struct pci_error_handlers igb_err_handler = {
 251         .error_detected = igb_io_error_detected,
 252         .slot_reset = igb_io_slot_reset,
 253         .resume = igb_io_resume,
 254 };
 255 #endif
 256
 257 static void igb_init_fw(struct igb_adapter *adapter);
 258 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
 259
 260 static struct pci_driver igb_driver = {
 261         .name     = igb_driver_name,
 262         .id_table = igb_pci_tbl,
 263         .probe    = igb_probe,
 264         .remove   = __devexit_p(igb_remove),
 265 #ifdef CONFIG_PM
 266 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
 267         .driver.pm = &igb_pm_ops,
 268 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
 269 #endif /* CONFIG_PM */
 270 #ifndef USE_REBOOT_NOTIFIER
 271         .shutdown = igb_shutdown,
 272 #endif
 273 #ifdef HAVE_PCI_ERS
 274         .err_handler = &igb_err_handler
 275 #endif
 276 };
 277
 278 //MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 279 //MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
 280 //MODULE_LICENSE("GPL");
 281 //MODULE_VERSION(DRV_VERSION);
 282
 283 static void igb_vfta_set(struct igb_adapter *adapter, u32 vid, bool add)
 284 {
 285         struct e1000_hw *hw = &adapter->hw;
 286         struct e1000_host_mng_dhcp_cookie *mng_cookie = &hw->mng_cookie;
 287         u32 index = (vid >> E1000_VFTA_ENTRY_SHIFT) & E1000_VFTA_ENTRY_MASK;
 288         u32 mask = 1 << (vid & E1000_VFTA_ENTRY_BIT_SHIFT_MASK);
 289         u32 vfta;
 290
 291         /*
 292          * if this is the management vlan the only option is to add it in so
 293          * that the management pass through will continue to work
 294          */
 295         if ((mng_cookie->status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
 296             (vid == mng_cookie->vlan_id))
 297                 add = TRUE;
 298
 299         vfta = adapter->shadow_vfta[index];
 300
 301         if (add)
 302                 vfta |= mask;
 303         else
 304                 vfta &= ~mask;
 305
 306         e1000_write_vfta(hw, index, vfta);
 307         adapter->shadow_vfta[index] = vfta;
 308 }
 309
 310 #ifdef HAVE_HW_TIME_STAMP
 311 /**
 312  * igb_read_clock - read raw cycle counter (to be used by time counter)
 313  */
 314 static cycle_t igb_read_clock(const struct cyclecounter *tc)
 315 {
 316         struct igb_adapter *adapter =
 317                 container_of(tc, struct igb_adapter, cycles);
 318         struct e1000_hw *hw = &adapter->hw;
 319         u64 stamp = 0;
 320         int shift = 0;
 321
 322         /*
 323          * The timestamp latches on lowest register read. For the 82580
 324          * the lowest register is SYSTIMR instead of SYSTIML.  However we never
 325          * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
 326          */
 327         if (hw->mac.type >= e1000_82580) {
 328                 stamp = E1000_READ_REG(hw, E1000_SYSTIMR) >> 8;
 329                 shift = IGB_82580_TSYNC_SHIFT;
 330         }
 331
 332         stamp |= (u64)E1000_READ_REG(hw, E1000_SYSTIML) << shift;
 333         stamp |= (u64)E1000_READ_REG(hw, E1000_SYSTIMH) << (shift + 32);
 334         return stamp;
 335 }
 336
 337 #endif /* SIOCSHWTSTAMP */
 338 static int debug = NETIF_MSG_DRV | NETIF_MSG_PROBE;
 339 //module_param(debug, int, 0);
 340 //MODULE_PARM_DESC(debug, "Debug level (0=none, ..., 16=all)");
 341
 342 /**
 343  * igb_init_module - Driver Registration Routine
 344  *
 345  * igb_init_module is the first routine called when the driver is
 346  * loaded. All it does is register with the PCI subsystem.
 347  **/
 348 static int __init igb_init_module(void)
 349 {
 350         int ret;
 351
 352         printk(KERN_INFO "%s - version %s\n",
 353                igb_driver_string, igb_driver_version);
 354
 355         printk(KERN_INFO "%s\n", igb_copyright);
 356 #ifdef IGB_SYSFS
 357 /* only use IGB_PROCFS if IGB_SYSFS is not defined */
 358 #else
 359 #ifdef IGB_PROCFS
 360         if (igb_procfs_topdir_init())
 361                 printk(KERN_INFO "Procfs failed to initialize topdir\n");
 362 #endif /* IGB_PROCFS */
 363 #endif /* IGB_SYSFS  */
 364
 365 #ifdef IGB_DCA
 366         dca_register_notify(&dca_notifier);
 367 #endif
 368         ret = pci_register_driver(&igb_driver);
 369 #ifdef USE_REBOOT_NOTIFIER
 370         if (ret >= 0) {
 371                 register_reboot_notifier(&igb_notifier_reboot);
 372         }
 373 #endif
 374         return ret;
 375 }
 376
 377 #undef module_init
 378 #define module_init(x) static int x(void)  __attribute__((__unused__));
 379 module_init(igb_init_module);
 380
 381 /**
 382  * igb_exit_module - Driver Exit Cleanup Routine
 383  *
 384  * igb_exit_module is called just before the driver is removed
 385  * from memory.
 386  **/
 387 static void __exit igb_exit_module(void)
 388 {
 389 #ifdef IGB_DCA
 390         dca_unregister_notify(&dca_notifier);
 391 #endif
 392 #ifdef USE_REBOOT_NOTIFIER
 393         unregister_reboot_notifier(&igb_notifier_reboot);
 394 #endif
 395         pci_unregister_driver(&igb_driver);
 396
 397 #ifdef IGB_SYSFS
 398 /* only compile IGB_PROCFS if IGB_SYSFS is not defined */
 399 #else
 400 #ifdef IGB_PROCFS
 401         igb_procfs_topdir_exit();
 402 #endif /* IGB_PROCFS */
 403 #endif /* IGB_SYSFS */
 404 }
 405
 406 #undef module_exit
 407 #define module_exit(x) static void x(void)  __attribute__((__unused__));
 408 module_exit(igb_exit_module);
 409
 410 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
 411 /**
 412  * igb_cache_ring_register - Descriptor ring to register mapping
 413  * @adapter: board private structure to initialize
 414  *
 415  * Once we know the feature-set enabled for the device, we'll cache
 416  * the register offset the descriptor ring is assigned to.
 417  **/
 418 static void igb_cache_ring_register(struct igb_adapter *adapter)
 419 {
 420         int i = 0, j = 0;
 421         u32 rbase_offset = adapter->vfs_allocated_count;
 422
 423         switch (adapter->hw.mac.type) {
 424         case e1000_82576:
 425                 /* The queues are allocated for virtualization such that VF 0
 426                  * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
 427                  * In order to avoid collision we start at the first free queue
 428                  * and continue consuming queues in the same sequence
 429                  */
 430                 if ((adapter->rss_queues > 1) && adapter->vmdq_pools) {
 431                         for (; i < adapter->rss_queues; i++)
 432                                 adapter->rx_ring[i]->reg_idx = rbase_offset +
 433                                                                Q_IDX_82576(i);
 434                 }
 435         case e1000_82575:
 436         case e1000_82580:
 437         case e1000_i350:
 438         default:
 439                 for (; i < adapter->num_rx_queues; i++)
 440                         adapter->rx_ring[i]->reg_idx = rbase_offset + i;
 441                 for (; j < adapter->num_tx_queues; j++)
 442                         adapter->tx_ring[j]->reg_idx = rbase_offset + j;
 443                 break;
 444         }
 445 }
 446
 447 static void igb_free_queues(struct igb_adapter *adapter)
 448 {
 449         int i;
 450
 451         for (i = 0; i < adapter->num_tx_queues; i++) {
 452                 kfree(adapter->tx_ring[i]);
 453                 adapter->tx_ring[i] = NULL;
 454         }
 455         for (i = 0; i < adapter->num_rx_queues; i++) {
 456                 kfree(adapter->rx_ring[i]);
 457                 adapter->rx_ring[i] = NULL;
 458         }
 459         adapter->num_rx_queues = 0;
 460         adapter->num_tx_queues = 0;
 461 }
 462
 463 /**
 464  * igb_alloc_queues - Allocate memory for all rings
 465  * @adapter: board private structure to initialize
 466  *
 467  * We allocate one ring per queue at run-time since we don't know the
 468  * number of queues at compile-time.
 469  **/
 470 static int igb_alloc_queues(struct igb_adapter *adapter)
 471 {
 472         struct igb_ring *ring;
 473         int i;
 474 #ifdef HAVE_DEVICE_NUMA_NODE
 475         int orig_node = adapter->node;
 476 #endif /* HAVE_DEVICE_NUMA_NODE */
 477
 478         for (i = 0; i < adapter->num_tx_queues; i++) {
 479 #ifdef HAVE_DEVICE_NUMA_NODE
 480                 if (orig_node == -1) {
 481                         int cur_node = next_online_node(adapter->node);
 482                         if (cur_node == MAX_NUMNODES)
 483                                 cur_node = first_online_node;
 484                         adapter->node = cur_node;
 485                 }
 486 #endif /* HAVE_DEVICE_NUMA_NODE */
 487                 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
 488                                     adapter->node);
 489                 if (!ring)
 490                         ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
 491                 if (!ring)
 492                         goto err;
 493                 ring->count = adapter->tx_ring_count;
 494                 ring->queue_index = i;
 495                 ring->dev = pci_dev_to_dev(adapter->pdev);
 496                 ring->netdev = adapter->netdev;
 497                 ring->numa_node = adapter->node;
 498                 /* For 82575, context index must be unique per ring. */
 499                 if (adapter->hw.mac.type == e1000_82575)
 500                         set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
 501                 adapter->tx_ring[i] = ring;
 502         }
 503 #ifdef HAVE_DEVICE_NUMA_NODE
 504         /* Restore the adapter's original node */
 505         adapter->node = orig_node;
 506 #endif /* HAVE_DEVICE_NUMA_NODE */
 507
 508         for (i = 0; i < adapter->num_rx_queues; i++) {
 509 #ifdef HAVE_DEVICE_NUMA_NODE
 510                 if (orig_node == -1) {
 511                         int cur_node = next_online_node(adapter->node);
 512                         if (cur_node == MAX_NUMNODES)
 513                                 cur_node = first_online_node;
 514                         adapter->node = cur_node;
 515                 }
 516 #endif /* HAVE_DEVICE_NUMA_NODE */
 517                 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
 518                                     adapter->node);
 519                 if (!ring)
 520                         ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
 521                 if (!ring)
 522                         goto err;
 523                 ring->count = adapter->rx_ring_count;
 524                 ring->queue_index = i;
 525                 ring->dev = pci_dev_to_dev(adapter->pdev);
 526                 ring->netdev = adapter->netdev;
 527                 ring->numa_node = adapter->node;
 528 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
 529                 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
 530 #endif
 531 #ifndef HAVE_NDO_SET_FEATURES
 532                 /* enable rx checksum */
 533                 set_bit(IGB_RING_FLAG_RX_CSUM, &ring->flags);
 534
 535 #endif
 536                 /* set flag indicating ring supports SCTP checksum offload */
 537                 if (adapter->hw.mac.type >= e1000_82576)
 538                         set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
 539
 540                 /* On i350, loopback VLAN packets have the tag byte-swapped. */
 541                 if (adapter->hw.mac.type == e1000_i350)
 542                         set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
 543
 544                 adapter->rx_ring[i] = ring;
 545         }
 546 #ifdef HAVE_DEVICE_NUMA_NODE
 547         /* Restore the adapter's original node */
 548         adapter->node = orig_node;
 549 #endif /* HAVE_DEVICE_NUMA_NODE */
 550
 551         igb_cache_ring_register(adapter);
 552
 553         return E1000_SUCCESS;
 554
 555 err:
 556 #ifdef HAVE_DEVICE_NUMA_NODE
 557         /* Restore the adapter's original node */
 558         adapter->node = orig_node;
 559 #endif /* HAVE_DEVICE_NUMA_NODE */
 560         igb_free_queues(adapter);
 561
 562         return -ENOMEM;
 563 }
 564
 565 static void igb_configure_lli(struct igb_adapter *adapter)
 566 {
 567         struct e1000_hw *hw = &adapter->hw;
 568         u16 port;
 569
 570         /* LLI should only be enabled for MSI-X or MSI interrupts */
 571         if (!adapter->msix_entries && !(adapter->flags & IGB_FLAG_HAS_MSI))
 572                 return;
 573
 574         if (adapter->lli_port) {
 575                 /* use filter 0 for port */
 576                 port = htons((u16)adapter->lli_port);
 577                 E1000_WRITE_REG(hw, E1000_IMIR(0),
 578                         (port | E1000_IMIR_PORT_IM_EN));
 579                 E1000_WRITE_REG(hw, E1000_IMIREXT(0),
 580                         (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
 581         }
 582
 583         if (adapter->flags & IGB_FLAG_LLI_PUSH) {
 584                 /* use filter 1 for push flag */
 585                 E1000_WRITE_REG(hw, E1000_IMIR(1),
 586                         (E1000_IMIR_PORT_BP | E1000_IMIR_PORT_IM_EN));
 587                 E1000_WRITE_REG(hw, E1000_IMIREXT(1),
 588                         (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_PSH));
 589         }
 590
 591         if (adapter->lli_size) {
 592                 /* use filter 2 for size */
 593                 E1000_WRITE_REG(hw, E1000_IMIR(2),
 594                         (E1000_IMIR_PORT_BP | E1000_IMIR_PORT_IM_EN));
 595                 E1000_WRITE_REG(hw, E1000_IMIREXT(2),
 596                         (adapter->lli_size | E1000_IMIREXT_CTRL_BP));
 597         }
 598
 599 }
 600
 601 /**
 602  *  igb_write_ivar - configure ivar for given MSI-X vector
 603  *  @hw: pointer to the HW structure
 604  *  @msix_vector: vector number we are allocating to a given ring
 605  *  @index: row index of IVAR register to write within IVAR table
 606  *  @offset: column offset of in IVAR, should be multiple of 8
 607  *
 608  *  This function is intended to handle the writing of the IVAR register
 609  *  for adapters 82576 and newer.  The IVAR table consists of 2 columns,
 610  *  each containing an cause allocation for an Rx and Tx ring, and a
 611  *  variable number of rows depending on the number of queues supported.
 612  **/
 613 static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
 614                            int index, int offset)
 615 {
 616         u32 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 617
 618         /* clear any bits that are currently set */
 619         ivar &= ~((u32)0xFF << offset);
 620
 621         /* write vector and valid bit */
 622         ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
 623
 624         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 625 }
 626
 627 #define IGB_N0_QUEUE -1
 628 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
 629 {
 630         struct igb_adapter *adapter = q_vector->adapter;
 631         struct e1000_hw *hw = &adapter->hw;
 632         int rx_queue = IGB_N0_QUEUE;
 633         int tx_queue = IGB_N0_QUEUE;
 634         u32 msixbm = 0;
 635
 636         if (q_vector->rx.ring)
 637                 rx_queue = q_vector->rx.ring->reg_idx;
 638         if (q_vector->tx.ring)
 639                 tx_queue = q_vector->tx.ring->reg_idx;
 640
 641         switch (hw->mac.type) {
 642         case e1000_82575:
 643                 /* The 82575 assigns vectors using a bitmask, which matches the
 644                    bitmask for the EICR/EIMS/EIMC registers.  To assign one
 645                    or more queues to a vector, we write the appropriate bits
 646                    into the MSIXBM register for that vector. */
 647                 if (rx_queue > IGB_N0_QUEUE)
 648                         msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
 649                 if (tx_queue > IGB_N0_QUEUE)
 650                         msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
 651                 if (!adapter->msix_entries && msix_vector == 0)
 652                         msixbm |= E1000_EIMS_OTHER;
 653                 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), msix_vector, msixbm);
 654                 q_vector->eims_value = msixbm;
 655                 break;
 656         case e1000_82576:
 657                 /*
 658                  * 82576 uses a table that essentially consists of 2 columns
 659                  * with 8 rows.  The ordering is column-major so we use the
 660                  * lower 3 bits as the row index, and the 4th bit as the
 661                  * column offset.
 662                  */
 663                 if (rx_queue > IGB_N0_QUEUE)
 664                         igb_write_ivar(hw, msix_vector,
 665                                        rx_queue & 0x7,
 666                                        (rx_queue & 0x8) << 1);
 667                 if (tx_queue > IGB_N0_QUEUE)
 668                         igb_write_ivar(hw, msix_vector,
 669                                        tx_queue & 0x7,
 670                                        ((tx_queue & 0x8) << 1) + 8);
 671                 q_vector->eims_value = 1 << msix_vector;
 672                 break;
 673         case e1000_82580:
 674         case e1000_i350:
 675                 /*
 676                  * On 82580 and newer adapters the scheme is similar to 82576
 677                  * however instead of ordering column-major we have things
 678                  * ordered row-major.  So we traverse the table by using
 679                  * bit 0 as the column offset, and the remaining bits as the
 680                  * row index.
 681                  */
 682                 if (rx_queue > IGB_N0_QUEUE)
 683                         igb_write_ivar(hw, msix_vector,
 684                                        rx_queue >> 1,
 685                                        (rx_queue & 0x1) << 4);
 686                 if (tx_queue > IGB_N0_QUEUE)
 687                         igb_write_ivar(hw, msix_vector,
 688                                        tx_queue >> 1,
 689                                        ((tx_queue & 0x1) << 4) + 8);
 690                 q_vector->eims_value = 1 << msix_vector;
 691                 break;
 692         default:
 693                 BUG();
 694                 break;
 695         }
 696
 697         /* add q_vector eims value to global eims_enable_mask */
 698         adapter->eims_enable_mask |= q_vector->eims_value;
 699
 700         /* configure q_vector to set itr on first interrupt */
 701         q_vector->set_itr = 1;
 702 }
 703
 704 /**
 705  * igb_configure_msix - Configure MSI-X hardware
 706  *
 707  * igb_configure_msix sets up the hardware to properly
 708  * generate MSI-X interrupts.
 709  **/
 710 static void igb_configure_msix(struct igb_adapter *adapter)
 711 {
 712         u32 tmp;
 713         int i, vector = 0;
 714         struct e1000_hw *hw = &adapter->hw;
 715
 716         adapter->eims_enable_mask = 0;
 717
 718         /* set vector for other causes, i.e. link changes */
 719         switch (hw->mac.type) {
 720         case e1000_82575:
 721                 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
 722                 /* enable MSI-X PBA support*/
 723                 tmp |= E1000_CTRL_EXT_PBA_CLR;
 724
 725                 /* Auto-Mask interrupts upon ICR read. */
 726                 tmp |= E1000_CTRL_EXT_EIAME;
 727                 tmp |= E1000_CTRL_EXT_IRCA;
 728
 729                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
 730
 731                 /* enable msix_other interrupt */
 732                 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), vector++,
 733                                       E1000_EIMS_OTHER);
 734                 adapter->eims_other = E1000_EIMS_OTHER;
 735
 736                 break;
 737
 738         case e1000_82576:
 739         case e1000_82580:
 740         case e1000_i350:
 741                 /* Turn on MSI-X capability first, or our settings
 742                  * won't stick.  And it will take days to debug. */
 743                 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE |
 744                                 E1000_GPIE_PBA | E1000_GPIE_EIAME |
 745                                 E1000_GPIE_NSICR);
 746
 747                 /* enable msix_other interrupt */
 748                 adapter->eims_other = 1 << vector;
 749                 tmp = (vector++ | E1000_IVAR_VALID) << 8;
 750
 751                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmp);
 752                 break;
 753         default:
 754                 /* do nothing, since nothing else supports MSI-X */
 755                 break;
 756         } /* switch (hw->mac.type) */
 757
 758         adapter->eims_enable_mask |= adapter->eims_other;
 759
 760         for (i = 0; i < adapter->num_q_vectors; i++)
 761                 igb_assign_vector(adapter->q_vector[i], vector++);
 762
 763         E1000_WRITE_FLUSH(hw);
 764 }
 765
 766 /**
 767  * igb_request_msix - Initialize MSI-X interrupts
 768  *
 769  * igb_request_msix allocates MSI-X vectors and requests interrupts from the
 770  * kernel.
 771  **/
 772 static int igb_request_msix(struct igb_adapter *adapter)
 773 {
 774         struct net_device *netdev = adapter->netdev;
 775         struct e1000_hw *hw = &adapter->hw;
 776         int i, err = 0, vector = 0;
 777
 778         err = request_irq(adapter->msix_entries[vector].vector,
 779                           &igb_msix_other, 0, netdev->name, adapter);
 780         if (err)
 781                 goto out;
 782         vector++;
 783
 784         for (i = 0; i < adapter->num_q_vectors; i++) {
 785                 struct igb_q_vector *q_vector = adapter->q_vector[i];
 786
 787                 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
 788
 789                 if (q_vector->rx.ring && q_vector->tx.ring)
 790                         sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
 791                                 q_vector->rx.ring->queue_index);
 792                 else if (q_vector->tx.ring)
 793                         sprintf(q_vector->name, "%s-tx-%u", netdev->name,
 794                                 q_vector->tx.ring->queue_index);
 795                 else if (q_vector->rx.ring)
 796                         sprintf(q_vector->name, "%s-rx-%u", netdev->name,
 797                                 q_vector->rx.ring->queue_index);
 798                 else
 799                         sprintf(q_vector->name, "%s-unused", netdev->name);
 800
 801                 err = request_irq(adapter->msix_entries[vector].vector,
 802                                   igb_msix_ring, 0, q_vector->name,
 803                                   q_vector);
 804                 if (err)
 805                         goto out;
 806                 vector++;
 807         }
 808
 809         igb_configure_msix(adapter);
 810         return 0;
 811 out:
 812         return err;
 813 }
 814
 815 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
 816 {
 817         if (adapter->msix_entries) {
 818                 pci_disable_msix(adapter->pdev);
 819                 kfree(adapter->msix_entries);
 820                 adapter->msix_entries = NULL;
 821         } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
 822                 pci_disable_msi(adapter->pdev);
 823         }
 824 }
 825
 826 /**
 827  * igb_free_q_vectors - Free memory allocated for interrupt vectors
 828  * @adapter: board private structure to initialize
 829  *
 830  * This function frees the memory allocated to the q_vectors.  In addition if
 831  * NAPI is enabled it will delete any references to the NAPI struct prior
 832  * to freeing the q_vector.
 833  **/
 834 static void igb_free_q_vectors(struct igb_adapter *adapter)
 835 {
 836         int v_idx;
 837
 838         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
 839                 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
 840                 adapter->q_vector[v_idx] = NULL;
 841                 if (!q_vector)
 842                         continue;
 843                 netif_napi_del(&q_vector->napi);
 844 #ifndef IGB_NO_LRO
 845                 if (q_vector->lrolist) {
 846                         __skb_queue_purge(&q_vector->lrolist->active);
 847                         vfree(q_vector->lrolist);
 848                         q_vector->lrolist = NULL;
 849                 }
 850 #endif
 851                 kfree(q_vector);
 852         }
 853         adapter->num_q_vectors = 0;
 854 }
 855
 856 /**
 857  * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
 858  *
 859  * This function resets the device so that it has 0 rx queues, tx queues, and
 860  * MSI-X interrupts allocated.
 861  */
 862 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
 863 {
 864         igb_free_queues(adapter);
 865         igb_free_q_vectors(adapter);
 866         igb_reset_interrupt_capability(adapter);
 867 }
 868
 869 /**
 870  * igb_process_mdd_event
 871  * @adapter - board private structure
 872  *
 873  * Identify a malicious VF, disable the VF TX/RX queues and log a message.
 874  */
 875 static void igb_process_mdd_event(struct igb_adapter *adapter)
 876 {
 877         struct e1000_hw *hw = &adapter->hw;
 878         u32 lvmmc, vfte, vfre, mdfb;
 879         u8 vf_queue;
 880
 881         lvmmc = E1000_READ_REG(hw, E1000_LVMMC);
 882         vf_queue = lvmmc >> 29;
 883
 884         /* VF index cannot be bigger or equal to VFs allocated */
 885         if (vf_queue >= adapter->vfs_allocated_count)
 886                 return;
 887
 888         netdev_info(adapter->netdev,
 889                     "VF %d misbehaved. VF queues are disabled. "
 890                     "VM misbehavior code is 0x%x\n", vf_queue, lvmmc);
 891
 892         /* Disable VFTE and VFRE related bits */
 893         vfte = E1000_READ_REG(hw, E1000_VFTE);
 894         vfte &= ~(1 << vf_queue);
 895         E1000_WRITE_REG(hw, E1000_VFTE, vfte);
 896
 897         vfre = E1000_READ_REG(hw, E1000_VFRE);
 898         vfre &= ~(1 << vf_queue);
 899         E1000_WRITE_REG(hw, E1000_VFRE, vfre);
 900
 901         /* Disable MDFB related bit */
 902         mdfb = E1000_READ_REG(hw, E1000_MDFB);
 903         mdfb &= ~(1 << vf_queue);
 904         E1000_WRITE_REG(hw, E1000_MDFB, mdfb);
 905
 906         /* Reset the specific VF */
 907         E1000_WRITE_REG(hw, E1000_VTCTRL(vf_queue), E1000_VTCTRL_RST);
 908 }
 909
 910 /**
 911  * igb_disable_mdd
 912  * @adapter - board private structure
 913  *
 914  * Disable MDD behavior in the HW
 915  **/
 916 static void igb_disable_mdd(struct igb_adapter *adapter)
 917 {
 918         struct e1000_hw *hw = &adapter->hw;
 919         u32 reg;
 920
 921         if (hw->mac.type != e1000_i350)
 922                 return;
 923
 924         reg = E1000_READ_REG(hw, E1000_DTXCTL);
 925         reg &= (~E1000_DTXCTL_MDP_EN);
 926         E1000_WRITE_REG(hw, E1000_DTXCTL, reg);
 927 }
 928
 929 /**
 930  * igb_enable_mdd
 931  * @adapter - board private structure
 932  *
 933  * Enable the HW to detect malicious driver and sends an interrupt to
 934  * the driver.
 935  *
 936  * Only available on i350 device
 937  **/
 938 static void igb_enable_mdd(struct igb_adapter *adapter)
 939 {
 940         struct e1000_hw *hw = &adapter->hw;
 941         u32 reg;
 942
 943         if (hw->mac.type != e1000_i350)
 944                 return;
 945
 946         reg = E1000_READ_REG(hw, E1000_DTXCTL);
 947         reg |= E1000_DTXCTL_MDP_EN;
 948         E1000_WRITE_REG(hw, E1000_DTXCTL, reg);
 949 }
 950
 951 /**
 952  * igb_reset_sriov_capability - disable SR-IOV if enabled
 953  *
 954  * Attempt to disable single root IO virtualization capabilites present in the
 955  * kernel.
 956  **/
 957 static void igb_reset_sriov_capability(struct igb_adapter *adapter)
 958 {
 959         struct pci_dev *pdev = adapter->pdev;
 960         struct e1000_hw *hw = &adapter->hw;
 961
 962         /* reclaim resources allocated to VFs */
 963         if (adapter->vf_data) {
 964                 if (!igb_check_vf_assignment(adapter)) {
 965                         /*
 966                          * disable iov and allow time for transactions to
 967                          * clear
 968                          */
 969                         pci_disable_sriov(pdev);
 970                         msleep(500);
 971
 972                         dev_info(pci_dev_to_dev(pdev), "IOV Disabled\n");
 973                 } else {
 974                         dev_info(pci_dev_to_dev(pdev), "IOV Not Disabled\n "
 975                                         "VF(s) are assigned to guests!\n");
 976                 }
 977                 /* Disable Malicious Driver Detection */
 978                 igb_disable_mdd(adapter);
 979
 980                 /* free vf data storage */
 981                 kfree(adapter->vf_data);
 982                 adapter->vf_data = NULL;
 983
 984                 /* switch rings back to PF ownership */
 985                 E1000_WRITE_REG(hw, E1000_IOVCTL,
 986                                 E1000_IOVCTL_REUSE_VFQ);
 987                 E1000_WRITE_FLUSH(hw);
 988                 msleep(100);
 989         }
 990
 991         adapter->vfs_allocated_count = 0;
 992 }
 993
 994 /**
 995  * igb_set_sriov_capability - setup SR-IOV if supported
 996  *
 997  * Attempt to enable single root IO virtualization capabilites present in the
 998  * kernel.
 999  **/
1000 static void igb_set_sriov_capability(struct igb_adapter *adapter)
1001 {
1002         struct pci_dev *pdev = adapter->pdev;
1003         int old_vfs = 0;
1004         int i;
1005
1006 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
1007         old_vfs = igb_find_enabled_vfs(adapter);
1008 #endif
1009         if (old_vfs) {
1010                 dev_info(pci_dev_to_dev(pdev),
1011                                 "%d pre-allocated VFs found - override "
1012                                 "max_vfs setting of %d\n", old_vfs,
1013                                 adapter->vfs_allocated_count);
1014                 adapter->vfs_allocated_count = old_vfs;
1015         }
1016         /* no VFs requested, do nothing */
1017         if (!adapter->vfs_allocated_count)
1018                 return;
1019
1020         /* allocate vf data storage */
1021         adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
1022                                    sizeof(struct vf_data_storage),
1023                                    GFP_KERNEL);
1024
1025         if (adapter->vf_data) {
1026                 if (!old_vfs) {
1027                         if (pci_enable_sriov(pdev,
1028                                         adapter->vfs_allocated_count))
1029                                 goto err_out;
1030                 }
1031                 for (i = 0; i < adapter->vfs_allocated_count; i++)
1032                         igb_vf_configure(adapter, i);
1033
1034                 /* DMA Coalescing is not supported in IOV mode. */
1035                 if (adapter->hw.mac.type >= e1000_i350)
1036                 adapter->dmac = IGB_DMAC_DISABLE;
1037                 if (adapter->hw.mac.type < e1000_i350)
1038                 adapter->flags |= IGB_FLAG_DETECT_BAD_DMA;
1039                 return;
1040
1041         }
1042
1043 err_out:
1044         kfree(adapter->vf_data);
1045         adapter->vf_data = NULL;
1046         adapter->vfs_allocated_count = 0;
1047         dev_warn(pci_dev_to_dev(pdev),
1048                         "Failed to initialize SR-IOV virtualization\n");
1049 }
1050
1051 /**
1052  * igb_set_interrupt_capability - set MSI or MSI-X if supported
1053  *
1054  * Attempt to configure interrupts using the best available
1055  * capabilities of the hardware and kernel.
1056  **/
1057 static void igb_set_interrupt_capability(struct igb_adapter *adapter)
1058 {
1059         struct pci_dev *pdev = adapter->pdev;
1060         int err;
1061         int numvecs, i;
1062
1063         /* Number of supported queues. */
1064         adapter->num_rx_queues = adapter->rss_queues;
1065
1066         if (adapter->vmdq_pools > 1)
1067                 adapter->num_rx_queues += adapter->vmdq_pools - 1;
1068
1069 #ifdef HAVE_TX_MQ
1070         if (adapter->vmdq_pools)
1071                 adapter->num_tx_queues = adapter->vmdq_pools;
1072         else
1073                 adapter->num_tx_queues = adapter->num_rx_queues;
1074 #else
1075         adapter->num_tx_queues = max_t(u32, 1, adapter->vmdq_pools);
1076 #endif
1077
1078         switch (adapter->int_mode) {
1079         case IGB_INT_MODE_MSIX:
1080                 /* start with one vector for every rx queue */
1081                 numvecs = adapter->num_rx_queues;
1082
1083                 /* if tx handler is seperate add 1 for every tx queue */
1084                 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1085                         numvecs += adapter->num_tx_queues;
1086
1087                 /* store the number of vectors reserved for queues */
1088                 adapter->num_q_vectors = numvecs;
1089
1090                 /* add 1 vector for link status interrupts */
1091                 numvecs++;
1092                 adapter->msix_entries = kcalloc(numvecs,
1093                                                 sizeof(struct msix_entry),
1094                                                 GFP_KERNEL);
1095                 if (adapter->msix_entries) {
1096                         for (i = 0; i < numvecs; i++)
1097                                 adapter->msix_entries[i].entry = i;
1098
1099                         err = pci_enable_msix(pdev,
1100                                               adapter->msix_entries, numvecs);
1101                         if (err == 0)
1102                                 break;
1103                 }
1104                 /* MSI-X failed, so fall through and try MSI */
1105                 dev_warn(pci_dev_to_dev(pdev), "Failed to initialize MSI-X interrupts. "
1106                          "Falling back to MSI interrupts.\n");
1107                 igb_reset_interrupt_capability(adapter);
1108         case IGB_INT_MODE_MSI:
1109                 if (!pci_enable_msi(pdev))
1110                         adapter->flags |= IGB_FLAG_HAS_MSI;
1111                 else
1112                         dev_warn(pci_dev_to_dev(pdev), "Failed to initialize MSI "
1113                                  "interrupts.  Falling back to legacy "
1114                                  "interrupts.\n");
1115                 /* Fall through */
1116         case IGB_INT_MODE_LEGACY:
1117                 /* disable advanced features and set number of queues to 1 */
1118                 igb_reset_sriov_capability(adapter);
1119                 adapter->vmdq_pools = 0;
1120                 adapter->rss_queues = 1;
1121                 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1122                 adapter->num_rx_queues = 1;
1123                 adapter->num_tx_queues = 1;
1124                 adapter->num_q_vectors = 1;
1125                 /* Don't do anything; this is system default */
1126                 break;
1127         }
1128
1129 #ifdef HAVE_TX_MQ
1130         /* Notify the stack of the (possibly) reduced Tx Queue count. */
1131 #ifdef CONFIG_NETDEVICES_MULTIQUEUE
1132         adapter->netdev->egress_subqueue_count = adapter->num_tx_queues;
1133 #else
1134         adapter->netdev->real_num_tx_queues =
1135                         (adapter->vmdq_pools ? 1 : adapter->num_tx_queues);
1136 #endif /* CONFIG_NETDEVICES_MULTIQUEUE */
1137 #endif /* HAVE_TX_MQ */
1138 }
1139
1140 /**
1141  * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1142  * @adapter: board private structure to initialize
1143  *
1144  * We allocate one q_vector per queue interrupt.  If allocation fails we
1145  * return -ENOMEM.
1146  **/
1147 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1148 {
1149         struct igb_q_vector *q_vector;
1150         struct e1000_hw *hw = &adapter->hw;
1151         int v_idx;
1152 #ifdef HAVE_DEVICE_NUMA_NODE
1153         int orig_node = adapter->node;
1154 #endif /* HAVE_DEVICE_NUMA_NODE */
1155
1156         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1157 #ifdef HAVE_DEVICE_NUMA_NODE
1158                 if ((adapter->num_q_vectors == (adapter->num_rx_queues +
1159                                                 adapter->num_tx_queues)) &&
1160                     (adapter->num_rx_queues == v_idx))
1161                         adapter->node = orig_node;
1162                 if (orig_node == -1) {
1163                         int cur_node = next_online_node(adapter->node);
1164                         if (cur_node == MAX_NUMNODES)
1165                                 cur_node = first_online_node;
1166                         adapter->node = cur_node;
1167                 }
1168 #endif /* HAVE_DEVICE_NUMA_NODE */
1169                 q_vector = kzalloc_node(sizeof(struct igb_q_vector), GFP_KERNEL,
1170                                         adapter->node);
1171                 if (!q_vector)
1172                         q_vector = kzalloc(sizeof(struct igb_q_vector),
1173                                            GFP_KERNEL);
1174                 if (!q_vector)
1175                         goto err_out;
1176                 q_vector->adapter = adapter;
1177                 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1178                 q_vector->itr_val = IGB_START_ITR;
1179                 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1180                 adapter->q_vector[v_idx] = q_vector;
1181 #ifndef IGB_NO_LRO
1182                 if (v_idx < adapter->num_rx_queues) {
1183                         int size = sizeof(struct igb_lro_list);
1184                         q_vector->lrolist = vzalloc_node(size, q_vector->numa_node);
1185                         if (!q_vector->lrolist)
1186                                 q_vector->lrolist = vzalloc(size);
1187                         if (!q_vector->lrolist)
1188                                 goto err_out;
1189                         __skb_queue_head_init(&q_vector->lrolist->active);
1190                 }
1191 #endif /* IGB_NO_LRO */
1192         }
1193 #ifdef HAVE_DEVICE_NUMA_NODE
1194         /* Restore the adapter's original node */
1195         adapter->node = orig_node;
1196 #endif /* HAVE_DEVICE_NUMA_NODE */
1197
1198         return 0;
1199
1200 err_out:
1201 #ifdef HAVE_DEVICE_NUMA_NODE
1202         /* Restore the adapter's original node */
1203         adapter->node = orig_node;
1204 #endif /* HAVE_DEVICE_NUMA_NODE */
1205         igb_free_q_vectors(adapter);
1206         return -ENOMEM;
1207 }
1208
1209 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1210                                       int ring_idx, int v_idx)
1211 {
1212         struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1213
1214         q_vector->rx.ring = adapter->rx_ring[ring_idx];
1215         q_vector->rx.ring->q_vector = q_vector;
1216         q_vector->rx.count++;
1217         q_vector->itr_val = adapter->rx_itr_setting;
1218         if (q_vector->itr_val && q_vector->itr_val <= 3)
1219                 q_vector->itr_val = IGB_START_ITR;
1220 }
1221
1222 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1223                                       int ring_idx, int v_idx)
1224 {
1225         struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1226
1227         q_vector->tx.ring = adapter->tx_ring[ring_idx];
1228         q_vector->tx.ring->q_vector = q_vector;
1229         q_vector->tx.count++;
1230         q_vector->itr_val = adapter->tx_itr_setting;
1231         q_vector->tx.work_limit = adapter->tx_work_limit;
1232         if (q_vector->itr_val && q_vector->itr_val <= 3)
1233                 q_vector->itr_val = IGB_START_ITR;
1234 }
1235
1236 /**
1237  * igb_map_ring_to_vector - maps allocated queues to vectors
1238  *
1239  * This function maps the recently allocated queues to vectors.
1240  **/
1241 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1242 {
1243         int i;
1244         int v_idx = 0;
1245
1246         if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1247             (adapter->num_q_vectors < adapter->num_tx_queues))
1248                 return -ENOMEM;
1249
1250         if (adapter->num_q_vectors >=
1251             (adapter->num_rx_queues + adapter->num_tx_queues)) {
1252                 for (i = 0; i < adapter->num_rx_queues; i++)
1253                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1254                 for (i = 0; i < adapter->num_tx_queues; i++)
1255                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1256         } else {
1257                 for (i = 0; i < adapter->num_rx_queues; i++) {
1258                         if (i < adapter->num_tx_queues)
1259                                 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1260                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1261                 }
1262                 for (; i < adapter->num_tx_queues; i++)
1263                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1264         }
1265         return 0;
1266 }
1267
1268 /**
1269  * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1270  *
1271  * This function initializes the interrupts and allocates all of the queues.
1272  **/
1273 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1274 {
1275         struct pci_dev *pdev = adapter->pdev;
1276         int err;
1277
1278         igb_set_interrupt_capability(adapter);
1279
1280         err = igb_alloc_q_vectors(adapter);
1281         if (err) {
1282                 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for vectors\n");
1283                 goto err_alloc_q_vectors;
1284         }
1285
1286         err = igb_alloc_queues(adapter);
1287         if (err) {
1288                 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for queues\n");
1289                 goto err_alloc_queues;
1290         }
1291
1292         err = igb_map_ring_to_vector(adapter);
1293         if (err) {
1294                 dev_err(pci_dev_to_dev(pdev), "Invalid q_vector to ring mapping\n");
1295                 goto err_map_queues;
1296         }
1297
1298
1299         return 0;
1300 err_map_queues:
1301         igb_free_queues(adapter);
1302 err_alloc_queues:
1303         igb_free_q_vectors(adapter);
1304 err_alloc_q_vectors:
1305         igb_reset_interrupt_capability(adapter);
1306         return err;
1307 }
1308
1309 /**
1310  * igb_request_irq - initialize interrupts
1311  *
1312  * Attempts to configure interrupts using the best available
1313  * capabilities of the hardware and kernel.
1314  **/
1315 static int igb_request_irq(struct igb_adapter *adapter)
1316 {
1317         struct net_device *netdev = adapter->netdev;
1318         struct pci_dev *pdev = adapter->pdev;
1319         int err = 0;
1320
1321         if (adapter->msix_entries) {
1322                 err = igb_request_msix(adapter);
1323                 if (!err)
1324                         goto request_done;
1325                 /* fall back to MSI */
1326                 igb_clear_interrupt_scheme(adapter);
1327                 igb_reset_sriov_capability(adapter);
1328                 if (!pci_enable_msi(pdev))
1329                         adapter->flags |= IGB_FLAG_HAS_MSI;
1330                 igb_free_all_tx_resources(adapter);
1331                 igb_free_all_rx_resources(adapter);
1332                 adapter->num_tx_queues = 1;
1333                 adapter->num_rx_queues = 1;
1334                 adapter->num_q_vectors = 1;
1335                 err = igb_alloc_q_vectors(adapter);
1336                 if (err) {
1337                         dev_err(pci_dev_to_dev(pdev),
1338                                 "Unable to allocate memory for vectors\n");
1339                         goto request_done;
1340                 }
1341                 err = igb_alloc_queues(adapter);
1342                 if (err) {
1343                         dev_err(pci_dev_to_dev(pdev),
1344                                 "Unable to allocate memory for queues\n");
1345                         igb_free_q_vectors(adapter);
1346                         goto request_done;
1347                 }
1348                 igb_setup_all_tx_resources(adapter);
1349                 igb_setup_all_rx_resources(adapter);
1350         }
1351
1352         igb_assign_vector(adapter->q_vector[0], 0);
1353
1354         if (adapter->flags & IGB_FLAG_HAS_MSI) {
1355                 err = request_irq(pdev->irq, &igb_intr_msi, 0,
1356                                   netdev->name, adapter);
1357                 if (!err)
1358                         goto request_done;
1359
1360                 /* fall back to legacy interrupts */
1361                 igb_reset_interrupt_capability(adapter);
1362                 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1363         }
1364
1365         err = request_irq(pdev->irq, &igb_intr, IRQF_SHARED,
1366                           netdev->name, adapter);
1367
1368         if (err)
1369                 dev_err(pci_dev_to_dev(pdev), "Error %d getting interrupt\n",
1370                         err);
1371
1372 request_done:
1373         return err;
1374 }
1375
1376 static void igb_free_irq(struct igb_adapter *adapter)
1377 {
1378         if (adapter->msix_entries) {
1379                 int vector = 0, i;
1380
1381                 free_irq(adapter->msix_entries[vector++].vector, adapter);
1382
1383                 for (i = 0; i < adapter->num_q_vectors; i++)
1384                         free_irq(adapter->msix_entries[vector++].vector,
1385                                  adapter->q_vector[i]);
1386         } else {
1387                 free_irq(adapter->pdev->irq, adapter);
1388         }
1389 }
1390
1391 /**
1392  * igb_irq_disable - Mask off interrupt generation on the NIC
1393  * @adapter: board private structure
1394  **/
1395 static void igb_irq_disable(struct igb_adapter *adapter)
1396 {
1397         struct e1000_hw *hw = &adapter->hw;
1398
1399         /*
1400          * we need to be careful when disabling interrupts.  The VFs are also
1401          * mapped into these registers and so clearing the bits can cause
1402          * issues on the VF drivers so we only need to clear what we set
1403          */
1404         if (adapter->msix_entries) {
1405                 u32 regval = E1000_READ_REG(hw, E1000_EIAM);
1406                 E1000_WRITE_REG(hw, E1000_EIAM, regval & ~adapter->eims_enable_mask);
1407                 E1000_WRITE_REG(hw, E1000_EIMC, adapter->eims_enable_mask);
1408                 regval = E1000_READ_REG(hw, E1000_EIAC);
1409                 E1000_WRITE_REG(hw, E1000_EIAC, regval & ~adapter->eims_enable_mask);
1410         }
1411
1412         E1000_WRITE_REG(hw, E1000_IAM, 0);
1413         E1000_WRITE_REG(hw, E1000_IMC, ~0);
1414         E1000_WRITE_FLUSH(hw);
1415
1416         if (adapter->msix_entries) {
1417                 int vector = 0, i;
1418
1419                 synchronize_irq(adapter->msix_entries[vector++].vector);
1420
1421                 for (i = 0; i < adapter->num_q_vectors; i++)
1422                         synchronize_irq(adapter->msix_entries[vector++].vector);
1423         } else {
1424                 synchronize_irq(adapter->pdev->irq);
1425         }
1426 }
1427
1428 /**
1429  * igb_irq_enable - Enable default interrupt generation settings
1430  * @adapter: board private structure
1431  **/
1432 static void igb_irq_enable(struct igb_adapter *adapter)
1433 {
1434         struct e1000_hw *hw = &adapter->hw;
1435
1436         if (adapter->msix_entries) {
1437                 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1438                 u32 regval = E1000_READ_REG(hw, E1000_EIAC);
1439                 E1000_WRITE_REG(hw, E1000_EIAC, regval | adapter->eims_enable_mask);
1440                 regval = E1000_READ_REG(hw, E1000_EIAM);
1441                 E1000_WRITE_REG(hw, E1000_EIAM, regval | adapter->eims_enable_mask);
1442                 E1000_WRITE_REG(hw, E1000_EIMS, adapter->eims_enable_mask);
1443                 if (adapter->vfs_allocated_count) {
1444                         E1000_WRITE_REG(hw, E1000_MBVFIMR, 0xFF);
1445                         ims |= E1000_IMS_VMMB;
1446                         /* For I350 device only enable MDD interrupts*/
1447                         if ((adapter->mdd) &&
1448                             (adapter->hw.mac.type == e1000_i350))
1449                                 ims |= E1000_IMS_MDDET;
1450                 }
1451                 E1000_WRITE_REG(hw, E1000_IMS, ims);
1452         } else {
1453                 E1000_WRITE_REG(hw, E1000_IMS, IMS_ENABLE_MASK |
1454                                 E1000_IMS_DRSTA);
1455                 E1000_WRITE_REG(hw, E1000_IAM, IMS_ENABLE_MASK |
1456                                 E1000_IMS_DRSTA);
1457         }
1458 }
1459
1460 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1461 {
1462         struct e1000_hw *hw = &adapter->hw;
1463         u16 vid = adapter->hw.mng_cookie.vlan_id;
1464         u16 old_vid = adapter->mng_vlan_id;
1465
1466         if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1467                 /* add VID to filter table */
1468                 igb_vfta_set(adapter, vid, TRUE);
1469                 adapter->mng_vlan_id = vid;
1470         } else {
1471                 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1472         }
1473
1474         if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1475             (vid != old_vid) &&
1476 #ifdef HAVE_VLAN_RX_REGISTER
1477             !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1478 #else
1479             !test_bit(old_vid, adapter->active_vlans)) {
1480 #endif
1481                 /* remove VID from filter table */
1482                 igb_vfta_set(adapter, old_vid, FALSE);
1483         }
1484 }
1485
1486 /**
1487  * igb_release_hw_control - release control of the h/w to f/w
1488  * @adapter: address of board private structure
1489  *
1490  * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1491  * For ASF and Pass Through versions of f/w this means that the
1492  * driver is no longer loaded.
1493  *
1494  **/
1495 static void igb_release_hw_control(struct igb_adapter *adapter)
1496 {
1497         struct e1000_hw *hw = &adapter->hw;
1498         u32 ctrl_ext;
1499
1500         /* Let firmware take over control of h/w */
1501         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1502         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1503                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1504 }
1505
1506 /**
1507  * igb_get_hw_control - get control of the h/w from f/w
1508  * @adapter: address of board private structure
1509  *
1510  * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1511  * For ASF and Pass Through versions of f/w this means that
1512  * the driver is loaded.
1513  *
1514  **/
1515 static void igb_get_hw_control(struct igb_adapter *adapter)
1516 {
1517         struct e1000_hw *hw = &adapter->hw;
1518         u32 ctrl_ext;
1519
1520         /* Let firmware know the driver has taken over */
1521         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1522         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1523                         ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1524 }
1525
1526 /**
1527  * igb_configure - configure the hardware for RX and TX
1528  * @adapter: private board structure
1529  **/
1530 static void igb_configure(struct igb_adapter *adapter)
1531 {
1532         struct net_device *netdev = adapter->netdev;
1533         int i;
1534
1535         igb_get_hw_control(adapter);
1536         igb_set_rx_mode(netdev);
1537
1538         igb_restore_vlan(adapter);
1539
1540         igb_setup_tctl(adapter);
1541         igb_setup_mrqc(adapter);
1542         igb_setup_rctl(adapter);
1543
1544         igb_configure_tx(adapter);
1545         igb_configure_rx(adapter);
1546
1547         e1000_rx_fifo_flush_82575(&adapter->hw);
1548 #ifdef CONFIG_NETDEVICES_MULTIQUEUE
1549
1550         if (adapter->num_tx_queues > 1)
1551                 netdev->features |= NETIF_F_MULTI_QUEUE;
1552         else
1553                 netdev->features &= ~NETIF_F_MULTI_QUEUE;
1554 #endif
1555
1556         /* call igb_desc_unused which always leaves
1557          * at least 1 descriptor unused to make sure
1558          * next_to_use != next_to_clean */
1559         for (i = 0; i < adapter->num_rx_queues; i++) {
1560                 struct igb_ring *ring = adapter->rx_ring[i];
1561                 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
1562         }
1563 }
1564
1565 /**
1566  * igb_power_up_link - Power up the phy/serdes link
1567  * @adapter: address of board private structure
1568  **/
1569 void igb_power_up_link(struct igb_adapter *adapter)
1570 {
1571         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1572                 e1000_power_up_phy(&adapter->hw);
1573         else
1574                 e1000_power_up_fiber_serdes_link(&adapter->hw);
1575
1576         e1000_phy_hw_reset(&adapter->hw);
1577 }
1578
1579 /**
1580  * igb_power_down_link - Power down the phy/serdes link
1581  * @adapter: address of board private structure
1582  */
1583 static void igb_power_down_link(struct igb_adapter *adapter)
1584 {
1585         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1586                 e1000_power_down_phy(&adapter->hw);
1587         else
1588                 e1000_shutdown_fiber_serdes_link(&adapter->hw);
1589 }
1590
1591 /**
1592  * igb_up - Open the interface and prepare it to handle traffic
1593  * @adapter: board private structure
1594  **/
1595 int igb_up(struct igb_adapter *adapter)
1596 {
1597         struct e1000_hw *hw = &adapter->hw;
1598         int i;
1599
1600         /* hardware has been reset, we need to reload some things */
1601         igb_configure(adapter);
1602
1603         clear_bit(__IGB_DOWN, &adapter->state);
1604
1605         for (i = 0; i < adapter->num_q_vectors; i++)
1606                 napi_enable(&(adapter->q_vector[i]->napi));
1607
1608         if (adapter->msix_entries)
1609                 igb_configure_msix(adapter);
1610         else
1611                 igb_assign_vector(adapter->q_vector[0], 0);
1612
1613         igb_configure_lli(adapter);
1614
1615         /* Clear any pending interrupts. */
1616         E1000_READ_REG(hw, E1000_ICR);
1617         igb_irq_enable(adapter);
1618
1619         /* notify VFs that reset has been completed */
1620         if (adapter->vfs_allocated_count) {
1621                 u32 reg_data = E1000_READ_REG(hw, E1000_CTRL_EXT);
1622                 reg_data |= E1000_CTRL_EXT_PFRSTD;
1623                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg_data);
1624         }
1625
1626         netif_tx_start_all_queues(adapter->netdev);
1627
1628         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
1629                 schedule_work(&adapter->dma_err_task);
1630         /* start the watchdog. */
1631         hw->mac.get_link_status = 1;
1632         schedule_work(&adapter->watchdog_task);
1633
1634         return 0;
1635 }
1636
1637 void igb_down(struct igb_adapter *adapter)
1638 {
1639         struct net_device *netdev = adapter->netdev;
1640         struct e1000_hw *hw = &adapter->hw;
1641         u32 tctl, rctl;
1642         int i;
1643
1644         /* signal that we're down so the interrupt handler does not
1645          * reschedule our watchdog timer */
1646         set_bit(__IGB_DOWN, &adapter->state);
1647
1648         /* disable receives in the hardware */
1649         rctl = E1000_READ_REG(hw, E1000_RCTL);
1650         E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
1651         /* flush and sleep below */
1652
1653         netif_tx_stop_all_queues(netdev);
1654
1655         /* disable transmits in the hardware */
1656         tctl = E1000_READ_REG(hw, E1000_TCTL);
1657         tctl &= ~E1000_TCTL_EN;
1658         E1000_WRITE_REG(hw, E1000_TCTL, tctl);
1659         /* flush both disables and wait for them to finish */
1660         E1000_WRITE_FLUSH(hw);
1661         usleep_range(10000, 20000);
1662
1663         for (i = 0; i < adapter->num_q_vectors; i++)
1664                 napi_disable(&(adapter->q_vector[i]->napi));
1665
1666         igb_irq_disable(adapter);
1667
1668         del_timer_sync(&adapter->watchdog_timer);
1669         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
1670                 del_timer_sync(&adapter->dma_err_timer);
1671         del_timer_sync(&adapter->phy_info_timer);
1672
1673         netif_carrier_off(netdev);
1674
1675         /* record the stats before reset*/
1676         igb_update_stats(adapter);
1677
1678         adapter->link_speed = 0;
1679         adapter->link_duplex = 0;
1680
1681 #ifdef HAVE_PCI_ERS
1682         if (!pci_channel_offline(adapter->pdev))
1683                 igb_reset(adapter);
1684 #else
1685         igb_reset(adapter);
1686 #endif
1687         igb_clean_all_tx_rings(adapter);
1688         igb_clean_all_rx_rings(adapter);
1689 #ifdef IGB_DCA
1690
1691         /* since we reset the hardware DCA settings were cleared */
1692         igb_setup_dca(adapter);
1693 #endif
1694 }
1695
1696 void igb_reinit_locked(struct igb_adapter *adapter)
1697 {
1698         WARN_ON(in_interrupt());
1699         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1700                 usleep_range(1000, 2000);
1701         igb_down(adapter);
1702         igb_up(adapter);
1703         clear_bit(__IGB_RESETTING, &adapter->state);
1704 }
1705
1706 void igb_reset(struct igb_adapter *adapter)
1707 {
1708         struct pci_dev *pdev = adapter->pdev;
1709         struct e1000_hw *hw = &adapter->hw;
1710         struct e1000_mac_info *mac = &hw->mac;
1711         struct e1000_fc_info *fc = &hw->fc;
1712         u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1713         u16 hwm;
1714
1715         /* Repartition Pba for greater than 9k mtu
1716          * To take effect CTRL.RST is required.
1717          */
1718         switch (mac->type) {
1719         case e1000_i350:
1720         case e1000_82580:
1721                 pba = E1000_READ_REG(hw, E1000_RXPBS);
1722                 pba = e1000_rxpbs_adjust_82580(pba);
1723                 break;
1724         case e1000_82576:
1725                 pba = E1000_READ_REG(hw, E1000_RXPBS);
1726                 pba &= E1000_RXPBS_SIZE_MASK_82576;
1727                 break;
1728         case e1000_82575:
1729         default:
1730                 pba = E1000_PBA_34K;
1731                 break;
1732         }
1733
1734         if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1735             (mac->type < e1000_82576)) {
1736                 /* adjust PBA for jumbo frames */
1737                 E1000_WRITE_REG(hw, E1000_PBA, pba);
1738
1739                 /* To maintain wire speed transmits, the Tx FIFO should be
1740                  * large enough to accommodate two full transmit packets,
1741                  * rounded up to the next 1KB and expressed in KB.  Likewise,
1742                  * the Rx FIFO should be large enough to accommodate at least
1743                  * one full receive packet and is similarly rounded up and
1744                  * expressed in KB. */
1745                 pba = E1000_READ_REG(hw, E1000_PBA);
1746                 /* upper 16 bits has Tx packet buffer allocation size in KB */
1747                 tx_space = pba >> 16;
1748                 /* lower 16 bits has Rx packet buffer allocation size in KB */
1749                 pba &= 0xffff;
1750                 /* the tx fifo also stores 16 bytes of information about the tx
1751                  * but don't include ethernet FCS because hardware appends it */
1752                 min_tx_space = (adapter->max_frame_size +
1753                                 sizeof(union e1000_adv_tx_desc) -
1754                                 ETH_FCS_LEN) * 2;
1755                 min_tx_space = ALIGN(min_tx_space, 1024);
1756                 min_tx_space >>= 10;
1757                 /* software strips receive CRC, so leave room for it */
1758                 min_rx_space = adapter->max_frame_size;
1759                 min_rx_space = ALIGN(min_rx_space, 1024);
1760                 min_rx_space >>= 10;
1761
1762                 /* If current Tx allocation is less than the min Tx FIFO size,
1763                  * and the min Tx FIFO size is less than the current Rx FIFO
1764                  * allocation, take space away from current Rx allocation */
1765                 if (tx_space < min_tx_space &&
1766                     ((min_tx_space - tx_space) < pba)) {
1767                         pba = pba - (min_tx_space - tx_space);
1768
1769                         /* if short on rx space, rx wins and must trump tx
1770                          * adjustment */
1771                         if (pba < min_rx_space)
1772                                 pba = min_rx_space;
1773                 }
1774                 E1000_WRITE_REG(hw, E1000_PBA, pba);
1775         }
1776
1777         /* flow control settings */
1778         /* The high water mark must be low enough to fit one full frame
1779          * (or the size used for early receive) above it in the Rx FIFO.
1780          * Set it to the lower of:
1781          * - 90% of the Rx FIFO size, or
1782          * - the full Rx FIFO size minus one full frame */
1783         hwm = min(((pba << 10) * 9 / 10),
1784                         ((pba << 10) - 2 * adapter->max_frame_size));
1785
1786         fc->high_water = hwm & 0xFFF0;  /* 16-byte granularity */
1787         fc->low_water = fc->high_water - 16;
1788         fc->pause_time = 0xFFFF;
1789         fc->send_xon = 1;
1790         fc->current_mode = fc->requested_mode;
1791
1792         /* disable receive for all VFs and wait one second */
1793         if (adapter->vfs_allocated_count) {
1794                 int i;
1795                 /*
1796                  * Clear all flags except indication that the PF has set
1797                  * the VF MAC addresses administratively
1798                  */
1799                 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1800                         adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
1801
1802                 /* ping all the active vfs to let them know we are going down */
1803                 igb_ping_all_vfs(adapter);
1804
1805                 /* disable transmits and receives */
1806                 E1000_WRITE_REG(hw, E1000_VFRE, 0);
1807                 E1000_WRITE_REG(hw, E1000_VFTE, 0);
1808         }
1809
1810         /* Allow time for pending master requests to run */
1811         e1000_reset_hw(hw);
1812         E1000_WRITE_REG(hw, E1000_WUC, 0);
1813
1814         if (e1000_init_hw(hw))
1815                 dev_err(pci_dev_to_dev(pdev), "Hardware Error\n");
1816
1817         igb_init_dmac(adapter, pba);
1818         /* Re-initialize the thermal sensor on i350 devices. */
1819         if (mac->type == e1000_i350 && hw->bus.func == 0) {
1820                 /*
1821                  * If present, re-initialize the external thermal sensor
1822                  * interface.
1823                  */
1824                 if (adapter->ets)
1825                         e1000_set_i2c_bb(hw);
1826                 e1000_init_thermal_sensor_thresh(hw);
1827         }
1828         if (!netif_running(adapter->netdev))
1829                 igb_power_down_link(adapter);
1830
1831         igb_update_mng_vlan(adapter);
1832
1833         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1834         E1000_WRITE_REG(hw, E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1835
1836         e1000_get_phy_info(hw);
1837 }
1838
1839 #ifdef HAVE_NDO_SET_FEATURES
1840 static netdev_features_t igb_fix_features(struct net_device *netdev,
1841                                           netdev_features_t features)
1842 {
1843         /*
1844          * Since there is no support for separate tx vlan accel
1845          * enabled make sure tx flag is cleared if rx is.
1846          */
1847         if (!(features & NETIF_F_HW_VLAN_RX))
1848                 features &= ~NETIF_F_HW_VLAN_TX;
1849
1850         /* If Rx checksum is disabled, then LRO should also be disabled */
1851         if (!(features & NETIF_F_RXCSUM))
1852                 features &= ~NETIF_F_LRO;
1853
1854         return features;
1855 }
1856
1857 static int igb_set_features(struct net_device *netdev,
1858                             netdev_features_t features)
1859 {
1860         u32 changed = netdev->features ^ features;
1861
1862         if (changed & NETIF_F_HW_VLAN_RX)
1863                 igb_vlan_mode(netdev, features);
1864
1865         return 0;
1866 }
1867
1868 #endif /* HAVE_NDO_SET_FEATURES */
1869 #ifdef HAVE_NET_DEVICE_OPS
1870 static const struct net_device_ops igb_netdev_ops = {
1871         .ndo_open               = igb_open,
1872         .ndo_stop               = igb_close,
1873         .ndo_start_xmit         = igb_xmit_frame,
1874         .ndo_get_stats          = igb_get_stats,
1875         .ndo_set_rx_mode        = igb_set_rx_mode,
1876         .ndo_set_mac_address    = igb_set_mac,
1877         .ndo_change_mtu         = igb_change_mtu,
1878         .ndo_do_ioctl           = igb_ioctl,
1879         .ndo_tx_timeout         = igb_tx_timeout,
1880         .ndo_validate_addr      = eth_validate_addr,
1881         .ndo_vlan_rx_add_vid    = igb_vlan_rx_add_vid,
1882         .ndo_vlan_rx_kill_vid   = igb_vlan_rx_kill_vid,
1883 #ifdef IFLA_VF_MAX
1884         .ndo_set_vf_mac         = igb_ndo_set_vf_mac,
1885         .ndo_set_vf_vlan        = igb_ndo_set_vf_vlan,
1886         .ndo_set_vf_tx_rate     = igb_ndo_set_vf_bw,
1887         .ndo_get_vf_config      = igb_ndo_get_vf_config,
1888 #endif
1889 #ifdef CONFIG_NET_POLL_CONTROLLER
1890         .ndo_poll_controller    = igb_netpoll,
1891 #endif
1892 #ifdef HAVE_NDO_SET_FEATURES
1893         .ndo_fix_features       = igb_fix_features,
1894         .ndo_set_features       = igb_set_features,
1895 #endif
1896 #ifdef HAVE_VLAN_RX_REGISTER
1897         .ndo_vlan_rx_register   = igb_vlan_mode,
1898 #endif
1899 };
1900
1901 #ifdef CONFIG_IGB_VMDQ_NETDEV
1902 static const struct net_device_ops igb_vmdq_ops = {
1903         .ndo_open               = &igb_vmdq_open,
1904         .ndo_stop               = &igb_vmdq_close,
1905         .ndo_start_xmit         = &igb_vmdq_xmit_frame,
1906         .ndo_get_stats          = &igb_vmdq_get_stats,
1907         .ndo_set_rx_mode        = &igb_vmdq_set_rx_mode,
1908         .ndo_validate_addr      = eth_validate_addr,
1909         .ndo_set_mac_address    = &igb_vmdq_set_mac,
1910         .ndo_change_mtu         = &igb_vmdq_change_mtu,
1911         .ndo_tx_timeout         = &igb_vmdq_tx_timeout,
1912         .ndo_vlan_rx_register   = &igb_vmdq_vlan_rx_register,
1913         .ndo_vlan_rx_add_vid    = &igb_vmdq_vlan_rx_add_vid,
1914         .ndo_vlan_rx_kill_vid   = &igb_vmdq_vlan_rx_kill_vid,
1915 };
1916
1917 #endif /* CONFIG_IGB_VMDQ_NETDEV */
1918 #endif /* HAVE_NET_DEVICE_OPS */
1919 #ifdef CONFIG_IGB_VMDQ_NETDEV
1920 void igb_assign_vmdq_netdev_ops(struct net_device *vnetdev)
1921 {
1922 #ifdef HAVE_NET_DEVICE_OPS
1923         vnetdev->netdev_ops = &igb_vmdq_ops;
1924 #else
1925         dev->open = &igb_vmdq_open;
1926         dev->stop = &igb_vmdq_close;
1927         dev->hard_start_xmit = &igb_vmdq_xmit_frame;
1928         dev->get_stats = &igb_vmdq_get_stats;
1929 #ifdef HAVE_SET_RX_MODE
1930         dev->set_rx_mode = &igb_vmdq_set_rx_mode;
1931 #endif
1932         dev->set_multicast_list = &igb_vmdq_set_rx_mode;
1933         dev->set_mac_address = &igb_vmdq_set_mac;
1934         dev->change_mtu = &igb_vmdq_change_mtu;
1935 #ifdef HAVE_TX_TIMEOUT
1936         dev->tx_timeout = &igb_vmdq_tx_timeout;
1937 #endif
1938 #ifdef NETIF_F_HW_VLAN_TX
1939         dev->vlan_rx_register = &igb_vmdq_vlan_rx_register;
1940         dev->vlan_rx_add_vid = &igb_vmdq_vlan_rx_add_vid;
1941         dev->vlan_rx_kill_vid = &igb_vmdq_vlan_rx_kill_vid;
1942 #endif
1943 #endif
1944         igb_vmdq_set_ethtool_ops(vnetdev);
1945         vnetdev->watchdog_timeo = 5 * HZ;
1946
1947 }
1948
1949 int igb_init_vmdq_netdevs(struct igb_adapter *adapter)
1950 {
1951         int pool, err = 0, base_queue;
1952         struct net_device *vnetdev;
1953         struct igb_vmdq_adapter *vmdq_adapter;
1954
1955         for (pool = 1; pool < adapter->vmdq_pools; pool++) {
1956                 int qpp = (!adapter->rss_queues ? 1 : adapter->rss_queues);
1957                 base_queue = pool * qpp;
1958                 vnetdev = alloc_etherdev(sizeof(struct igb_vmdq_adapter));
1959                 if (!vnetdev) {
1960                         err = -ENOMEM;
1961                         break;
1962                 }
1963                 vmdq_adapter = netdev_priv(vnetdev);
1964                 vmdq_adapter->vnetdev = vnetdev;
1965                 vmdq_adapter->real_adapter = adapter;
1966                 vmdq_adapter->rx_ring = adapter->rx_ring[base_queue];
1967                 vmdq_adapter->tx_ring = adapter->tx_ring[base_queue];
1968                 igb_assign_vmdq_netdev_ops(vnetdev);
1969                 snprintf(vnetdev->name, IFNAMSIZ, "%sv%d",
1970                          adapter->netdev->name, pool);
1971                 vnetdev->features = adapter->netdev->features;
1972 #ifdef HAVE_NETDEV_VLAN_FEATURES
1973                 vnetdev->vlan_features = adapter->netdev->vlan_features;
1974 #endif
1975                 adapter->vmdq_netdev[pool-1] = vnetdev;
1976                 err = register_netdev(vnetdev);
1977                 if (err)
1978                         break;
1979         }
1980         return err;
1981 }
1982
1983 int igb_remove_vmdq_netdevs(struct igb_adapter *adapter)
1984 {
1985         int pool, err = 0;
1986
1987         for (pool = 1; pool < adapter->vmdq_pools; pool++) {
1988                 unregister_netdev(adapter->vmdq_netdev[pool-1]);
1989                 free_netdev(adapter->vmdq_netdev[pool-1]);
1990                 adapter->vmdq_netdev[pool-1] = NULL;
1991         }
1992         return err;
1993 }
1994 #endif /* CONFIG_IGB_VMDQ_NETDEV */
1995
1996 /**
1997  * igb_probe - Device Initialization Routine
1998  * @pdev: PCI device information struct
1999  * @ent: entry in igb_pci_tbl
2000  *
2001  * Returns 0 on success, negative on failure
2002  *
2003  * igb_probe initializes an adapter identified by a pci_dev structure.
2004  * The OS initialization, configuring of the adapter private structure,
2005  * and a hardware reset occur.
2006  **/
2007 static int __devinit igb_probe(struct pci_dev *pdev,
2008                                const struct pci_device_id *ent)
2009 {
2010         struct net_device *netdev;
2011         struct igb_adapter *adapter;
2012         struct e1000_hw *hw;
2013         u16 eeprom_data = 0;
2014         u8 pba_str[E1000_PBANUM_LENGTH];
2015         s32 ret_val;
2016         static int global_quad_port_a; /* global quad port a indication */
2017         int i, err, pci_using_dac;
2018         static int cards_found;
2019
2020         err = pci_enable_device_mem(pdev);
2021         if (err)
2022                 return err;
2023
2024         pci_using_dac = 0;
2025         err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
2026         if (!err) {
2027                 err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
2028                 if (!err)
2029                         pci_using_dac = 1;
2030         } else {
2031                 err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
2032                 if (err) {
2033                         err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
2034                         if (err) {
2035                                 IGB_ERR("No usable DMA configuration, "
2036                                         "aborting\n");
2037                                 goto err_dma;
2038                         }
2039                 }
2040         }
2041
2042 #ifndef HAVE_ASPM_QUIRKS
2043         /* 82575 requires that the pci-e link partner disable the L0s state */
2044         switch (pdev->device) {
2045         case E1000_DEV_ID_82575EB_COPPER:
2046         case E1000_DEV_ID_82575EB_FIBER_SERDES:
2047         case E1000_DEV_ID_82575GB_QUAD_COPPER:
2048                 pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S);
2049         default:
2050                 break;
2051         }
2052
2053 #endif /* HAVE_ASPM_QUIRKS */
2054         err = pci_request_selected_regions(pdev,
2055                                            pci_select_bars(pdev,
2056                                                            IORESOURCE_MEM),
2057                                            igb_driver_name);
2058         if (err)
2059                 goto err_pci_reg;
2060
2061         pci_enable_pcie_error_reporting(pdev);
2062
2063         pci_set_master(pdev);
2064
2065         err = -ENOMEM;
2066 #ifdef HAVE_TX_MQ
2067         netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
2068                                    IGB_MAX_TX_QUEUES);
2069 #else
2070         netdev = alloc_etherdev(sizeof(struct igb_adapter));
2071 #endif /* HAVE_TX_MQ */
2072         if (!netdev)
2073                 goto err_alloc_etherdev;
2074
2075         SET_MODULE_OWNER(netdev);
2076         SET_NETDEV_DEV(netdev, &pdev->dev);
2077
2078         pci_set_drvdata(pdev, netdev);
2079         adapter = netdev_priv(netdev);
2080         adapter->netdev = netdev;
2081         adapter->pdev = pdev;
2082         hw = &adapter->hw;
2083         hw->back = adapter;
2084         adapter->port_num = hw->bus.func;
2085         adapter->msg_enable = (1 << debug) - 1;
2086
2087 #ifdef HAVE_PCI_ERS
2088         err = pci_save_state(pdev);
2089         if (err)
2090                 goto err_ioremap;
2091 #endif
2092         err = -EIO;
2093         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
2094                               pci_resource_len(pdev, 0));
2095         if (!hw->hw_addr)
2096                 goto err_ioremap;
2097
2098 #ifdef HAVE_NET_DEVICE_OPS
2099         netdev->netdev_ops = &igb_netdev_ops;
2100 #else /* HAVE_NET_DEVICE_OPS */
2101         netdev->open = &igb_open;
2102         netdev->stop = &igb_close;
2103         netdev->get_stats = &igb_get_stats;
2104 #ifdef HAVE_SET_RX_MODE
2105         netdev->set_rx_mode = &igb_set_rx_mode;
2106 #endif
2107         netdev->set_multicast_list = &igb_set_rx_mode;
2108         netdev->set_mac_address = &igb_set_mac;
2109         netdev->change_mtu = &igb_change_mtu;
2110         netdev->do_ioctl = &igb_ioctl;
2111 #ifdef HAVE_TX_TIMEOUT
2112         netdev->tx_timeout = &igb_tx_timeout;
2113 #endif
2114         netdev->vlan_rx_register = igb_vlan_mode;
2115         netdev->vlan_rx_add_vid = igb_vlan_rx_add_vid;
2116         netdev->vlan_rx_kill_vid = igb_vlan_rx_kill_vid;
2117 #ifdef CONFIG_NET_POLL_CONTROLLER
2118         netdev->poll_controller = igb_netpoll;
2119 #endif
2120         netdev->hard_start_xmit = &igb_xmit_frame;
2121 #endif /* HAVE_NET_DEVICE_OPS */
2122         igb_set_ethtool_ops(netdev);
2123 #ifdef HAVE_TX_TIMEOUT
2124         netdev->watchdog_timeo = 5 * HZ;
2125 #endif
2126
2127         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
2128
2129         adapter->bd_number = cards_found;
2130
2131         /* setup the private structure */
2132         err = igb_sw_init(adapter);
2133         if (err)
2134                 goto err_sw_init;
2135
2136         e1000_get_bus_info(hw);
2137
2138         hw->phy.autoneg_wait_to_complete = FALSE;
2139         hw->mac.adaptive_ifs = FALSE;
2140
2141         /* Copper options */
2142         if (hw->phy.media_type == e1000_media_type_copper) {
2143 #ifdef ETH_TP_MDI_X
2144                 hw->phy.mdix = ETH_TP_MDI_INVALID;
2145 #else
2146                 hw->phy.mdix = AUTO_ALL_MODES;
2147 #endif /* ETH_TP_MDI_X */
2148                 hw->phy.disable_polarity_correction = FALSE;
2149                 hw->phy.ms_type = e1000_ms_hw_default;
2150         }
2151
2152         if (e1000_check_reset_block(hw))
2153                 dev_info(pci_dev_to_dev(pdev),
2154                         "PHY reset is blocked due to SOL/IDER session.\n");
2155
2156         /*
2157          * features is initialized to 0 in allocation, it might have bits
2158          * set by igb_sw_init so we should use an or instead of an
2159          * assignment.
2160          */
2161         netdev->features |= NETIF_F_SG |
2162                             NETIF_F_IP_CSUM |
2163 #ifdef NETIF_F_IPV6_CSUM
2164                             NETIF_F_IPV6_CSUM |
2165 #endif
2166 #ifdef NETIF_F_TSO
2167                             NETIF_F_TSO |
2168 #ifdef NETIF_F_TSO6
2169                             NETIF_F_TSO6 |
2170 #endif
2171 #endif /* NETIF_F_TSO */
2172 #ifdef NETIF_F_RXHASH
2173                             NETIF_F_RXHASH |
2174 #endif
2175 #ifdef HAVE_NDO_SET_FEATURES
2176                             NETIF_F_RXCSUM |
2177 #endif
2178                             NETIF_F_HW_VLAN_RX |
2179                             NETIF_F_HW_VLAN_TX;
2180
2181 #ifdef HAVE_NDO_SET_FEATURES
2182         /* copy netdev features into list of user selectable features */
2183         netdev->hw_features |= netdev->features;
2184 #ifndef IGB_NO_LRO
2185
2186         /* give us the option of enabling LRO later */
2187         netdev->hw_features |= NETIF_F_LRO;
2188 #endif
2189 #else
2190 #ifdef NETIF_F_GRO
2191
2192         /* this is only needed on kernels prior to 2.6.39 */
2193         netdev->features |= NETIF_F_GRO;
2194 #endif
2195 #endif
2196
2197         /* set this bit last since it cannot be part of hw_features */
2198         netdev->features |= NETIF_F_HW_VLAN_FILTER;
2199
2200 #ifdef HAVE_NETDEV_VLAN_FEATURES
2201         netdev->vlan_features |= NETIF_F_TSO |
2202                                  NETIF_F_TSO6 |
2203                                  NETIF_F_IP_CSUM |
2204                                  NETIF_F_IPV6_CSUM |
2205                                  NETIF_F_SG;
2206
2207 #endif
2208         if (pci_using_dac)
2209                 netdev->features |= NETIF_F_HIGHDMA;
2210
2211         if (hw->mac.type >= e1000_82576)
2212                 netdev->features |= NETIF_F_SCTP_CSUM;
2213
2214         adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
2215
2216         /* before reading the NVM, reset the controller to put the device in a
2217          * known good starting state */
2218         e1000_reset_hw(hw);
2219
2220         /* make sure the NVM is good */
2221         if (e1000_validate_nvm_checksum(hw) < 0) {
2222                 dev_err(pci_dev_to_dev(pdev), "The NVM Checksum Is Not"
2223                         " Valid\n");
2224                 err = -EIO;
2225                 goto err_eeprom;
2226         }
2227
2228         /* copy the MAC address out of the NVM */
2229         if (e1000_read_mac_addr(hw))
2230                 dev_err(pci_dev_to_dev(pdev), "NVM Read Error\n");
2231         memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2232 #ifdef ETHTOOL_GPERMADDR
2233         memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
2234
2235         if (!is_valid_ether_addr(netdev->perm_addr)) {
2236 #else
2237         if (!is_valid_ether_addr(netdev->dev_addr)) {
2238 #endif
2239                 dev_err(pci_dev_to_dev(pdev), "Invalid MAC Address\n");
2240                 err = -EIO;
2241                 goto err_eeprom;
2242         }
2243
2244         memcpy(&adapter->mac_table[0].addr, hw->mac.addr, netdev->addr_len);
2245         adapter->mac_table[0].queue = adapter->vfs_allocated_count;
2246         adapter->mac_table[0].state = (IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE);
2247         igb_rar_set(adapter, 0);
2248
2249         /* get firmware version for ethtool -i */
2250         e1000_read_nvm(&adapter->hw, 5, 1, &adapter->fw_version);
2251         setup_timer(&adapter->watchdog_timer, &igb_watchdog,
2252                     (unsigned long) adapter);
2253         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2254                 setup_timer(&adapter->dma_err_timer, &igb_dma_err_timer,
2255                             (unsigned long) adapter);
2256         setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
2257                     (unsigned long) adapter);
2258
2259         INIT_WORK(&adapter->reset_task, igb_reset_task);
2260         INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2261         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2262                 INIT_WORK(&adapter->dma_err_task, igb_dma_err_task);
2263
2264         /* Initialize link properties that are user-changeable */
2265         adapter->fc_autoneg = true;
2266         hw->mac.autoneg = true;
2267         hw->phy.autoneg_advertised = 0x2f;
2268
2269         hw->fc.requested_mode = e1000_fc_default;
2270         hw->fc.current_mode = e1000_fc_default;
2271
2272         e1000_validate_mdi_setting(hw);
2273
2274         /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2275          * enable the ACPI Magic Packet filter
2276          */
2277
2278         if (hw->bus.func == 0)
2279                 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
2280         else if (hw->mac.type >= e1000_82580)
2281                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2282                                  NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2283                                  &eeprom_data);
2284         else if (hw->bus.func == 1)
2285                 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
2286
2287         if (eeprom_data & IGB_EEPROM_APME)
2288                 adapter->eeprom_wol |= E1000_WUFC_MAG;
2289
2290         /* now that we have the eeprom settings, apply the special cases where
2291          * the eeprom may be wrong or the board simply won't support wake on
2292          * lan on a particular port */
2293         switch (pdev->device) {
2294         case E1000_DEV_ID_82575GB_QUAD_COPPER:
2295                 adapter->eeprom_wol = 0;
2296                 break;
2297         case E1000_DEV_ID_82575EB_FIBER_SERDES:
2298         case E1000_DEV_ID_82576_FIBER:
2299         case E1000_DEV_ID_82576_SERDES:
2300                 /* Wake events only supported on port A for dual fiber
2301                  * regardless of eeprom setting */
2302                 if (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_FUNC_1)
2303                         adapter->eeprom_wol = 0;
2304                 break;
2305         case E1000_DEV_ID_82576_QUAD_COPPER:
2306         case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
2307                 /* if quad port adapter, disable WoL on all but port A */
2308                 if (global_quad_port_a != 0)
2309                         adapter->eeprom_wol = 0;
2310                 else
2311                         adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2312                 /* Reset for multiple quad port adapters */
2313                 if (++global_quad_port_a == 4)
2314                         global_quad_port_a = 0;
2315                 break;
2316         }
2317
2318         /* initialize the wol settings based on the eeprom settings */
2319         adapter->wol = adapter->eeprom_wol;
2320         device_set_wakeup_enable(pci_dev_to_dev(adapter->pdev), adapter->wol);
2321
2322         /* reset the hardware with the new settings */
2323         igb_reset(adapter);
2324
2325         /* let the f/w know that the h/w is now under the control of the
2326          * driver. */
2327         igb_get_hw_control(adapter);
2328
2329         strncpy(netdev->name, "eth%d", IFNAMSIZ);
2330         err = register_netdev(netdev);
2331         if (err)
2332                 goto err_register;
2333
2334 #ifdef CONFIG_IGB_VMDQ_NETDEV
2335         err = igb_init_vmdq_netdevs(adapter);
2336         if (err)
2337                 goto err_register;
2338 #endif
2339         /* carrier off reporting is important to ethtool even BEFORE open */
2340         netif_carrier_off(netdev);
2341
2342 #ifdef IGB_DCA
2343         if (dca_add_requester(&pdev->dev) == E1000_SUCCESS) {
2344                 adapter->flags |= IGB_FLAG_DCA_ENABLED;
2345                 dev_info(pci_dev_to_dev(pdev), "DCA enabled\n");
2346                 igb_setup_dca(adapter);
2347         }
2348
2349 #endif
2350 #ifdef HAVE_HW_TIME_STAMP
2351         /* do hw tstamp init after resetting */
2352         igb_init_hw_timer(adapter);
2353
2354 #endif
2355         dev_info(pci_dev_to_dev(pdev), "Intel(R) Gigabit Ethernet Network Connection\n");
2356         /* print bus type/speed/width info */
2357         dev_info(pci_dev_to_dev(pdev), "%s: (PCIe:%s:%s) ",
2358                  netdev->name,
2359                  ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5GT/s" :
2360                   (hw->bus.speed == e1000_bus_speed_5000) ? "5.0GT/s" :
2361                                                             "unknown"),
2362                  ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4\n" :
2363                   (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2\n" :
2364                   (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1\n" :
2365                    "unknown"));
2366         dev_info(pci_dev_to_dev(pdev), "%s: MAC: ", netdev->name);
2367         for (i = 0; i < 6; i++)
2368                 printk("%2.2x%c", netdev->dev_addr[i], i == 5 ? '\n' : ':');
2369
2370         ret_val = e1000_read_pba_string(hw, pba_str, E1000_PBANUM_LENGTH);
2371         if (ret_val)
2372                 strncpy(pba_str, "Unknown", sizeof(pba_str) - 1);
2373         dev_info(pci_dev_to_dev(pdev), "%s: PBA No: %s\n", netdev->name,
2374                  pba_str);
2375
2376
2377         /* Initialize the thermal sensor on i350 devices. */
2378         if (hw->mac.type == e1000_i350 && hw->bus.func == 0) {
2379                 u16 ets_word;
2380
2381                 /*
2382                  * Read the NVM to determine if this i350 device supports an
2383                  * external thermal sensor.
2384                  */
2385                 e1000_read_nvm(hw, NVM_ETS_CFG, 1, &ets_word);
2386                 if (ets_word != 0x0000 && ets_word != 0xFFFF)
2387                         adapter->ets = true;
2388                 else
2389                         adapter->ets = false;
2390 #ifdef IGB_SYSFS
2391                 igb_sysfs_init(adapter);
2392 #else
2393 #ifdef IGB_PROCFS
2394                 igb_procfs_init(adapter);
2395 #endif /* IGB_PROCFS */
2396 #endif /* IGB_SYSFS */
2397         } else {
2398                 adapter->ets = false;
2399         }
2400
2401         switch (hw->mac.type) {
2402         case e1000_i350:
2403                 /* Enable EEE for internal copper PHY devices */
2404                 if (hw->phy.media_type == e1000_media_type_copper)
2405                         e1000_set_eee_i350(hw);
2406
2407                 /* send driver version info to firmware */
2408                 igb_init_fw(adapter);
2409                 break;
2410         default:
2411                 break;
2412         }
2413 #ifndef IGB_NO_LRO
2414         if (netdev->features & NETIF_F_LRO)
2415                 dev_info(pci_dev_to_dev(pdev), "Internal LRO is enabled \n");
2416         else
2417                 dev_info(pci_dev_to_dev(pdev), "LRO is disabled \n");
2418 #endif
2419         dev_info(pci_dev_to_dev(pdev),
2420                  "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2421                  adapter->msix_entries ? "MSI-X" :
2422                  (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
2423                  adapter->num_rx_queues, adapter->num_tx_queues);
2424
2425         cards_found++;
2426
2427         pm_runtime_put_noidle(&pdev->dev);
2428         return 0;
2429
2430 err_register:
2431         igb_release_hw_control(adapter);
2432 err_eeprom:
2433         if (!e1000_check_reset_block(hw))
2434                 e1000_phy_hw_reset(hw);
2435
2436         if (hw->flash_address)
2437                 iounmap(hw->flash_address);
2438 err_sw_init:
2439         igb_clear_interrupt_scheme(adapter);
2440         igb_reset_sriov_capability(adapter);
2441         iounmap(hw->hw_addr);
2442 err_ioremap:
2443         free_netdev(netdev);
2444 err_alloc_etherdev:
2445         pci_release_selected_regions(pdev,
2446                                      pci_select_bars(pdev, IORESOURCE_MEM));
2447 err_pci_reg:
2448 err_dma:
2449         pci_disable_device(pdev);
2450         return err;
2451 }
2452
2453 /**
2454  * igb_remove - Device Removal Routine
2455  * @pdev: PCI device information struct
2456  *
2457  * igb_remove is called by the PCI subsystem to alert the driver
2458  * that it should release a PCI device.  The could be caused by a
2459  * Hot-Plug event, or because the driver is going to be removed from
2460  * memory.
2461  **/
2462 static void __devexit igb_remove(struct pci_dev *pdev)
2463 {
2464         struct net_device *netdev = pci_get_drvdata(pdev);
2465         struct igb_adapter *adapter = netdev_priv(netdev);
2466         struct e1000_hw *hw = &adapter->hw;
2467
2468         pm_runtime_get_noresume(&pdev->dev);
2469
2470         /* flush_scheduled work may reschedule our watchdog task, so
2471          * explicitly disable watchdog tasks from being rescheduled  */
2472         set_bit(__IGB_DOWN, &adapter->state);
2473         del_timer_sync(&adapter->watchdog_timer);
2474         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2475                 del_timer_sync(&adapter->dma_err_timer);
2476         del_timer_sync(&adapter->phy_info_timer);
2477
2478         flush_scheduled_work();
2479
2480 #ifdef IGB_DCA
2481         if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2482                 dev_info(pci_dev_to_dev(pdev), "DCA disabled\n");
2483                 dca_remove_requester(&pdev->dev);
2484                 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2485                 E1000_WRITE_REG(hw, E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_DISABLE);
2486         }
2487 #endif
2488
2489         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
2490          * would have already happened in close and is redundant. */
2491         igb_release_hw_control(adapter);
2492
2493         unregister_netdev(netdev);
2494 #ifdef CONFIG_IGB_VMDQ_NETDEV
2495         igb_remove_vmdq_netdevs(adapter);
2496 #endif
2497
2498         igb_clear_interrupt_scheme(adapter);
2499         igb_reset_sriov_capability(adapter);
2500
2501         iounmap(hw->hw_addr);
2502         if (hw->flash_address)
2503                 iounmap(hw->flash_address);
2504         pci_release_selected_regions(pdev,
2505                                      pci_select_bars(pdev, IORESOURCE_MEM));
2506
2507         kfree(adapter->mac_table);
2508         kfree(adapter->shadow_vfta);
2509         free_netdev(netdev);
2510
2511         pci_disable_pcie_error_reporting(pdev);
2512
2513         pci_disable_device(pdev);
2514
2515 #ifdef IGB_SYSFS
2516         igb_sysfs_exit(adapter);
2517 #else
2518 #ifdef IGB_PROCFS
2519         igb_procfs_exit(adapter);
2520 #endif /* IGB_PROCFS */
2521 #endif /* IGB_SYSFS */
2522 }
2523
2524 #ifdef HAVE_HW_TIME_STAMP
2525 /**
2526  * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2527  * @adapter: board private structure to initialize
2528  *
2529  * igb_init_hw_timer initializes the function pointer and values for the hw
2530  * timer found in hardware.
2531  **/
2532 static void igb_init_hw_timer(struct igb_adapter *adapter)
2533 {
2534         struct e1000_hw *hw = &adapter->hw;
2535
2536         switch (hw->mac.type) {
2537         case e1000_i350:
2538         case e1000_82580:
2539                 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2540                 adapter->cycles.read = igb_read_clock;
2541                 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2542                 adapter->cycles.mult = 1;
2543                 /*
2544                  * The 82580 timesync updates the system timer every 8ns by 8ns
2545                  * and the value cannot be shifted.  Instead we need to shift
2546                  * the registers to generate a 64bit timer value.  As a result
2547                  * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2548                  * 24 in order to generate a larger value for synchronization.
2549                  */
2550                 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2551                 /* disable system timer temporarily by setting bit 31 */
2552                 E1000_WRITE_REG(hw, E1000_TSAUXC, 0x80000000);
2553                 E1000_WRITE_FLUSH(hw);
2554
2555                 /* Set registers so that rollover occurs soon to test this. */
2556                 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x00000000);
2557                 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x80000000);
2558                 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x000000FF);
2559                 E1000_WRITE_FLUSH(hw);
2560
2561                 /* enable system timer by clearing bit 31 */
2562                 E1000_WRITE_REG(hw, E1000_TSAUXC, 0x0);
2563                 E1000_WRITE_FLUSH(hw);
2564
2565                 timecounter_init(&adapter->clock,
2566                                  &adapter->cycles,
2567                                  ktime_to_ns(ktime_get_real()));
2568                 /*
2569                  * Synchronize our NIC clock against system wall clock. NIC
2570                  * time stamp reading requires ~3us per sample, each sample
2571                  * was pretty stable even under load => only require 10
2572                  * samples for each offset comparison.
2573                  */
2574                 memset(&adapter->compare, 0, sizeof(adapter->compare));
2575                 adapter->compare.source = &adapter->clock;
2576                 adapter->compare.target = ktime_get_real;
2577                 adapter->compare.num_samples = 10;
2578                 timecompare_update(&adapter->compare, 0);
2579                 break;
2580         case e1000_82576:
2581                 /*
2582                  * Initialize hardware timer: we keep it running just in case
2583                  * that some program needs it later on.
2584                  */
2585                 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2586                 adapter->cycles.read = igb_read_clock;
2587                 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2588                 adapter->cycles.mult = 1;
2589                 /**
2590                  * Scale the NIC clock cycle by a large factor so that
2591                  * relatively small clock corrections can be added or
2592                  * subtracted at each clock tick. The drawbacks of a large
2593                  * factor are a) that the clock register overflows more quickly
2594                  * (not such a big deal) and b) that the increment per tick has
2595                  * to fit into 24 bits.  As a result we need to use a shift of
2596                  * 19 so we can fit a value of 16 into the TIMINCA register.
2597                  */
2598                 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2599                 E1000_WRITE_REG(hw, E1000_TIMINCA,
2600                                 (1 << E1000_TIMINCA_16NS_SHIFT) |
2601                                 (16 << IGB_82576_TSYNC_SHIFT));
2602
2603                 /* Set registers so that rollover occurs soon to test this. */
2604                 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x00000000);
2605                 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0xFF800000);
2606                 E1000_WRITE_FLUSH(hw);
2607
2608                 timecounter_init(&adapter->clock,
2609                                  &adapter->cycles,
2610                                  ktime_to_ns(ktime_get_real()));
2611                 /*
2612                  * Synchronize our NIC clock against system wall clock. NIC
2613                  * time stamp reading requires ~3us per sample, each sample
2614                  * was pretty stable even under load => only require 10
2615                  * samples for each offset comparison.
2616                  */
2617                 memset(&adapter->compare, 0, sizeof(adapter->compare));
2618                 adapter->compare.source = &adapter->clock;
2619                 adapter->compare.target = ktime_get_real;
2620                 adapter->compare.num_samples = 10;
2621                 timecompare_update(&adapter->compare, 0);
2622                 break;
2623         case e1000_82575:
2624                 /* 82575 does not support timesync */
2625         default:
2626                 break;
2627         }
2628 }
2629
2630 #endif /* HAVE_HW_TIME_STAMP */
2631 /**
2632  * igb_sw_init - Initialize general software structures (struct igb_adapter)
2633  * @adapter: board private structure to initialize
2634  *
2635  * igb_sw_init initializes the Adapter private data structure.
2636  * Fields are initialized based on PCI device information and
2637  * OS network device settings (MTU size).
2638  **/
2639 static int igb_sw_init(struct igb_adapter *adapter)
2640 {
2641         struct e1000_hw *hw = &adapter->hw;
2642         struct net_device *netdev = adapter->netdev;
2643         struct pci_dev *pdev = adapter->pdev;
2644
2645         /* PCI config space info */
2646
2647         hw->vendor_id = pdev->vendor;
2648         hw->device_id = pdev->device;
2649         hw->subsystem_vendor_id = pdev->subsystem_vendor;
2650         hw->subsystem_device_id = pdev->subsystem_device;
2651
2652         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
2653
2654         pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2655
2656         /* set default ring sizes */
2657         adapter->tx_ring_count = IGB_DEFAULT_TXD;
2658         adapter->rx_ring_count = IGB_DEFAULT_RXD;
2659
2660         /* set default work limits */
2661         adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2662
2663         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2664                                               VLAN_HLEN;
2665
2666         /* Initialize the hardware-specific values */
2667         if (e1000_setup_init_funcs(hw, TRUE)) {
2668                 dev_err(pci_dev_to_dev(pdev), "Hardware Initialization Failure\n");
2669                 return -EIO;
2670         }
2671
2672         adapter->mac_table = kzalloc(sizeof(struct igb_mac_addr) *
2673                                      hw->mac.rar_entry_count,
2674                                      GFP_ATOMIC);
2675
2676         /* Setup and initialize a copy of the hw vlan table array */
2677         adapter->shadow_vfta = (u32 *)kzalloc(sizeof(u32) * E1000_VFTA_ENTRIES,
2678                                         GFP_ATOMIC);
2679 #ifdef NO_KNI
2680         /* These calls may decrease the number of queues */
2681         igb_set_sriov_capability(adapter);
2682
2683         if (igb_init_interrupt_scheme(adapter)) {
2684                 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for queues\n");
2685                 return -ENOMEM;
2686         }
2687
2688         /* Explicitly disable IRQ since the NIC can be in any state. */
2689         igb_irq_disable(adapter);
2690
2691         set_bit(__IGB_DOWN, &adapter->state);
2692 #endif
2693         return 0;
2694 }
2695
2696 /**
2697  * igb_open - Called when a network interface is made active
2698  * @netdev: network interface device structure
2699  *
2700  * Returns 0 on success, negative value on failure
2701  *
2702  * The open entry point is called when a network interface is made
2703  * active by the system (IFF_UP).  At this point all resources needed
2704  * for transmit and receive operations are allocated, the interrupt
2705  * handler is registered with the OS, the watchdog timer is started,
2706  * and the stack is notified that the interface is ready.
2707  **/
2708 static int __igb_open(struct net_device *netdev, bool resuming)
2709 {
2710         struct igb_adapter *adapter = netdev_priv(netdev);
2711         struct e1000_hw *hw = &adapter->hw;
2712 #ifdef CONFIG_PM_RUNTIME
2713         struct pci_dev *pdev = adapter->pdev;
2714 #endif /* CONFIG_PM_RUNTIME */
2715         int err;
2716         int i;
2717
2718         /* disallow open during test */
2719         if (test_bit(__IGB_TESTING, &adapter->state)) {
2720                 WARN_ON(resuming);
2721                 return -EBUSY;
2722         }
2723
2724 #ifdef CONFIG_PM_RUNTIME
2725         if (!resuming)
2726                 pm_runtime_get_sync(&pdev->dev);
2727 #endif /* CONFIG_PM_RUNTIME */
2728
2729         netif_carrier_off(netdev);
2730
2731         /* allocate transmit descriptors */
2732         err = igb_setup_all_tx_resources(adapter);
2733         if (err)
2734                 goto err_setup_tx;
2735
2736         /* allocate receive descriptors */
2737         err = igb_setup_all_rx_resources(adapter);
2738         if (err)
2739                 goto err_setup_rx;
2740
2741         igb_power_up_link(adapter);
2742
2743         /* before we allocate an interrupt, we must be ready to handle it.
2744          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2745          * as soon as we call pci_request_irq, so we have to setup our
2746          * clean_rx handler before we do so.  */
2747         igb_configure(adapter);
2748
2749         err = igb_request_irq(adapter);
2750         if (err)
2751                 goto err_req_irq;
2752
2753         /* From here on the code is the same as igb_up() */
2754         clear_bit(__IGB_DOWN, &adapter->state);
2755
2756         for (i = 0; i < adapter->num_q_vectors; i++)
2757                 napi_enable(&(adapter->q_vector[i]->napi));
2758         igb_configure_lli(adapter);
2759
2760         /* Clear any pending interrupts. */
2761         E1000_READ_REG(hw, E1000_ICR);
2762
2763         igb_irq_enable(adapter);
2764
2765         /* notify VFs that reset has been completed */
2766         if (adapter->vfs_allocated_count) {
2767                 u32 reg_data = E1000_READ_REG(hw, E1000_CTRL_EXT);
2768                 reg_data |= E1000_CTRL_EXT_PFRSTD;
2769                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg_data);
2770         }
2771
2772         netif_tx_start_all_queues(netdev);
2773
2774         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2775                 schedule_work(&adapter->dma_err_task);
2776
2777         /* start the watchdog. */
2778         hw->mac.get_link_status = 1;
2779         schedule_work(&adapter->watchdog_task);
2780
2781         return E1000_SUCCESS;
2782
2783 err_req_irq:
2784         igb_release_hw_control(adapter);
2785         igb_power_down_link(adapter);
2786         igb_free_all_rx_resources(adapter);
2787 err_setup_rx:
2788         igb_free_all_tx_resources(adapter);
2789 err_setup_tx:
2790         igb_reset(adapter);
2791
2792 #ifdef CONFIG_PM_RUNTIME
2793         if (!resuming)
2794                 pm_runtime_put(&pdev->dev);
2795 #endif /* CONFIG_PM_RUNTIME */
2796
2797         return err;
2798 }
2799
2800 static int igb_open(struct net_device *netdev)
2801 {
2802         return __igb_open(netdev, false);
2803 }
2804
2805 /**
2806  * igb_close - Disables a network interface
2807  * @netdev: network interface device structure
2808  *
2809  * Returns 0, this is not allowed to fail
2810  *
2811  * The close entry point is called when an interface is de-activated
2812  * by the OS.  The hardware is still under the driver's control, but
2813  * needs to be disabled.  A global MAC reset is issued to stop the
2814  * hardware, and all transmit and receive resources are freed.
2815  **/
2816 static int __igb_close(struct net_device *netdev, bool suspending)
2817 {
2818         struct igb_adapter *adapter = netdev_priv(netdev);
2819 #ifdef CONFIG_PM_RUNTIME
2820         struct pci_dev *pdev = adapter->pdev;
2821 #endif /* CONFIG_PM_RUNTIME */
2822
2823         WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2824
2825 #ifdef CONFIG_PM_RUNTIME
2826         if (!suspending)
2827                 pm_runtime_get_sync(&pdev->dev);
2828 #endif /* CONFIG_PM_RUNTIME */
2829
2830         igb_down(adapter);
2831
2832         igb_release_hw_control(adapter);
2833
2834         igb_free_irq(adapter);
2835
2836         igb_free_all_tx_resources(adapter);
2837         igb_free_all_rx_resources(adapter);
2838
2839 #ifdef CONFIG_PM_RUNTIME
2840         if (!suspending)
2841                 pm_runtime_put_sync(&pdev->dev);
2842 #endif /* CONFIG_PM_RUNTIME */
2843
2844         return 0;
2845 }
2846
2847 static int igb_close(struct net_device *netdev)
2848 {
2849         return __igb_close(netdev, false);
2850 }
2851
2852 /**
2853  * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2854  * @tx_ring: tx descriptor ring (for a specific queue) to setup
2855  *
2856  * Return 0 on success, negative on failure
2857  **/
2858 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2859 {
2860         struct device *dev = tx_ring->dev;
2861         int orig_node = dev_to_node(dev);
2862         int size;
2863
2864         size = sizeof(struct igb_tx_buffer) * tx_ring->count;
2865         tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
2866         if (!tx_ring->tx_buffer_info)
2867                 tx_ring->tx_buffer_info = vzalloc(size);
2868         if (!tx_ring->tx_buffer_info)
2869                 goto err;
2870
2871         /* round up to nearest 4K */
2872         tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2873         tx_ring->size = ALIGN(tx_ring->size, 4096);
2874
2875         set_dev_node(dev, tx_ring->numa_node);
2876         tx_ring->desc = dma_alloc_coherent(dev,
2877                                            tx_ring->size,
2878                                            &tx_ring->dma,
2879                                            GFP_KERNEL);
2880         set_dev_node(dev, orig_node);
2881         if (!tx_ring->desc)
2882                 tx_ring->desc = dma_alloc_coherent(dev,
2883                                                    tx_ring->size,
2884                                                    &tx_ring->dma,
2885                                                    GFP_KERNEL);
2886
2887         if (!tx_ring->desc)
2888                 goto err;
2889
2890         tx_ring->next_to_use = 0;
2891         tx_ring->next_to_clean = 0;
2892
2893         return 0;
2894
2895 err:
2896         vfree(tx_ring->tx_buffer_info);
2897         dev_err(dev,
2898                 "Unable to allocate memory for the transmit descriptor ring\n");
2899         return -ENOMEM;
2900 }
2901
2902 /**
2903  * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2904  *                                (Descriptors) for all queues
2905  * @adapter: board private structure
2906  *
2907  * Return 0 on success, negative on failure
2908  **/
2909 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2910 {
2911         struct pci_dev *pdev = adapter->pdev;
2912         int i, err = 0;
2913
2914         for (i = 0; i < adapter->num_tx_queues; i++) {
2915                 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2916                 if (err) {
2917                         dev_err(pci_dev_to_dev(pdev),
2918                                 "Allocation for Tx Queue %u failed\n", i);
2919                         for (i--; i >= 0; i--)
2920                                 igb_free_tx_resources(adapter->tx_ring[i]);
2921                         break;
2922                 }
2923         }
2924
2925         return err;
2926 }
2927
2928 /**
2929  * igb_setup_tctl - configure the transmit control registers
2930  * @adapter: Board private structure
2931  **/
2932 void igb_setup_tctl(struct igb_adapter *adapter)
2933 {
2934         struct e1000_hw *hw = &adapter->hw;
2935         u32 tctl;
2936
2937         /* disable queue 0 which is enabled by default on 82575 and 82576 */
2938         E1000_WRITE_REG(hw, E1000_TXDCTL(0), 0);
2939
2940         /* Program the Transmit Control Register */
2941         tctl = E1000_READ_REG(hw, E1000_TCTL);
2942         tctl &= ~E1000_TCTL_CT;
2943         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2944                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2945
2946         e1000_config_collision_dist(hw);
2947
2948         /* Enable transmits */
2949         tctl |= E1000_TCTL_EN;
2950
2951         E1000_WRITE_REG(hw, E1000_TCTL, tctl);
2952 }
2953
2954 /**
2955  * igb_configure_tx_ring - Configure transmit ring after Reset
2956  * @adapter: board private structure
2957  * @ring: tx ring to configure
2958  *
2959  * Configure a transmit ring after a reset.
2960  **/
2961 void igb_configure_tx_ring(struct igb_adapter *adapter,
2962                            struct igb_ring *ring)
2963 {
2964         struct e1000_hw *hw = &adapter->hw;
2965         u32 txdctl = 0;
2966         u64 tdba = ring->dma;
2967         int reg_idx = ring->reg_idx;
2968
2969         /* disable the queue */
2970         E1000_WRITE_REG(hw, E1000_TXDCTL(reg_idx), 0);
2971         E1000_WRITE_FLUSH(hw);
2972         mdelay(10);
2973
2974         E1000_WRITE_REG(hw, E1000_TDLEN(reg_idx),
2975                         ring->count * sizeof(union e1000_adv_tx_desc));
2976         E1000_WRITE_REG(hw, E1000_TDBAL(reg_idx),
2977                         tdba & 0x00000000ffffffffULL);
2978         E1000_WRITE_REG(hw, E1000_TDBAH(reg_idx), tdba >> 32);
2979
2980         ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2981         E1000_WRITE_REG(hw, E1000_TDH(reg_idx), 0);
2982         writel(0, ring->tail);
2983
2984         txdctl |= IGB_TX_PTHRESH;
2985         txdctl |= IGB_TX_HTHRESH << 8;
2986         txdctl |= IGB_TX_WTHRESH << 16;
2987
2988         txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2989         E1000_WRITE_REG(hw, E1000_TXDCTL(reg_idx), txdctl);
2990 }
2991
2992 /**
2993  * igb_configure_tx - Configure transmit Unit after Reset
2994  * @adapter: board private structure
2995  *
2996  * Configure the Tx unit of the MAC after a reset.
2997  **/
2998 static void igb_configure_tx(struct igb_adapter *adapter)
2999 {
3000         int i;
3001
3002         for (i = 0; i < adapter->num_tx_queues; i++)
3003                 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
3004 }
3005
3006 /**
3007  * igb_setup_rx_resources - allocate Rx resources (Descriptors)
3008  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
3009  *
3010  * Returns 0 on success, negative on failure
3011  **/
3012 int igb_setup_rx_resources(struct igb_ring *rx_ring)
3013 {
3014         struct device *dev = rx_ring->dev;
3015         int orig_node = dev_to_node(dev);
3016         int size, desc_len;
3017
3018         size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3019         rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
3020         if (!rx_ring->rx_buffer_info)
3021                 rx_ring->rx_buffer_info = vzalloc(size);
3022         if (!rx_ring->rx_buffer_info)
3023                 goto err;
3024
3025         desc_len = sizeof(union e1000_adv_rx_desc);
3026
3027         /* Round up to nearest 4K */
3028         rx_ring->size = rx_ring->count * desc_len;
3029         rx_ring->size = ALIGN(rx_ring->size, 4096);
3030
3031         set_dev_node(dev, rx_ring->numa_node);
3032         rx_ring->desc = dma_alloc_coherent(dev,
3033                                            rx_ring->size,
3034                                            &rx_ring->dma,
3035                                            GFP_KERNEL);
3036         set_dev_node(dev, orig_node);
3037         if (!rx_ring->desc)
3038                 rx_ring->desc = dma_alloc_coherent(dev,
3039                                                    rx_ring->size,
3040                                                    &rx_ring->dma,
3041                                                    GFP_KERNEL);
3042
3043         if (!rx_ring->desc)
3044                 goto err;
3045
3046         rx_ring->next_to_clean = 0;
3047         rx_ring->next_to_use = 0;
3048
3049         return 0;
3050
3051 err:
3052         vfree(rx_ring->rx_buffer_info);
3053         rx_ring->rx_buffer_info = NULL;
3054         dev_err(dev, "Unable to allocate memory for the receive descriptor"
3055                 " ring\n");
3056         return -ENOMEM;
3057 }
3058
3059 /**
3060  * igb_setup_all_rx_resources - wrapper to allocate Rx resources
3061  *                                (Descriptors) for all queues
3062  * @adapter: board private structure
3063  *
3064  * Return 0 on success, negative on failure
3065  **/
3066 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
3067 {
3068         struct pci_dev *pdev = adapter->pdev;
3069         int i, err = 0;
3070
3071         for (i = 0; i < adapter->num_rx_queues; i++) {
3072                 err = igb_setup_rx_resources(adapter->rx_ring[i]);
3073                 if (err) {
3074                         dev_err(pci_dev_to_dev(pdev),
3075                                 "Allocation for Rx Queue %u failed\n", i);
3076                         for (i--; i >= 0; i--)
3077                                 igb_free_rx_resources(adapter->rx_ring[i]);
3078                         break;
3079                 }
3080         }
3081
3082         return err;
3083 }
3084
3085 /**
3086  * igb_setup_mrqc - configure the multiple receive queue control registers
3087  * @adapter: Board private structure
3088  **/
3089 static void igb_setup_mrqc(struct igb_adapter *adapter)
3090 {
3091         struct e1000_hw *hw = &adapter->hw;
3092         u32 mrqc, rxcsum;
3093         u32 j, num_rx_queues, shift = 0, shift2 = 0;
3094         union e1000_reta {
3095                 u32 dword;
3096                 u8  bytes[4];
3097         } reta;
3098         static const u8 rsshash[40] = {
3099                 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
3100                 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
3101                 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
3102                 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
3103
3104         /* Fill out hash function seeds */
3105         for (j = 0; j < 10; j++) {
3106                 u32 rsskey = rsshash[(j * 4)];
3107                 rsskey |= rsshash[(j * 4) + 1] << 8;
3108                 rsskey |= rsshash[(j * 4) + 2] << 16;
3109                 rsskey |= rsshash[(j * 4) + 3] << 24;
3110                 E1000_WRITE_REG_ARRAY(hw, E1000_RSSRK(0), j, rsskey);
3111         }
3112
3113         num_rx_queues = adapter->rss_queues;
3114
3115         if (adapter->vfs_allocated_count || adapter->vmdq_pools) {
3116                 /* 82575 and 82576 supports 2 RSS queues for VMDq */
3117                 switch (hw->mac.type) {
3118                 case e1000_i350:
3119                 case e1000_82580:
3120                         num_rx_queues = 1;
3121                         shift = 0;
3122                         break;
3123                 case e1000_82576:
3124                         shift = 3;
3125                         num_rx_queues = 2;
3126                         break;
3127                 case e1000_82575:
3128                         shift = 2;
3129                         shift2 = 6;
3130                 default:
3131                         break;
3132                 }
3133         } else {
3134                 if (hw->mac.type == e1000_82575)
3135                         shift = 6;
3136         }
3137
3138         for (j = 0; j < (32 * 4); j++) {
3139                 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
3140                 if (shift2)
3141                         reta.bytes[j & 3] |= num_rx_queues << shift2;
3142                 if ((j & 3) == 3)
3143                         E1000_WRITE_REG(hw, E1000_RETA(j >> 2), reta.dword);
3144         }
3145
3146         /*
3147          * Disable raw packet checksumming so that RSS hash is placed in
3148          * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
3149          * offloads as they are enabled by default
3150          */
3151         rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
3152         rxcsum |= E1000_RXCSUM_PCSD;
3153
3154         if (adapter->hw.mac.type >= e1000_82576)
3155                 /* Enable Receive Checksum Offload for SCTP */
3156                 rxcsum |= E1000_RXCSUM_CRCOFL;
3157
3158         /* Don't need to set TUOFL or IPOFL, they default to 1 */
3159         E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
3160
3161         /* If VMDq is enabled then we set the appropriate mode for that, else
3162          * we default to RSS so that an RSS hash is calculated per packet even
3163          * if we are only using one queue */
3164         if (adapter->vfs_allocated_count || adapter->vmdq_pools) {
3165                 if (hw->mac.type > e1000_82575) {
3166                         /* Set the default pool for the PF's first queue */
3167                         u32 vtctl = E1000_READ_REG(hw, E1000_VT_CTL);
3168                         vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
3169                                    E1000_VT_CTL_DISABLE_DEF_POOL);
3170                         vtctl |= adapter->vfs_allocated_count <<
3171                                 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
3172                         E1000_WRITE_REG(hw, E1000_VT_CTL, vtctl);
3173                 } else if (adapter->rss_queues > 1) {
3174                         /* set default queue for pool 1 to queue 2 */
3175                         E1000_WRITE_REG(hw, E1000_VT_CTL,
3176                                         adapter->rss_queues << 7);
3177                 }
3178                 if (adapter->rss_queues > 1)
3179                         mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
3180                 else
3181                         mrqc = E1000_MRQC_ENABLE_VMDQ;
3182         } else {
3183                 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
3184         }
3185
3186         igb_vmm_control(adapter);
3187
3188         /*
3189          * Generate RSS hash based on TCP port numbers and/or
3190          * IPv4/v6 src and dst addresses since UDP cannot be
3191          * hashed reliably due to IP fragmentation
3192          */
3193         mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
3194                 E1000_MRQC_RSS_FIELD_IPV4_TCP |
3195                 E1000_MRQC_RSS_FIELD_IPV6 |
3196                 E1000_MRQC_RSS_FIELD_IPV6_TCP |
3197                 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
3198
3199         E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
3200 }
3201
3202 /**
3203  * igb_setup_rctl - configure the receive control registers
3204  * @adapter: Board private structure
3205  **/
3206 void igb_setup_rctl(struct igb_adapter *adapter)
3207 {
3208         struct e1000_hw *hw = &adapter->hw;
3209         u32 rctl;
3210
3211         rctl = E1000_READ_REG(hw, E1000_RCTL);
3212
3213         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3214         rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
3215
3216         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
3217                 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
3218
3219         /*
3220          * enable stripping of CRC. It's unlikely this will break BMC
3221          * redirection as it did with e1000. Newer features require
3222          * that the HW strips the CRC.
3223          */
3224         rctl |= E1000_RCTL_SECRC;
3225
3226         /* disable store bad packets and clear size bits. */
3227         rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
3228
3229         /* enable LPE to prevent packets larger than max_frame_size */
3230         rctl |= E1000_RCTL_LPE;
3231
3232         /* disable queue 0 to prevent tail write w/o re-config */
3233         E1000_WRITE_REG(hw, E1000_RXDCTL(0), 0);
3234
3235         /* Attention!!!  For SR-IOV PF driver operations you must enable
3236          * queue drop for all VF and PF queues to prevent head of line blocking
3237          * if an un-trusted VF does not provide descriptors to hardware.
3238          */
3239         if (adapter->vfs_allocated_count) {
3240                 /* set all queue drop enable bits */
3241                 E1000_WRITE_REG(hw, E1000_QDE, ALL_QUEUES);
3242         }
3243
3244         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3245 }
3246
3247 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
3248                                    int vfn)
3249 {
3250         struct e1000_hw *hw = &adapter->hw;
3251         u32 vmolr;
3252
3253         /* if it isn't the PF check to see if VFs are enabled and
3254          * increase the size to support vlan tags */
3255         if (vfn < adapter->vfs_allocated_count &&
3256             adapter->vf_data[vfn].vlans_enabled)
3257                 size += VLAN_HLEN;
3258
3259 #ifdef CONFIG_IGB_VMDQ_NETDEV
3260         if (vfn >= adapter->vfs_allocated_count) {
3261                 int queue = vfn - adapter->vfs_allocated_count;
3262                 struct igb_vmdq_adapter *vadapter;
3263
3264                 vadapter = netdev_priv(adapter->vmdq_netdev[queue-1]);
3265                 if (vadapter->vlgrp)
3266                         size += VLAN_HLEN;
3267         }
3268 #endif
3269         vmolr = E1000_READ_REG(hw, E1000_VMOLR(vfn));
3270         vmolr &= ~E1000_VMOLR_RLPML_MASK;
3271         vmolr |= size | E1000_VMOLR_LPE;
3272         E1000_WRITE_REG(hw, E1000_VMOLR(vfn), vmolr);
3273
3274         return 0;
3275 }
3276
3277 /**
3278  * igb_rlpml_set - set maximum receive packet size
3279  * @adapter: board private structure
3280  *
3281  * Configure maximum receivable packet size.
3282  **/
3283 static void igb_rlpml_set(struct igb_adapter *adapter)
3284 {
3285         u32 max_frame_size = adapter->max_frame_size;
3286         struct e1000_hw *hw = &adapter->hw;
3287         u16 pf_id = adapter->vfs_allocated_count;
3288
3289         if (adapter->vmdq_pools && hw->mac.type != e1000_82575) {
3290                 int i;
3291                 for (i = 0; i < adapter->vmdq_pools; i++)
3292                         igb_set_vf_rlpml(adapter, max_frame_size, pf_id + i);
3293                 /*
3294                  * If we're in VMDQ or SR-IOV mode, then set global RLPML
3295                  * to our max jumbo frame size, in case we need to enable
3296                  * jumbo frames on one of the rings later.
3297                  * This will not pass over-length frames into the default
3298                  * queue because it's gated by the VMOLR.RLPML.
3299                  */
3300                 max_frame_size = MAX_JUMBO_FRAME_SIZE;
3301         }
3302         /* Set VF RLPML for the PF device. */
3303         if (adapter->vfs_allocated_count)
3304                 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
3305
3306         E1000_WRITE_REG(hw, E1000_RLPML, max_frame_size);
3307 }
3308
3309 static inline void igb_set_vf_vlan_strip(struct igb_adapter *adapter,
3310                                         int vfn, bool enable)
3311 {
3312         struct e1000_hw *hw = &adapter->hw;
3313         u32 val;
3314         void __iomem *reg;
3315
3316         if (hw->mac.type < e1000_82576)
3317                 return;
3318
3319         if (hw->mac.type == e1000_i350)
3320                 reg = hw->hw_addr + E1000_DVMOLR(vfn);
3321         else
3322                 reg = hw->hw_addr + E1000_VMOLR(vfn);
3323
3324         val = readl(reg);
3325         if (enable)
3326                 val |= E1000_VMOLR_STRVLAN;
3327         else
3328                 val &= ~(E1000_VMOLR_STRVLAN);
3329         writel(val, reg);
3330 }
3331 static inline void igb_set_vmolr(struct igb_adapter *adapter,
3332                                  int vfn, bool aupe)
3333 {
3334         struct e1000_hw *hw = &adapter->hw;
3335         u32 vmolr;
3336
3337         /*
3338          * This register exists only on 82576 and newer so if we are older then
3339          * we should exit and do nothing
3340          */
3341         if (hw->mac.type < e1000_82576)
3342                 return;
3343
3344         vmolr = E1000_READ_REG(hw, E1000_VMOLR(vfn));
3345
3346         if (aupe)
3347                 vmolr |= E1000_VMOLR_AUPE;        /* Accept untagged packets */
3348         else
3349                 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
3350
3351         /* clear all bits that might not be set */
3352         vmolr &= ~E1000_VMOLR_RSSE;
3353
3354         if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
3355                 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3356
3357         vmolr |= E1000_VMOLR_BAM;          /* Accept broadcast */
3358         vmolr |= E1000_VMOLR_LPE;          /* Accept long packets */
3359
3360         E1000_WRITE_REG(hw, E1000_VMOLR(vfn), vmolr);
3361 }
3362
3363 /**
3364  * igb_configure_rx_ring - Configure a receive ring after Reset
3365  * @adapter: board private structure
3366  * @ring: receive ring to be configured
3367  *
3368  * Configure the Rx unit of the MAC after a reset.
3369  **/
3370 void igb_configure_rx_ring(struct igb_adapter *adapter,
3371                            struct igb_ring *ring)
3372 {
3373         struct e1000_hw *hw = &adapter->hw;
3374         u64 rdba = ring->dma;
3375         int reg_idx = ring->reg_idx;
3376         u32 srrctl = 0, rxdctl = 0;
3377
3378         /* disable the queue */
3379         E1000_WRITE_REG(hw, E1000_RXDCTL(reg_idx), 0);
3380
3381         /* Set DMA base address registers */
3382         E1000_WRITE_REG(hw, E1000_RDBAL(reg_idx),
3383                         rdba & 0x00000000ffffffffULL);
3384         E1000_WRITE_REG(hw, E1000_RDBAH(reg_idx), rdba >> 32);
3385         E1000_WRITE_REG(hw, E1000_RDLEN(reg_idx),
3386                        ring->count * sizeof(union e1000_adv_rx_desc));
3387
3388         /* initialize head and tail */
3389         ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
3390         E1000_WRITE_REG(hw, E1000_RDH(reg_idx), 0);
3391         writel(0, ring->tail);
3392
3393         /* set descriptor configuration */
3394 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
3395         srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3396 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
3397         srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3398 #else
3399         srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3400 #endif
3401         srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3402 #else /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
3403         srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
3404                  E1000_SRRCTL_BSIZEPKT_SHIFT;
3405         srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
3406 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
3407 #ifdef IGB_PER_PKT_TIMESTAMP
3408         if (hw->mac.type >= e1000_82580)
3409                 srrctl |= E1000_SRRCTL_TIMESTAMP;
3410 #endif
3411         /*
3412          * We should set the drop enable bit if:
3413          *  SR-IOV is enabled
3414          *   or
3415          *  Flow Control is disabled and number of RX queues > 1
3416          *
3417          *  This allows us to avoid head of line blocking for security
3418          *  and performance reasons.
3419          */
3420         if (adapter->vfs_allocated_count ||
3421             (adapter->num_rx_queues > 1 &&
3422              (hw->fc.requested_mode == e1000_fc_none ||
3423               hw->fc.requested_mode == e1000_fc_rx_pause)))
3424                 srrctl |= E1000_SRRCTL_DROP_EN;
3425
3426         E1000_WRITE_REG(hw, E1000_SRRCTL(reg_idx), srrctl);
3427
3428         /* set filtering for VMDQ pools */
3429         igb_set_vmolr(adapter, reg_idx & 0x7, true);
3430
3431         rxdctl |= IGB_RX_PTHRESH;
3432         rxdctl |= IGB_RX_HTHRESH << 8;
3433         rxdctl |= IGB_RX_WTHRESH << 16;
3434
3435         /* enable receive descriptor fetching */
3436         rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3437         E1000_WRITE_REG(hw, E1000_RXDCTL(reg_idx), rxdctl);
3438 }
3439
3440 /**
3441  * igb_configure_rx - Configure receive Unit after Reset
3442  * @adapter: board private structure
3443  *
3444  * Configure the Rx unit of the MAC after a reset.
3445  **/
3446 static void igb_configure_rx(struct igb_adapter *adapter)
3447 {
3448         int i;
3449
3450         /* set UTA to appropriate mode */
3451         igb_set_uta(adapter);
3452
3453         igb_full_sync_mac_table(adapter);
3454         /* Setup the HW Rx Head and Tail Descriptor Pointers and
3455          * the Base and Length of the Rx Descriptor Ring */
3456         for (i = 0; i < adapter->num_rx_queues; i++)
3457                 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
3458 }
3459
3460 /**
3461  * igb_free_tx_resources - Free Tx Resources per Queue
3462  * @tx_ring: Tx descriptor ring for a specific queue
3463  *
3464  * Free all transmit software resources
3465  **/
3466 void igb_free_tx_resources(struct igb_ring *tx_ring)
3467 {
3468         igb_clean_tx_ring(tx_ring);
3469
3470         vfree(tx_ring->tx_buffer_info);
3471         tx_ring->tx_buffer_info = NULL;
3472
3473         /* if not set, then don't free */
3474         if (!tx_ring->desc)
3475                 return;
3476
3477         dma_free_coherent(tx_ring->dev, tx_ring->size,
3478                           tx_ring->desc, tx_ring->dma);
3479
3480         tx_ring->desc = NULL;
3481 }
3482
3483 /**
3484  * igb_free_all_tx_resources - Free Tx Resources for All Queues
3485  * @adapter: board private structure
3486  *
3487  * Free all transmit software resources
3488  **/
3489 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3490 {
3491         int i;
3492
3493         for (i = 0; i < adapter->num_tx_queues; i++)
3494                 igb_free_tx_resources(adapter->tx_ring[i]);
3495 }
3496
3497 void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
3498                                     struct igb_tx_buffer *tx_buffer)
3499 {
3500         if (tx_buffer->skb) {
3501                 dev_kfree_skb_any(tx_buffer->skb);
3502                 if (dma_unmap_len(tx_buffer, len))
3503                         dma_unmap_single(ring->dev,
3504                                          dma_unmap_addr(tx_buffer, dma),
3505                                          dma_unmap_len(tx_buffer, len),
3506                                          DMA_TO_DEVICE);
3507         } else if (dma_unmap_len(tx_buffer, len)) {
3508                 dma_unmap_page(ring->dev,
3509                                dma_unmap_addr(tx_buffer, dma),
3510                                dma_unmap_len(tx_buffer, len),
3511                                DMA_TO_DEVICE);
3512         }
3513         tx_buffer->next_to_watch = NULL;
3514         tx_buffer->skb = NULL;
3515         dma_unmap_len_set(tx_buffer, len, 0);
3516         /* buffer_info must be completely set up in the transmit path */
3517 }
3518
3519 /**
3520  * igb_clean_tx_ring - Free Tx Buffers
3521  * @tx_ring: ring to be cleaned
3522  **/
3523 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3524 {
3525         struct igb_tx_buffer *buffer_info;
3526         unsigned long size;
3527         u16 i;
3528
3529         if (!tx_ring->tx_buffer_info)
3530                 return;
3531         /* Free all the Tx ring sk_buffs */
3532
3533         for (i = 0; i < tx_ring->count; i++) {
3534                 buffer_info = &tx_ring->tx_buffer_info[i];
3535                 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3536         }
3537
3538 #ifdef CONFIG_BQL
3539         netdev_tx_reset_queue(txring_txq(tx_ring));
3540 #endif /* CONFIG_BQL */
3541
3542         size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3543         memset(tx_ring->tx_buffer_info, 0, size);
3544
3545         /* Zero out the descriptor ring */
3546         memset(tx_ring->desc, 0, tx_ring->size);
3547
3548         tx_ring->next_to_use = 0;
3549         tx_ring->next_to_clean = 0;
3550 }
3551
3552 /**
3553  * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3554  * @adapter: board private structure
3555  **/
3556 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3557 {
3558         int i;
3559
3560         for (i = 0; i < adapter->num_tx_queues; i++)
3561                 igb_clean_tx_ring(adapter->tx_ring[i]);
3562 }
3563
3564 /**
3565  * igb_free_rx_resources - Free Rx Resources
3566  * @rx_ring: ring to clean the resources from
3567  *
3568  * Free all receive software resources
3569  **/
3570 void igb_free_rx_resources(struct igb_ring *rx_ring)
3571 {
3572         igb_clean_rx_ring(rx_ring);
3573
3574         vfree(rx_ring->rx_buffer_info);
3575         rx_ring->rx_buffer_info = NULL;
3576
3577         /* if not set, then don't free */
3578         if (!rx_ring->desc)
3579                 return;
3580
3581         dma_free_coherent(rx_ring->dev, rx_ring->size,
3582                           rx_ring->desc, rx_ring->dma);
3583
3584         rx_ring->desc = NULL;
3585 }
3586
3587 /**
3588  * igb_free_all_rx_resources - Free Rx Resources for All Queues
3589  * @adapter: board private structure
3590  *
3591  * Free all receive software resources
3592  **/
3593 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3594 {
3595         int i;
3596
3597         for (i = 0; i < adapter->num_rx_queues; i++)
3598                 igb_free_rx_resources(adapter->rx_ring[i]);
3599 }
3600
3601 /**
3602  * igb_clean_rx_ring - Free Rx Buffers per Queue
3603  * @rx_ring: ring to free buffers from
3604  **/
3605 void igb_clean_rx_ring(struct igb_ring *rx_ring)
3606 {
3607         unsigned long size;
3608 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
3609         const int bufsz = rx_ring->rx_buffer_len;
3610 #else
3611         const int bufsz = IGB_RX_HDR_LEN;
3612 #endif
3613         u16 i;
3614
3615         if (!rx_ring->rx_buffer_info)
3616                 return;
3617
3618         /* Free all the Rx ring sk_buffs */
3619         for (i = 0; i < rx_ring->count; i++) {
3620                 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
3621                 if (buffer_info->dma) {
3622                         dma_unmap_single(rx_ring->dev,
3623                                          buffer_info->dma,
3624                                          bufsz,
3625                                          DMA_FROM_DEVICE);
3626                         buffer_info->dma = 0;
3627                 }
3628
3629                 if (buffer_info->skb) {
3630                         dev_kfree_skb(buffer_info->skb);
3631                         buffer_info->skb = NULL;
3632                 }
3633 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
3634                 if (buffer_info->page_dma) {
3635                         dma_unmap_page(rx_ring->dev,
3636                                        buffer_info->page_dma,
3637                                        PAGE_SIZE / 2,
3638                                        DMA_FROM_DEVICE);
3639                         buffer_info->page_dma = 0;
3640                 }
3641                 if (buffer_info->page) {
3642                         put_page(buffer_info->page);
3643                         buffer_info->page = NULL;
3644                         buffer_info->page_offset = 0;
3645                 }
3646 #endif
3647         }
3648
3649         size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3650         memset(rx_ring->rx_buffer_info, 0, size);
3651
3652         /* Zero out the descriptor ring */
3653         memset(rx_ring->desc, 0, rx_ring->size);
3654
3655         rx_ring->next_to_clean = 0;
3656         rx_ring->next_to_use = 0;
3657 }
3658
3659 /**
3660  * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3661  * @adapter: board private structure
3662  **/
3663 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3664 {
3665         int i;
3666
3667         for (i = 0; i < adapter->num_rx_queues; i++)
3668                 igb_clean_rx_ring(adapter->rx_ring[i]);
3669 }
3670
3671 /**
3672  * igb_set_mac - Change the Ethernet Address of the NIC
3673  * @netdev: network interface device structure
3674  * @p: pointer to an address structure
3675  *
3676  * Returns 0 on success, negative on failure
3677  **/
3678 static int igb_set_mac(struct net_device *netdev, void *p)
3679 {
3680         struct igb_adapter *adapter = netdev_priv(netdev);
3681         struct e1000_hw *hw = &adapter->hw;
3682         struct sockaddr *addr = p;
3683
3684         if (!is_valid_ether_addr(addr->sa_data))
3685                 return -EADDRNOTAVAIL;
3686
3687         igb_del_mac_filter(adapter, hw->mac.addr,
3688                            adapter->vfs_allocated_count);
3689         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3690         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3691
3692         /* set the correct pool for the new PF MAC address in entry 0 */
3693         return igb_add_mac_filter(adapter, hw->mac.addr,
3694                            adapter->vfs_allocated_count);
3695 }
3696
3697 /**
3698  * igb_write_mc_addr_list - write multicast addresses to MTA
3699  * @netdev: network interface device structure
3700  *
3701  * Writes multicast address list to the MTA hash table.
3702  * Returns: -ENOMEM on failure
3703  *                0 on no addresses written
3704  *                X on writing X addresses to MTA
3705  **/
3706 int igb_write_mc_addr_list(struct net_device *netdev)
3707 {
3708         struct igb_adapter *adapter = netdev_priv(netdev);
3709         struct e1000_hw *hw = &adapter->hw;
3710 #ifdef NETDEV_HW_ADDR_T_MULTICAST
3711         struct netdev_hw_addr *ha;
3712 #else
3713         struct dev_mc_list *ha;
3714 #endif
3715         u8  *mta_list;
3716         int i, count;
3717 #ifdef CONFIG_IGB_VMDQ_NETDEV
3718         int vm;
3719 #endif
3720         count = netdev_mc_count(netdev);
3721 #ifdef CONFIG_IGB_VMDQ_NETDEV
3722         for (vm = 1; vm < adapter->vmdq_pools; vm++) {
3723                 if (!adapter->vmdq_netdev[vm])
3724                         break;
3725                 if (!netif_running(adapter->vmdq_netdev[vm]))
3726                         continue;
3727                 count += netdev_mc_count(adapter->vmdq_netdev[vm]);
3728         }
3729 #endif
3730
3731         if (!count) {
3732                 e1000_update_mc_addr_list(hw, NULL, 0);
3733                 return 0;
3734         }
3735         mta_list = kzalloc(count * 6, GFP_ATOMIC);
3736         if (!mta_list)
3737                 return -ENOMEM;
3738
3739         /* The shared function expects a packed array of only addresses. */
3740         i = 0;
3741         netdev_for_each_mc_addr(ha, netdev)
3742 #ifdef NETDEV_HW_ADDR_T_MULTICAST
3743                 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3744 #else
3745                 memcpy(mta_list + (i++ * ETH_ALEN), ha->dmi_addr, ETH_ALEN);
3746 #endif
3747 #ifdef CONFIG_IGB_VMDQ_NETDEV
3748         for (vm = 1; vm < adapter->vmdq_pools; vm++) {
3749                 if (!adapter->vmdq_netdev[vm])
3750                         break;
3751                 if (!netif_running(adapter->vmdq_netdev[vm]) ||
3752                     !netdev_mc_count(adapter->vmdq_netdev[vm]))
3753                         continue;
3754                 netdev_for_each_mc_addr(ha, adapter->vmdq_netdev[vm])
3755 #ifdef NETDEV_HW_ADDR_T_MULTICAST
3756                         memcpy(mta_list + (i++ * ETH_ALEN),
3757                                ha->addr, ETH_ALEN);
3758 #else
3759                         memcpy(mta_list + (i++ * ETH_ALEN),
3760                                ha->dmi_addr, ETH_ALEN);
3761 #endif
3762         }
3763 #endif
3764         e1000_update_mc_addr_list(hw, mta_list, i);
3765         kfree(mta_list);
3766
3767         return count;
3768 }
3769
3770 void igb_rar_set(struct igb_adapter *adapter, u32 index)
3771 {
3772         u32 rar_low, rar_high;
3773         struct e1000_hw *hw = &adapter->hw;
3774         u8 *addr = adapter->mac_table[index].addr;
3775         /* HW expects these in little endian so we reverse the byte order
3776          * from network order (big endian) to little endian
3777          */
3778         rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
3779                   ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
3780         rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
3781
3782         /* Indicate to hardware the Address is Valid. */
3783         if (adapter->mac_table[index].state & IGB_MAC_STATE_IN_USE)
3784                 rar_high |= E1000_RAH_AV;
3785
3786         if (hw->mac.type == e1000_82575)
3787                 rar_high |= E1000_RAH_POOL_1 * adapter->mac_table[index].queue;
3788         else
3789                 rar_high |= E1000_RAH_POOL_1 << adapter->mac_table[index].queue;
3790
3791         E1000_WRITE_REG(hw, E1000_RAL(index), rar_low);
3792         E1000_WRITE_FLUSH(hw);
3793         E1000_WRITE_REG(hw, E1000_RAH(index), rar_high);
3794         E1000_WRITE_FLUSH(hw);
3795 }
3796
3797 void igb_full_sync_mac_table(struct igb_adapter *adapter)
3798 {
3799         struct e1000_hw *hw = &adapter->hw;
3800         int i;
3801         for (i = 0; i < hw->mac.rar_entry_count; i++) {
3802                         igb_rar_set(adapter, i);
3803         }
3804 }
3805
3806 void igb_sync_mac_table(struct igb_adapter *adapter)
3807 {
3808         struct e1000_hw *hw = &adapter->hw;
3809         int i;
3810         for (i = 0; i < hw->mac.rar_entry_count; i++) {
3811                 if (adapter->mac_table[i].state & IGB_MAC_STATE_MODIFIED)
3812                         igb_rar_set(adapter, i);
3813                 adapter->mac_table[i].state &= ~(IGB_MAC_STATE_MODIFIED);
3814         }
3815 }
3816
3817 int igb_available_rars(struct igb_adapter *adapter)
3818 {
3819         struct e1000_hw *hw = &adapter->hw;
3820         int i, count = 0;
3821
3822         for (i = 0; i < hw->mac.rar_entry_count; i++) {
3823                 if (adapter->mac_table[i].state == 0)
3824                         count++;
3825         }
3826         return count;
3827 }
3828
3829 #ifdef HAVE_SET_RX_MODE
3830 /**
3831  * igb_write_uc_addr_list - write unicast addresses to RAR table
3832  * @netdev: network interface device structure
3833  *
3834  * Writes unicast address list to the RAR table.
3835  * Returns: -ENOMEM on failure/insufficient address space
3836  *                0 on no addresses written
3837  *                X on writing X addresses to the RAR table
3838  **/
3839 static int igb_write_uc_addr_list(struct net_device *netdev)
3840 {
3841         struct igb_adapter *adapter = netdev_priv(netdev);
3842         unsigned int vfn = adapter->vfs_allocated_count;
3843         int count = 0;
3844
3845         /* return ENOMEM indicating insufficient memory for addresses */
3846         if (netdev_uc_count(netdev) > igb_available_rars(adapter))
3847                 return -ENOMEM;
3848         if (!netdev_uc_empty(netdev)) {
3849 #ifdef NETDEV_HW_ADDR_T_UNICAST
3850                 struct netdev_hw_addr *ha;
3851 #else
3852                 struct dev_mc_list *ha;
3853 #endif
3854                 netdev_for_each_uc_addr(ha, netdev) {
3855 #ifdef NETDEV_HW_ADDR_T_UNICAST
3856                         igb_del_mac_filter(adapter, ha->addr, vfn);
3857                         igb_add_mac_filter(adapter, ha->addr, vfn);
3858 #else
3859                         igb_del_mac_filter(adapter, ha->da_addr, vfn);
3860                         igb_add_mac_filter(adapter, ha->da_addr, vfn);
3861 #endif
3862                         count++;
3863                 }
3864         }
3865         return count;
3866 }
3867
3868 #endif /* HAVE_SET_RX_MODE */
3869 /**
3870  * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3871  * @netdev: network interface device structure
3872  *
3873  * The set_rx_mode entry point is called whenever the unicast or multicast
3874  * address lists or the network interface flags are updated.  This routine is
3875  * responsible for configuring the hardware for proper unicast, multicast,
3876  * promiscuous mode, and all-multi behavior.
3877  **/
3878 static void igb_set_rx_mode(struct net_device *netdev)
3879 {
3880         struct igb_adapter *adapter = netdev_priv(netdev);
3881         struct e1000_hw *hw = &adapter->hw;
3882         unsigned int vfn = adapter->vfs_allocated_count;
3883         u32 rctl, vmolr = 0;
3884         int count;
3885
3886         /* Check for Promiscuous and All Multicast modes */
3887         rctl = E1000_READ_REG(hw, E1000_RCTL);
3888
3889         /* clear the effected bits */
3890         rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3891
3892         if (netdev->flags & IFF_PROMISC) {
3893                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3894                 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3895         } else {
3896                 if (netdev->flags & IFF_ALLMULTI) {
3897                         rctl |= E1000_RCTL_MPE;
3898                         vmolr |= E1000_VMOLR_MPME;
3899                 } else {
3900                         /*
3901                          * Write addresses to the MTA, if the attempt fails
3902                          * then we should just turn on promiscuous mode so
3903                          * that we can at least receive multicast traffic
3904                          */
3905                         count = igb_write_mc_addr_list(netdev);
3906                         if (count < 0) {
3907                                 rctl |= E1000_RCTL_MPE;
3908                                 vmolr |= E1000_VMOLR_MPME;
3909                         } else if (count) {
3910                                 vmolr |= E1000_VMOLR_ROMPE;
3911                         }
3912                 }
3913 #ifdef HAVE_SET_RX_MODE
3914                 /*
3915                  * Write addresses to available RAR registers, if there is not
3916                  * sufficient space to store all the addresses then enable
3917                  * unicast promiscuous mode
3918                  */
3919                 count = igb_write_uc_addr_list(netdev);
3920                 if (count < 0) {
3921                         rctl |= E1000_RCTL_UPE;
3922                         vmolr |= E1000_VMOLR_ROPE;
3923                 }
3924 #endif /* HAVE_SET_RX_MODE */
3925                 rctl |= E1000_RCTL_VFE;
3926         }
3927         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3928
3929         /*
3930          * In order to support SR-IOV and eventually VMDq it is necessary to set
3931          * the VMOLR to enable the appropriate modes.  Without this workaround
3932          * we will have issues with VLAN tag stripping not being done for frames
3933          * that are only arriving because we are the default pool
3934          */
3935         if (hw->mac.type < e1000_82576)
3936                 return;
3937
3938         vmolr |= E1000_READ_REG(hw, E1000_VMOLR(vfn)) &
3939                  ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3940         E1000_WRITE_REG(hw, E1000_VMOLR(vfn), vmolr);
3941         igb_restore_vf_multicasts(adapter);
3942 }
3943
3944 static void igb_check_wvbr(struct igb_adapter *adapter)
3945 {
3946         struct e1000_hw *hw = &adapter->hw;
3947         u32 wvbr = 0;
3948
3949         switch (hw->mac.type) {
3950         case e1000_82576:
3951         case e1000_i350:
3952                 if (!(wvbr = E1000_READ_REG(hw, E1000_WVBR)))
3953                         return;
3954                 break;
3955         default:
3956                 break;
3957         }
3958
3959         adapter->wvbr |= wvbr;
3960 }
3961
3962 #define IGB_STAGGERED_QUEUE_OFFSET 8
3963
3964 static void igb_spoof_check(struct igb_adapter *adapter)
3965 {
3966         int j;
3967
3968         if (!adapter->wvbr)
3969                 return;
3970
3971         for(j = 0; j < adapter->vfs_allocated_count; j++) {
3972                 if (adapter->wvbr & (1 << j) ||
3973                     adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3974                         DPRINTK(DRV, WARNING,
3975                                 "Spoof event(s) detected on VF %d\n", j);
3976                         adapter->wvbr &=
3977                                 ~((1 << j) |
3978                                   (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3979                 }
3980         }
3981 }
3982
3983 /* Need to wait a few seconds after link up to get diagnostic information from
3984  * the phy */
3985 static void igb_update_phy_info(unsigned long data)
3986 {
3987         struct igb_adapter *adapter = (struct igb_adapter *) data;
3988         e1000_get_phy_info(&adapter->hw);
3989 }
3990
3991 /**
3992  * igb_has_link - check shared code for link and determine up/down
3993  * @adapter: pointer to driver private info
3994  **/
3995 bool igb_has_link(struct igb_adapter *adapter)
3996 {
3997         struct e1000_hw *hw = &adapter->hw;
3998         bool link_active = FALSE;
3999
4000         /* get_link_status is set on LSC (link status) interrupt or
4001          * rx sequence error interrupt.  get_link_status will stay
4002          * false until the e1000_check_for_link establishes link
4003          * for copper adapters ONLY
4004          */
4005         switch (hw->phy.media_type) {
4006         case e1000_media_type_copper:
4007                 if (!hw->mac.get_link_status)
4008                         return true;
4009         case e1000_media_type_internal_serdes:
4010                 e1000_check_for_link(hw);
4011                 link_active = !hw->mac.get_link_status;
4012                 break;
4013         case e1000_media_type_unknown:
4014         default:
4015                 break;
4016         }
4017
4018         return link_active;
4019 }
4020
4021 /**
4022  * igb_watchdog - Timer Call-back
4023  * @data: pointer to adapter cast into an unsigned long
4024  **/
4025 static void igb_watchdog(unsigned long data)
4026 {
4027         struct igb_adapter *adapter = (struct igb_adapter *)data;
4028         /* Do the rest outside of interrupt context */
4029         schedule_work(&adapter->watchdog_task);
4030 }
4031
4032 static void igb_watchdog_task(struct work_struct *work)
4033 {
4034         struct igb_adapter *adapter = container_of(work,
4035                                                    struct igb_adapter,
4036                                                    watchdog_task);
4037         struct e1000_hw *hw = &adapter->hw;
4038         struct net_device *netdev = adapter->netdev;
4039         u32 link;
4040         int i;
4041         u32 thstat, ctrl_ext;
4042
4043
4044         link = igb_has_link(adapter);
4045         if (link) {
4046                 /* Cancel scheduled suspend requests. */
4047                 pm_runtime_resume(netdev->dev.parent);
4048
4049                 if (!netif_carrier_ok(netdev)) {
4050                         u32 ctrl;
4051                         e1000_get_speed_and_duplex(hw,
4052                                                    &adapter->link_speed,
4053                                                    &adapter->link_duplex);
4054
4055                         ctrl = E1000_READ_REG(hw, E1000_CTRL);
4056                         /* Links status message must follow this format */
4057                         printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
4058                                  "Flow Control: %s\n",
4059                                netdev->name,
4060                                adapter->link_speed,
4061                                adapter->link_duplex == FULL_DUPLEX ?
4062                                  "Full Duplex" : "Half Duplex",
4063                                ((ctrl & E1000_CTRL_TFCE) &&
4064                                 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX":
4065                                ((ctrl & E1000_CTRL_RFCE) ?  "RX" :
4066                                ((ctrl & E1000_CTRL_TFCE) ?  "TX" : "None")));
4067                         /* adjust timeout factor according to speed/duplex */
4068                         adapter->tx_timeout_factor = 1;
4069                         switch (adapter->link_speed) {
4070                         case SPEED_10:
4071                                 adapter->tx_timeout_factor = 14;
4072                                 break;
4073                         case SPEED_100:
4074                                 /* maybe add some timeout factor ? */
4075                                 break;
4076                         }
4077
4078                         netif_carrier_on(netdev);
4079                         netif_tx_wake_all_queues(netdev);
4080
4081                         igb_ping_all_vfs(adapter);
4082 #ifdef IFLA_VF_MAX
4083                         igb_check_vf_rate_limit(adapter);
4084 #endif /* IFLA_VF_MAX */
4085
4086                         /* link state has changed, schedule phy info update */
4087                         if (!test_bit(__IGB_DOWN, &adapter->state))
4088                                 mod_timer(&adapter->phy_info_timer,
4089                                           round_jiffies(jiffies + 2 * HZ));
4090                 }
4091         } else {
4092                 if (netif_carrier_ok(netdev)) {
4093                         adapter->link_speed = 0;
4094                         adapter->link_duplex = 0;
4095                         /* check for thermal sensor event on i350 */
4096                         if (hw->mac.type == e1000_i350) {
4097                                 thstat = E1000_READ_REG(hw, E1000_THSTAT);
4098                                 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
4099                                 if ((hw->phy.media_type ==
4100                                         e1000_media_type_copper) &&
4101                                         !(ctrl_ext &
4102                                         E1000_CTRL_EXT_LINK_MODE_SGMII)) {
4103                                         if (thstat & E1000_THSTAT_PWR_DOWN) {
4104                                                 printk(KERN_ERR "igb: %s The "
4105                                                 "network adapter was stopped "
4106                                                 "because it overheated.\n",
4107                                                 netdev->name);
4108                                         }
4109                                         if (thstat & E1000_THSTAT_LINK_THROTTLE) {
4110                                                 printk(KERN_INFO
4111                                                         "igb: %s The network "
4112                                                         "adapter supported "
4113                                                         "link speed "
4114                                                         "was downshifted "
4115                                                         "because it "
4116                                                         "overheated.\n",
4117                                                         netdev->name);
4118                                         }
4119                                 }
4120                         }
4121
4122                         /* Links status message must follow this format */
4123                         printk(KERN_INFO "igb: %s NIC Link is Down\n",
4124                                netdev->name);
4125                         netif_carrier_off(netdev);
4126                         netif_tx_stop_all_queues(netdev);
4127
4128                         igb_ping_all_vfs(adapter);
4129
4130                         /* link state has changed, schedule phy info update */
4131                         if (!test_bit(__IGB_DOWN, &adapter->state))
4132                                 mod_timer(&adapter->phy_info_timer,
4133                                           round_jiffies(jiffies + 2 * HZ));
4134
4135                         pm_schedule_suspend(netdev->dev.parent,
4136                                             MSEC_PER_SEC * 5);
4137                 }
4138         }
4139
4140         igb_update_stats(adapter);
4141
4142         for (i = 0; i < adapter->num_tx_queues; i++) {
4143                 struct igb_ring *tx_ring = adapter->tx_ring[i];
4144                 if (!netif_carrier_ok(netdev)) {
4145                         /* We've lost link, so the controller stops DMA,
4146                          * but we've got queued Tx work that's never going
4147                          * to get done, so reset controller to flush Tx.
4148                          * (Do the reset outside of interrupt context). */
4149                         if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
4150                                 adapter->tx_timeout_count++;
4151                                 schedule_work(&adapter->reset_task);
4152                                 /* return immediately since reset is imminent */
4153                                 return;
4154                         }
4155                 }
4156
4157                 /* Force detection of hung controller every watchdog period */
4158                 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
4159         }
4160
4161         /* Cause software interrupt to ensure rx ring is cleaned */
4162         if (adapter->msix_entries) {
4163                 u32 eics = 0;
4164                 for (i = 0; i < adapter->num_q_vectors; i++)
4165                         eics |= adapter->q_vector[i]->eims_value;
4166                 E1000_WRITE_REG(hw, E1000_EICS, eics);
4167         } else {
4168                 E1000_WRITE_REG(hw, E1000_ICS, E1000_ICS_RXDMT0);
4169         }
4170
4171         igb_spoof_check(adapter);
4172
4173         /* Reset the timer */
4174         if (!test_bit(__IGB_DOWN, &adapter->state))
4175                 mod_timer(&adapter->watchdog_timer,
4176                           round_jiffies(jiffies + 2 * HZ));
4177 }
4178
4179 static void igb_dma_err_task(struct work_struct *work)
4180 {
4181         struct igb_adapter *adapter = container_of(work,
4182                                                    struct igb_adapter,
4183                                                    dma_err_task);
4184         int vf;
4185         struct e1000_hw *hw = &adapter->hw;
4186         struct net_device *netdev = adapter->netdev;
4187         u32 hgptc;
4188         u32 ciaa, ciad;
4189
4190         hgptc = E1000_READ_REG(hw, E1000_HGPTC);
4191         if (hgptc) /* If incrementing then no need for the check below */
4192                 goto dma_timer_reset;
4193         /*
4194          * Check to see if a bad DMA write target from an errant or
4195          * malicious VF has caused a PCIe error.  If so then we can
4196          * issue a VFLR to the offending VF(s) and then resume without
4197          * requesting a full slot reset.
4198          */
4199
4200         for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4201                 ciaa = (vf << 16) | 0x80000000;
4202                 /* 32 bit read so align, we really want status at offset 6 */
4203                 ciaa |= PCI_COMMAND;
4204                 E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4205                 ciad = E1000_READ_REG(hw, E1000_CIAD);
4206                 ciaa &= 0x7FFFFFFF;
4207                 /* disable debug mode asap after reading data */
4208                 E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4209                 /* Get the upper 16 bits which will be the PCI status reg */
4210                 ciad >>= 16;
4211                 if (ciad & (PCI_STATUS_REC_MASTER_ABORT |
4212                             PCI_STATUS_REC_TARGET_ABORT |
4213                             PCI_STATUS_SIG_SYSTEM_ERROR)) {
4214                         netdev_err(netdev, "VF %d suffered error\n", vf);
4215                         /* Issue VFLR */
4216                         ciaa = (vf << 16) | 0x80000000;
4217                         ciaa |= 0xA8;
4218                         E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4219                         ciad = 0x00008000;  /* VFLR */
4220                         E1000_WRITE_REG(hw, E1000_CIAD, ciad);
4221                         ciaa &= 0x7FFFFFFF;
4222                         E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4223                 }
4224         }
4225 dma_timer_reset:
4226         /* Reset the timer */
4227         if (!test_bit(__IGB_DOWN, &adapter->state))
4228                 mod_timer(&adapter->dma_err_timer,
4229                           round_jiffies(jiffies + HZ / 10));
4230 }
4231
4232 /**
4233  * igb_dma_err_timer - Timer Call-back
4234  * @data: pointer to adapter cast into an unsigned long
4235  **/
4236 static void igb_dma_err_timer(unsigned long data)
4237 {
4238         struct igb_adapter *adapter = (struct igb_adapter *)data;
4239         /* Do the rest outside of interrupt context */
4240         schedule_work(&adapter->dma_err_task);
4241 }
4242
4243 enum latency_range {
4244         lowest_latency = 0,
4245         low_latency = 1,
4246         bulk_latency = 2,
4247         latency_invalid = 255
4248 };
4249
4250 /**
4251  * igb_update_ring_itr - update the dynamic ITR value based on packet size
4252  *
4253  *      Stores a new ITR value based on strictly on packet size.  This
4254  *      algorithm is less sophisticated than that used in igb_update_itr,
4255  *      due to the difficulty of synchronizing statistics across multiple
4256  *      receive rings.  The divisors and thresholds used by this function
4257  *      were determined based on theoretical maximum wire speed and testing
4258  *      data, in order to minimize response time while increasing bulk
4259  *      throughput.
4260  *      This functionality is controlled by the InterruptThrottleRate module
4261  *      parameter (see igb_param.c)
4262  *      NOTE:  This function is called only when operating in a multiqueue
4263  *             receive environment.
4264  * @q_vector: pointer to q_vector
4265  **/
4266 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
4267 {
4268         int new_val = q_vector->itr_val;
4269         int avg_wire_size = 0;
4270         struct igb_adapter *adapter = q_vector->adapter;
4271         unsigned int packets;
4272
4273         /* For non-gigabit speeds, just fix the interrupt rate at 4000
4274          * ints/sec - ITR timer value of 120 ticks.
4275          */
4276         if (adapter->link_speed != SPEED_1000) {
4277                 new_val = IGB_4K_ITR;
4278                 goto set_itr_val;
4279         }
4280
4281         packets = q_vector->rx.total_packets;
4282         if (packets)
4283                 avg_wire_size = q_vector->rx.total_bytes / packets;
4284
4285         packets = q_vector->tx.total_packets;
4286         if (packets)
4287                 avg_wire_size = max_t(u32, avg_wire_size,
4288                                       q_vector->tx.total_bytes / packets);
4289
4290         /* if avg_wire_size isn't set no work was done */
4291         if (!avg_wire_size)
4292                 goto clear_counts;
4293
4294         /* Add 24 bytes to size to account for CRC, preamble, and gap */
4295         avg_wire_size += 24;
4296
4297         /* Don't starve jumbo frames */
4298         avg_wire_size = min(avg_wire_size, 3000);
4299
4300         /* Give a little boost to mid-size frames */
4301         if ((avg_wire_size > 300) && (avg_wire_size < 1200))
4302                 new_val = avg_wire_size / 3;
4303         else
4304                 new_val = avg_wire_size / 2;
4305
4306         /* conservative mode (itr 3) eliminates the lowest_latency setting */
4307         if (new_val < IGB_20K_ITR &&
4308             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4309              (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4310                 new_val = IGB_20K_ITR;
4311
4312 set_itr_val:
4313         if (new_val != q_vector->itr_val) {
4314                 q_vector->itr_val = new_val;
4315                 q_vector->set_itr = 1;
4316         }
4317 clear_counts:
4318         q_vector->rx.total_bytes = 0;
4319         q_vector->rx.total_packets = 0;
4320         q_vector->tx.total_bytes = 0;
4321         q_vector->tx.total_packets = 0;
4322 }
4323
4324 /**
4325  * igb_update_itr - update the dynamic ITR value based on statistics
4326  *      Stores a new ITR value based on packets and byte
4327  *      counts during the last interrupt.  The advantage of per interrupt
4328  *      computation is faster updates and more accurate ITR for the current
4329  *      traffic pattern.  Constants in this function were computed
4330  *      based on theoretical maximum wire speed and thresholds were set based
4331  *      on testing data as well as attempting to minimize response time
4332  *      while increasing bulk throughput.
4333  *      this functionality is controlled by the InterruptThrottleRate module
4334  *      parameter (see igb_param.c)
4335  *      NOTE:  These calculations are only valid when operating in a single-
4336  *             queue environment.
4337  * @q_vector: pointer to q_vector
4338  * @ring_container: ring info to update the itr for
4339  **/
4340 static void igb_update_itr(struct igb_q_vector *q_vector,
4341                            struct igb_ring_container *ring_container)
4342 {
4343         unsigned int packets = ring_container->total_packets;
4344         unsigned int bytes = ring_container->total_bytes;
4345         u8 itrval = ring_container->itr;
4346
4347         /* no packets, exit with status unchanged */
4348         if (packets == 0)
4349                 return;
4350
4351         switch (itrval) {
4352         case lowest_latency:
4353                 /* handle TSO and jumbo frames */
4354                 if (bytes/packets > 8000)
4355                         itrval = bulk_latency;
4356                 else if ((packets < 5) && (bytes > 512))
4357                         itrval = low_latency;
4358                 break;
4359         case low_latency:  /* 50 usec aka 20000 ints/s */
4360                 if (bytes > 10000) {
4361                         /* this if handles the TSO accounting */
4362                         if (bytes/packets > 8000) {
4363                                 itrval = bulk_latency;
4364                         } else if ((packets < 10) || ((bytes/packets) > 1200)) {
4365                                 itrval = bulk_latency;
4366                         } else if ((packets > 35)) {
4367                                 itrval = lowest_latency;
4368                         }
4369                 } else if (bytes/packets > 2000) {
4370                         itrval = bulk_latency;
4371                 } else if (packets <= 2 && bytes < 512) {
4372                         itrval = lowest_latency;
4373                 }
4374                 break;
4375         case bulk_latency: /* 250 usec aka 4000 ints/s */
4376                 if (bytes > 25000) {
4377                         if (packets > 35)
4378                                 itrval = low_latency;
4379                 } else if (bytes < 1500) {
4380                         itrval = low_latency;
4381                 }
4382                 break;
4383         }
4384
4385         /* clear work counters since we have the values we need */
4386         ring_container->total_bytes = 0;
4387         ring_container->total_packets = 0;
4388
4389         /* write updated itr to ring container */
4390         ring_container->itr = itrval;
4391 }
4392
4393 static void igb_set_itr(struct igb_q_vector *q_vector)
4394 {
4395         struct igb_adapter *adapter = q_vector->adapter;
4396         u32 new_itr = q_vector->itr_val;
4397         u8 current_itr = 0;
4398
4399         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4400         if (adapter->link_speed != SPEED_1000) {
4401                 current_itr = 0;
4402                 new_itr = IGB_4K_ITR;
4403                 goto set_itr_now;
4404         }
4405
4406         igb_update_itr(q_vector, &q_vector->tx);
4407         igb_update_itr(q_vector, &q_vector->rx);
4408
4409         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
4410
4411         /* conservative mode (itr 3) eliminates the lowest_latency setting */
4412         if (current_itr == lowest_latency &&
4413             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4414              (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4415                 current_itr = low_latency;
4416
4417         switch (current_itr) {
4418         /* counts and packets in update_itr are dependent on these numbers */
4419         case lowest_latency:
4420                 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
4421                 break;
4422         case low_latency:
4423                 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
4424                 break;
4425         case bulk_latency:
4426                 new_itr = IGB_4K_ITR;  /* 4,000 ints/sec */
4427                 break;
4428         default:
4429                 break;
4430         }
4431
4432 set_itr_now:
4433         if (new_itr != q_vector->itr_val) {
4434                 /* this attempts to bias the interrupt rate towards Bulk
4435                  * by adding intermediate steps when interrupt rate is
4436                  * increasing */
4437                 new_itr = new_itr > q_vector->itr_val ?
4438                              max((new_itr * q_vector->itr_val) /
4439                                  (new_itr + (q_vector->itr_val >> 2)),
4440                                  new_itr) :
4441                              new_itr;
4442                 /* Don't write the value here; it resets the adapter's
4443                  * internal timer, and causes us to delay far longer than
4444                  * we should between interrupts.  Instead, we write the ITR
4445                  * value at the beginning of the next interrupt so the timing
4446                  * ends up being correct.
4447                  */
4448                 q_vector->itr_val = new_itr;
4449                 q_vector->set_itr = 1;
4450         }
4451 }
4452
4453 void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
4454                      u32 type_tucmd, u32 mss_l4len_idx)
4455 {
4456         struct e1000_adv_tx_context_desc *context_desc;
4457         u16 i = tx_ring->next_to_use;
4458
4459         context_desc = IGB_TX_CTXTDESC(tx_ring, i);
4460
4461         i++;
4462         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
4463
4464         /* set bits to identify this as an advanced context descriptor */
4465         type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
4466
4467         /* For 82575, context index must be unique per ring. */
4468         if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4469                 mss_l4len_idx |= tx_ring->reg_idx << 4;
4470
4471         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
4472         context_desc->seqnum_seed       = 0;
4473         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
4474         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
4475 }
4476
4477 static int igb_tso(struct igb_ring *tx_ring,
4478                    struct igb_tx_buffer *first,
4479                    u8 *hdr_len)
4480 {
4481 #ifdef NETIF_F_TSO
4482         struct sk_buff *skb = first->skb;
4483         u32 vlan_macip_lens, type_tucmd;
4484         u32 mss_l4len_idx, l4len;
4485
4486         if (!skb_is_gso(skb))
4487 #endif /* NETIF_F_TSO */
4488                 return 0;
4489 #ifdef NETIF_F_TSO
4490
4491         if (skb_header_cloned(skb)) {
4492                 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
4493                 if (err)
4494                         return err;
4495         }
4496
4497         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4498         type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
4499
4500         if (first->protocol == __constant_htons(ETH_P_IP)) {
4501                 struct iphdr *iph = ip_hdr(skb);
4502                 iph->tot_len = 0;
4503                 iph->check = 0;
4504                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4505                                                          iph->daddr, 0,
4506                                                          IPPROTO_TCP,
4507                                                          0);
4508                 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4509                 first->tx_flags |= IGB_TX_FLAGS_TSO |
4510                                    IGB_TX_FLAGS_CSUM |
4511                                    IGB_TX_FLAGS_IPV4;
4512 #ifdef NETIF_F_TSO6
4513         } else if (skb_is_gso_v6(skb)) {
4514                 ipv6_hdr(skb)->payload_len = 0;
4515                 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4516                                                        &ipv6_hdr(skb)->daddr,
4517                                                        0, IPPROTO_TCP, 0);
4518                 first->tx_flags |= IGB_TX_FLAGS_TSO |
4519                                    IGB_TX_FLAGS_CSUM;
4520 #endif
4521         }
4522
4523         /* compute header lengths */
4524         l4len = tcp_hdrlen(skb);
4525         *hdr_len = skb_transport_offset(skb) + l4len;
4526
4527         /* update gso size and bytecount with header size */
4528         first->gso_segs = skb_shinfo(skb)->gso_segs;
4529         first->bytecount += (first->gso_segs - 1) * *hdr_len;
4530
4531         /* MSS L4LEN IDX */
4532         mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
4533         mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
4534
4535         /* VLAN MACLEN IPLEN */
4536         vlan_macip_lens = skb_network_header_len(skb);
4537         vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4538         vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4539
4540         igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4541
4542         return 1;
4543 #endif  /* NETIF_F_TSO */
4544 }
4545
4546 static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
4547 {
4548         struct sk_buff *skb = first->skb;
4549         u32 vlan_macip_lens = 0;
4550         u32 mss_l4len_idx = 0;
4551         u32 type_tucmd = 0;
4552
4553         if (skb->ip_summed != CHECKSUM_PARTIAL) {
4554                 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
4555                         return;
4556         } else {
4557                 u8 l4_hdr = 0;
4558                 switch (first->protocol) {
4559                 case __constant_htons(ETH_P_IP):
4560                         vlan_macip_lens |= skb_network_header_len(skb);
4561                         type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4562                         l4_hdr = ip_hdr(skb)->protocol;
4563                         break;
4564 #ifdef NETIF_F_IPV6_CSUM
4565                 case __constant_htons(ETH_P_IPV6):
4566                         vlan_macip_lens |= skb_network_header_len(skb);
4567                         l4_hdr = ipv6_hdr(skb)->nexthdr;
4568                         break;
4569 #endif
4570                 default:
4571                         if (unlikely(net_ratelimit())) {
4572                                 dev_warn(tx_ring->dev,
4573                                  "partial checksum but proto=%x!\n",
4574                                  first->protocol);
4575                         }
4576                         break;
4577                 }
4578
4579                 switch (l4_hdr) {
4580                 case IPPROTO_TCP:
4581                         type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4582                         mss_l4len_idx = tcp_hdrlen(skb) <<
4583                                         E1000_ADVTXD_L4LEN_SHIFT;
4584                         break;
4585 #ifdef HAVE_SCTP
4586                 case IPPROTO_SCTP:
4587                         type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4588                         mss_l4len_idx = sizeof(struct sctphdr) <<
4589                                         E1000_ADVTXD_L4LEN_SHIFT;
4590                         break;
4591 #endif
4592                 case IPPROTO_UDP:
4593                         mss_l4len_idx = sizeof(struct udphdr) <<
4594                                         E1000_ADVTXD_L4LEN_SHIFT;
4595                         break;
4596                 default:
4597                         if (unlikely(net_ratelimit())) {
4598                                 dev_warn(tx_ring->dev,
4599                                  "partial checksum but l4 proto=%x!\n",
4600                                  l4_hdr);
4601                         }
4602                         break;
4603                 }
4604
4605                 /* update TX checksum flag */
4606                 first->tx_flags |= IGB_TX_FLAGS_CSUM;
4607         }
4608
4609         vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4610         vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4611
4612         igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4613 }
4614
4615 static __le32 igb_tx_cmd_type(u32 tx_flags)
4616 {
4617         /* set type for advanced descriptor with frame checksum insertion */
4618         __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA |
4619                                       E1000_ADVTXD_DCMD_IFCS |
4620                                       E1000_ADVTXD_DCMD_DEXT);
4621
4622         /* set HW vlan bit if vlan is present */
4623         if (tx_flags & IGB_TX_FLAGS_VLAN)
4624                 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE);
4625
4626         /* set timestamp bit if present */
4627         if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4628                 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP);
4629
4630         /* set segmentation bits for TSO */
4631         if (tx_flags & IGB_TX_FLAGS_TSO)
4632                 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE);
4633
4634         return cmd_type;
4635 }
4636
4637 static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
4638                                  union e1000_adv_tx_desc *tx_desc,
4639                                  u32 tx_flags, unsigned int paylen)
4640 {
4641         u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
4642
4643         /* 82575 requires a unique index per ring if any offload is enabled */
4644         if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) &&
4645             test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4646                 olinfo_status |= tx_ring->reg_idx << 4;
4647
4648         /* insert L4 checksum */
4649         if (tx_flags & IGB_TX_FLAGS_CSUM) {
4650                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4651
4652                 /* insert IPv4 checksum */
4653                 if (tx_flags & IGB_TX_FLAGS_IPV4)
4654                         olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4655         }
4656
4657         tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4658 }
4659
4660 /*
4661  * The largest size we can write to the descriptor is 65535.  In order to
4662  * maintain a power of two alignment we have to limit ourselves to 32K.
4663  */
4664 #define IGB_MAX_TXD_PWR 15
4665 #define IGB_MAX_DATA_PER_TXD    (1<<IGB_MAX_TXD_PWR)
4666
4667 static void igb_tx_map(struct igb_ring *tx_ring,
4668                        struct igb_tx_buffer *first,
4669                        const u8 hdr_len)
4670 {
4671         struct sk_buff *skb = first->skb;
4672         struct igb_tx_buffer *tx_buffer;
4673         union e1000_adv_tx_desc *tx_desc;
4674         dma_addr_t dma;
4675 #ifdef MAX_SKB_FRAGS
4676         struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
4677         unsigned int data_len = skb->data_len;
4678 #endif
4679         unsigned int size = skb_headlen(skb);
4680         unsigned int paylen = skb->len - hdr_len;
4681         __le32 cmd_type;
4682         u32 tx_flags = first->tx_flags;
4683         u16 i = tx_ring->next_to_use;
4684
4685         tx_desc = IGB_TX_DESC(tx_ring, i);
4686
4687         igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen);
4688         cmd_type = igb_tx_cmd_type(tx_flags);
4689
4690         dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4691         if (dma_mapping_error(tx_ring->dev, dma))
4692                 goto dma_error;
4693
4694         /* record length, and DMA address */
4695         dma_unmap_len_set(first, len, size);
4696         dma_unmap_addr_set(first, dma, dma);
4697         tx_desc->read.buffer_addr = cpu_to_le64(dma);
4698
4699 #ifdef MAX_SKB_FRAGS
4700         for (;;) {
4701 #endif
4702                 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
4703                         tx_desc->read.cmd_type_len =
4704                                 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD);
4705
4706                         i++;
4707                         tx_desc++;
4708                         if (i == tx_ring->count) {
4709                                 tx_desc = IGB_TX_DESC(tx_ring, 0);
4710                                 i = 0;
4711                         }
4712
4713                         dma += IGB_MAX_DATA_PER_TXD;
4714                         size -= IGB_MAX_DATA_PER_TXD;
4715
4716                         tx_desc->read.olinfo_status = 0;
4717                         tx_desc->read.buffer_addr = cpu_to_le64(dma);
4718                 }
4719
4720 #ifdef MAX_SKB_FRAGS
4721                 if (likely(!data_len))
4722                         break;
4723
4724                 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
4725
4726                 i++;
4727                 tx_desc++;
4728                 if (i == tx_ring->count) {
4729                         tx_desc = IGB_TX_DESC(tx_ring, 0);
4730                         i = 0;
4731                 }
4732
4733                 size = skb_frag_size(frag);
4734                 data_len -= size;
4735
4736                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
4737                                 DMA_TO_DEVICE);
4738                 if (dma_mapping_error(tx_ring->dev, dma))
4739                         goto dma_error;
4740
4741                 tx_buffer = &tx_ring->tx_buffer_info[i];
4742                 dma_unmap_len_set(tx_buffer, len, size);
4743                 dma_unmap_addr_set(tx_buffer, dma, dma);
4744
4745                 tx_desc->read.olinfo_status = 0;
4746                 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4747
4748                 frag++;
4749         }
4750
4751 #endif /* MAX_SKB_FRAGS */
4752 #ifdef CONFIG_BQL
4753         netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
4754 #endif /* CONFIG_BQL */
4755
4756         /* write last descriptor with RS and EOP bits */
4757         cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD);
4758         tx_desc->read.cmd_type_len = cmd_type;
4759
4760         /* set the timestamp */
4761         first->time_stamp = jiffies;
4762
4763         /*
4764          * Force memory writes to complete before letting h/w know there
4765          * are new descriptors to fetch.  (Only applicable for weak-ordered
4766          * memory model archs, such as IA-64).
4767          *
4768          * We also need this memory barrier to make certain all of the
4769          * status bits have been updated before next_to_watch is written.
4770          */
4771         wmb();
4772
4773         /* set next_to_watch value indicating a packet is present */
4774         first->next_to_watch = tx_desc;
4775
4776         i++;
4777         if (i == tx_ring->count)
4778                 i = 0;
4779
4780         tx_ring->next_to_use = i;
4781
4782         writel(i, tx_ring->tail);
4783
4784         /* we need this if more than one processor can write to our tail
4785          * at a time, it syncronizes IO on IA64/Altix systems */
4786         mmiowb();
4787
4788         return;
4789
4790 dma_error:
4791         dev_err(tx_ring->dev, "TX DMA map failed\n");
4792
4793         /* clear dma mappings for failed tx_buffer_info map */
4794         for (;;) {
4795                 tx_buffer= &tx_ring->tx_buffer_info[i];
4796                 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer);
4797                 if (tx_buffer == first)
4798                         break;
4799                 if (i == 0)
4800                         i = tx_ring->count;
4801                 i--;
4802         }
4803
4804         tx_ring->next_to_use = i;
4805 }
4806
4807 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4808 {
4809         struct net_device *netdev = netdev_ring(tx_ring);
4810
4811         if (netif_is_multiqueue(netdev))
4812                 netif_stop_subqueue(netdev, ring_queue_index(tx_ring));
4813         else
4814                 netif_stop_queue(netdev);
4815
4816         /* Herbert's original patch had:
4817          *  smp_mb__after_netif_stop_queue();
4818          * but since that doesn't exist yet, just open code it. */
4819         smp_mb();
4820
4821         /* We need to check again in a case another CPU has just
4822          * made room available. */
4823         if (igb_desc_unused(tx_ring) < size)
4824                 return -EBUSY;
4825
4826         /* A reprieve! */
4827         if (netif_is_multiqueue(netdev))
4828                 netif_wake_subqueue(netdev, ring_queue_index(tx_ring));
4829         else
4830                 netif_wake_queue(netdev);
4831
4832         tx_ring->tx_stats.restart_queue++;
4833
4834         return 0;
4835 }
4836
4837 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4838 {
4839         if (igb_desc_unused(tx_ring) >= size)
4840                 return 0;
4841         return __igb_maybe_stop_tx(tx_ring, size);
4842 }
4843
4844 netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4845                                 struct igb_ring *tx_ring)
4846 {
4847         struct igb_tx_buffer *first;
4848         int tso;
4849         u32 tx_flags = 0;
4850         __be16 protocol = vlan_get_protocol(skb);
4851         u8 hdr_len = 0;
4852
4853         /* need: 1 descriptor per page,
4854          *       + 2 desc gap to keep tail from touching head,
4855          *       + 1 desc for skb->data,
4856          *       + 1 desc for context descriptor,
4857          * otherwise try next time */
4858         if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
4859                 /* this is a hard error */
4860                 return NETDEV_TX_BUSY;
4861         }
4862
4863         /* record the location of the first descriptor for this packet */
4864         first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4865         first->skb = skb;
4866         first->bytecount = skb->len;
4867         first->gso_segs = 1;
4868
4869 #ifdef HAVE_HW_TIME_STAMP
4870 #ifdef SKB_SHARED_TX_IS_UNION
4871         if (unlikely(skb_shinfo(skb)->tx_flags.flags & SKBTX_HW_TSTAMP)) {
4872                 skb_shinfo(skb)->tx_flags.flags |= SKBTX_IN_PROGRESS;
4873                 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4874         }
4875 #else
4876         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4877                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4878                 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4879         }
4880 #endif
4881
4882 #endif
4883         if (vlan_tx_tag_present(skb)) {
4884                 tx_flags |= IGB_TX_FLAGS_VLAN;
4885                 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4886         }
4887
4888         /* record initial flags and protocol */
4889         first->tx_flags = tx_flags;
4890         first->protocol = protocol;
4891
4892         tso = igb_tso(tx_ring, first, &hdr_len);
4893         if (tso < 0)
4894                 goto out_drop;
4895         else if (!tso)
4896                 igb_tx_csum(tx_ring, first);
4897
4898         igb_tx_map(tx_ring, first, hdr_len);
4899
4900 #ifndef HAVE_TRANS_START_IN_QUEUE
4901         netdev_ring(tx_ring)->trans_start = jiffies;
4902
4903 #endif
4904         /* Make sure there is space in the ring for the next send. */
4905         igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
4906
4907         return NETDEV_TX_OK;
4908
4909 out_drop:
4910         igb_unmap_and_free_tx_resource(tx_ring, first);
4911
4912         return NETDEV_TX_OK;
4913 }
4914
4915 #ifdef HAVE_TX_MQ
4916 static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4917                                                     struct sk_buff *skb)
4918 {
4919         unsigned int r_idx = skb->queue_mapping;
4920
4921         if (r_idx >= adapter->num_tx_queues)
4922                 r_idx = r_idx % adapter->num_tx_queues;
4923
4924         return adapter->tx_ring[r_idx];
4925 }
4926 #else
4927 #define igb_tx_queue_mapping(_adapter, _skb) (_adapter)->tx_ring[0]
4928 #endif
4929
4930 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4931                                   struct net_device *netdev)
4932 {
4933         struct igb_adapter *adapter = netdev_priv(netdev);
4934
4935         if (test_bit(__IGB_DOWN, &adapter->state)) {
4936                 dev_kfree_skb_any(skb);
4937                 return NETDEV_TX_OK;
4938         }
4939
4940         if (skb->len <= 0) {
4941                 dev_kfree_skb_any(skb);
4942                 return NETDEV_TX_OK;
4943         }
4944
4945         /*
4946          * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4947          * in order to meet this minimum size requirement.
4948          */
4949         if (skb->len < 17) {
4950                 if (skb_padto(skb, 17))
4951                         return NETDEV_TX_OK;
4952                 skb->len = 17;
4953         }
4954
4955         return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
4956 }
4957
4958 /**
4959  * igb_tx_timeout - Respond to a Tx Hang
4960  * @netdev: network interface device structure
4961  **/
4962 static void igb_tx_timeout(struct net_device *netdev)
4963 {
4964         struct igb_adapter *adapter = netdev_priv(netdev);
4965         struct e1000_hw *hw = &adapter->hw;
4966
4967         /* Do the reset outside of interrupt context */
4968         adapter->tx_timeout_count++;
4969
4970         if (hw->mac.type >= e1000_82580)
4971                 hw->dev_spec._82575.global_device_reset = true;
4972
4973         schedule_work(&adapter->reset_task);
4974         E1000_WRITE_REG(hw, E1000_EICS,
4975                         (adapter->eims_enable_mask & ~adapter->eims_other));
4976 }
4977
4978 static void igb_reset_task(struct work_struct *work)
4979 {
4980         struct igb_adapter *adapter;
4981         adapter = container_of(work, struct igb_adapter, reset_task);
4982
4983         igb_reinit_locked(adapter);
4984 }
4985
4986 /**
4987  * igb_get_stats - Get System Network Statistics
4988  * @netdev: network interface device structure
4989  *
4990  * Returns the address of the device statistics structure.
4991  * The statistics are updated here and also from the timer callback.
4992  **/
4993 static struct net_device_stats *igb_get_stats(struct net_device *netdev)
4994 {
4995         struct igb_adapter *adapter = netdev_priv(netdev);
4996
4997         if (!test_bit(__IGB_RESETTING, &adapter->state))
4998                 igb_update_stats(adapter);
4999
5000 #ifdef HAVE_NETDEV_STATS_IN_NETDEV
5001         /* only return the current stats */
5002         return &netdev->stats;
5003 #else
5004         /* only return the current stats */
5005         return &adapter->net_stats;
5006 #endif /* HAVE_NETDEV_STATS_IN_NETDEV */
5007 }
5008
5009 /**
5010  * igb_change_mtu - Change the Maximum Transfer Unit
5011  * @netdev: network interface device structure
5012  * @new_mtu: new value for maximum frame size
5013  *
5014  * Returns 0 on success, negative on failure
5015  **/
5016 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
5017 {
5018         struct igb_adapter *adapter = netdev_priv(netdev);
5019         struct pci_dev *pdev = adapter->pdev;
5020         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
5021 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
5022         u32 rx_buffer_len, i;
5023 #endif
5024
5025         if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
5026                 dev_err(pci_dev_to_dev(pdev), "Invalid MTU setting\n");
5027                 return -EINVAL;
5028         }
5029
5030 #define MAX_STD_JUMBO_FRAME_SIZE 9238
5031         if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
5032                 dev_err(pci_dev_to_dev(pdev), "MTU > 9216 not supported.\n");
5033                 return -EINVAL;
5034         }
5035
5036         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
5037                 usleep_range(1000, 2000);
5038
5039         /* igb_down has a dependency on max_frame_size */
5040         adapter->max_frame_size = max_frame;
5041
5042 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
5043 #ifdef IGB_PER_PKT_TIMESTAMP
5044         if (adapter->hw.mac.type >= e1000_82580)
5045                 max_frame += IGB_TS_HDR_LEN;
5046
5047 #endif
5048         /*
5049          * RLPML prevents us from receiving a frame larger than max_frame so
5050          * it is safe to just set the rx_buffer_len to max_frame without the
5051          * risk of an skb over panic.
5052          */
5053         if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
5054                 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
5055         else
5056                 rx_buffer_len = max_frame;
5057
5058 #endif
5059         if (netif_running(netdev))
5060                 igb_down(adapter);
5061
5062         dev_info(pci_dev_to_dev(pdev), "changing MTU from %d to %d\n",
5063                 netdev->mtu, new_mtu);
5064         netdev->mtu = new_mtu;
5065
5066 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
5067         for (i = 0; i < adapter->num_rx_queues; i++)
5068                 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
5069
5070 #endif
5071         if (netif_running(netdev))
5072                 igb_up(adapter);
5073         else
5074                 igb_reset(adapter);
5075
5076         clear_bit(__IGB_RESETTING, &adapter->state);
5077
5078         return 0;
5079 }
5080
5081 /**
5082  * igb_update_stats - Update the board statistics counters
5083  * @adapter: board private structure
5084  **/
5085
5086 void igb_update_stats(struct igb_adapter *adapter)
5087 {
5088 #ifdef HAVE_NETDEV_STATS_IN_NETDEV
5089         struct net_device_stats *net_stats = &adapter->netdev->stats;
5090 #else
5091         struct net_device_stats *net_stats = &adapter->net_stats;
5092 #endif /* HAVE_NETDEV_STATS_IN_NETDEV */
5093         struct e1000_hw *hw = &adapter->hw;
5094 #ifdef HAVE_PCI_ERS
5095         struct pci_dev *pdev = adapter->pdev;
5096 #endif
5097         u32 reg, mpc;
5098         u16 phy_tmp;
5099         int i;
5100         u64 bytes, packets;
5101 #ifndef IGB_NO_LRO
5102         u32 flushed = 0, coal = 0, recycled = 0;
5103         struct igb_q_vector *q_vector;
5104 #endif
5105
5106 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
5107
5108         /*
5109          * Prevent stats update while adapter is being reset, or if the pci
5110          * connection is down.
5111          */
5112         if (adapter->link_speed == 0)
5113                 return;
5114 #ifdef HAVE_PCI_ERS
5115         if (pci_channel_offline(pdev))
5116                 return;
5117
5118 #endif
5119 #ifndef IGB_NO_LRO
5120         for (i = 0; i < adapter->num_q_vectors; i++) {
5121                 q_vector = adapter->q_vector[i];
5122                 if (!q_vector || !q_vector->lrolist)
5123                         continue;
5124                 flushed += q_vector->lrolist->stats.flushed;
5125                 coal += q_vector->lrolist->stats.coal;
5126                 recycled += q_vector->lrolist->stats.recycled;
5127         }
5128         adapter->lro_stats.flushed = flushed;
5129         adapter->lro_stats.coal = coal;
5130         adapter->lro_stats.recycled = recycled;
5131
5132 #endif
5133         bytes = 0;
5134         packets = 0;
5135         for (i = 0; i < adapter->num_rx_queues; i++) {
5136                 u32 rqdpc_tmp = E1000_READ_REG(hw, E1000_RQDPC(i)) & 0x0FFF;
5137                 struct igb_ring *ring = adapter->rx_ring[i];
5138                 ring->rx_stats.drops += rqdpc_tmp;
5139                 net_stats->rx_fifo_errors += rqdpc_tmp;
5140 #ifdef CONFIG_IGB_VMDQ_NETDEV
5141                 if (!ring->vmdq_netdev) {
5142                         bytes += ring->rx_stats.bytes;
5143                         packets += ring->rx_stats.packets;
5144                 }
5145 #else
5146                 bytes += ring->rx_stats.bytes;
5147                 packets += ring->rx_stats.packets;
5148 #endif
5149         }
5150
5151         net_stats->rx_bytes = bytes;
5152         net_stats->rx_packets = packets;
5153
5154         bytes = 0;
5155         packets = 0;
5156         for (i = 0; i < adapter->num_tx_queues; i++) {
5157                 struct igb_ring *ring = adapter->tx_ring[i];
5158 #ifdef CONFIG_IGB_VMDQ_NETDEV
5159                 if (!ring->vmdq_netdev) {
5160                         bytes += ring->tx_stats.bytes;
5161                         packets += ring->tx_stats.packets;
5162                 }
5163 #else
5164                 bytes += ring->tx_stats.bytes;
5165                 packets += ring->tx_stats.packets;
5166 #endif
5167         }
5168         net_stats->tx_bytes = bytes;
5169         net_stats->tx_packets = packets;
5170
5171         /* read stats registers */
5172         adapter->stats.crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
5173         adapter->stats.gprc += E1000_READ_REG(hw, E1000_GPRC);
5174         adapter->stats.gorc += E1000_READ_REG(hw, E1000_GORCL);
5175         E1000_READ_REG(hw, E1000_GORCH); /* clear GORCL */
5176         adapter->stats.bprc += E1000_READ_REG(hw, E1000_BPRC);
5177         adapter->stats.mprc += E1000_READ_REG(hw, E1000_MPRC);
5178         adapter->stats.roc += E1000_READ_REG(hw, E1000_ROC);
5179
5180         adapter->stats.prc64 += E1000_READ_REG(hw, E1000_PRC64);
5181         adapter->stats.prc127 += E1000_READ_REG(hw, E1000_PRC127);
5182         adapter->stats.prc255 += E1000_READ_REG(hw, E1000_PRC255);
5183         adapter->stats.prc511 += E1000_READ_REG(hw, E1000_PRC511);
5184         adapter->stats.prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
5185         adapter->stats.prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
5186         adapter->stats.symerrs += E1000_READ_REG(hw, E1000_SYMERRS);
5187         adapter->stats.sec += E1000_READ_REG(hw, E1000_SEC);
5188
5189         mpc = E1000_READ_REG(hw, E1000_MPC);
5190         adapter->stats.mpc += mpc;
5191         net_stats->rx_fifo_errors += mpc;
5192         adapter->stats.scc += E1000_READ_REG(hw, E1000_SCC);
5193         adapter->stats.ecol += E1000_READ_REG(hw, E1000_ECOL);
5194         adapter->stats.mcc += E1000_READ_REG(hw, E1000_MCC);
5195         adapter->stats.latecol += E1000_READ_REG(hw, E1000_LATECOL);
5196         adapter->stats.dc += E1000_READ_REG(hw, E1000_DC);
5197         adapter->stats.rlec += E1000_READ_REG(hw, E1000_RLEC);
5198         adapter->stats.xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
5199         adapter->stats.xontxc += E1000_READ_REG(hw, E1000_XONTXC);
5200         adapter->stats.xoffrxc += E1000_READ_REG(hw, E1000_XOFFRXC);
5201         adapter->stats.xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
5202         adapter->stats.fcruc += E1000_READ_REG(hw, E1000_FCRUC);
5203         adapter->stats.gptc += E1000_READ_REG(hw, E1000_GPTC);
5204         adapter->stats.gotc += E1000_READ_REG(hw, E1000_GOTCL);
5205         E1000_READ_REG(hw, E1000_GOTCH); /* clear GOTCL */
5206         adapter->stats.rnbc += E1000_READ_REG(hw, E1000_RNBC);
5207         adapter->stats.ruc += E1000_READ_REG(hw, E1000_RUC);
5208         adapter->stats.rfc += E1000_READ_REG(hw, E1000_RFC);
5209         adapter->stats.rjc += E1000_READ_REG(hw, E1000_RJC);
5210         adapter->stats.tor += E1000_READ_REG(hw, E1000_TORH);
5211         adapter->stats.tot += E1000_READ_REG(hw, E1000_TOTH);
5212         adapter->stats.tpr += E1000_READ_REG(hw, E1000_TPR);
5213
5214         adapter->stats.ptc64 += E1000_READ_REG(hw, E1000_PTC64);
5215         adapter->stats.ptc127 += E1000_READ_REG(hw, E1000_PTC127);
5216         adapter->stats.ptc255 += E1000_READ_REG(hw, E1000_PTC255);
5217         adapter->stats.ptc511 += E1000_READ_REG(hw, E1000_PTC511);
5218         adapter->stats.ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
5219         adapter->stats.ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
5220
5221         adapter->stats.mptc += E1000_READ_REG(hw, E1000_MPTC);
5222         adapter->stats.bptc += E1000_READ_REG(hw, E1000_BPTC);
5223
5224         adapter->stats.tpt += E1000_READ_REG(hw, E1000_TPT);
5225         adapter->stats.colc += E1000_READ_REG(hw, E1000_COLC);
5226
5227         adapter->stats.algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
5228         /* read internal phy sepecific stats */
5229         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
5230         if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
5231                 adapter->stats.rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
5232                 adapter->stats.tncrs += E1000_READ_REG(hw, E1000_TNCRS);
5233         }
5234
5235         adapter->stats.tsctc += E1000_READ_REG(hw, E1000_TSCTC);
5236         adapter->stats.tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
5237
5238         adapter->stats.iac += E1000_READ_REG(hw, E1000_IAC);
5239         adapter->stats.icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
5240         adapter->stats.icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
5241         adapter->stats.icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
5242         adapter->stats.ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
5243         adapter->stats.ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
5244         adapter->stats.ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
5245         adapter->stats.ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
5246         adapter->stats.icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
5247
5248         /* Fill out the OS statistics structure */
5249         net_stats->multicast = adapter->stats.mprc;
5250         net_stats->collisions = adapter->stats.colc;
5251
5252         /* Rx Errors */
5253
5254         /* RLEC on some newer hardware can be incorrect so build
5255          * our own version based on RUC and ROC */
5256         net_stats->rx_errors = adapter->stats.rxerrc +
5257                 adapter->stats.crcerrs + adapter->stats.algnerrc +
5258                 adapter->stats.ruc + adapter->stats.roc +
5259                 adapter->stats.cexterr;
5260         net_stats->rx_length_errors = adapter->stats.ruc +
5261                                       adapter->stats.roc;
5262         net_stats->rx_crc_errors = adapter->stats.crcerrs;
5263         net_stats->rx_frame_errors = adapter->stats.algnerrc;
5264         net_stats->rx_missed_errors = adapter->stats.mpc;
5265
5266         /* Tx Errors */
5267         net_stats->tx_errors = adapter->stats.ecol +
5268                                adapter->stats.latecol;
5269         net_stats->tx_aborted_errors = adapter->stats.ecol;
5270         net_stats->tx_window_errors = adapter->stats.latecol;
5271         net_stats->tx_carrier_errors = adapter->stats.tncrs;
5272
5273         /* Tx Dropped needs to be maintained elsewhere */
5274
5275         /* Phy Stats */
5276         if (hw->phy.media_type == e1000_media_type_copper) {
5277                 if ((adapter->link_speed == SPEED_1000) &&
5278                    (!e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
5279                         phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
5280                         adapter->phy_stats.idle_errors += phy_tmp;
5281                 }
5282         }
5283
5284         /* Management Stats */
5285         adapter->stats.mgptc += E1000_READ_REG(hw, E1000_MGTPTC);
5286         adapter->stats.mgprc += E1000_READ_REG(hw, E1000_MGTPRC);
5287         if (hw->mac.type > e1000_82580) {
5288                 adapter->stats.o2bgptc += E1000_READ_REG(hw, E1000_O2BGPTC);
5289                 adapter->stats.o2bspc += E1000_READ_REG(hw, E1000_O2BSPC);
5290                 adapter->stats.b2ospc += E1000_READ_REG(hw, E1000_B2OSPC);
5291                 adapter->stats.b2ogprc += E1000_READ_REG(hw, E1000_B2OGPRC);
5292         }
5293 }
5294
5295 static irqreturn_t igb_msix_other(int irq, void *data)
5296 {
5297         struct igb_adapter *adapter = data;
5298         struct e1000_hw *hw = &adapter->hw;
5299         u32 icr = E1000_READ_REG(hw, E1000_ICR);
5300         /* reading ICR causes bit 31 of EICR to be cleared */
5301
5302         if (icr & E1000_ICR_DRSTA)
5303                 schedule_work(&adapter->reset_task);
5304
5305         if (icr & E1000_ICR_DOUTSYNC) {
5306                 /* HW is reporting DMA is out of sync */
5307                 adapter->stats.doosync++;
5308                 /* The DMA Out of Sync is also indication of a spoof event
5309                  * in IOV mode. Check the Wrong VM Behavior register to
5310                  * see if it is really a spoof event. */
5311                 igb_check_wvbr(adapter);
5312         }
5313
5314         /* Check for a mailbox event */
5315         if (icr & E1000_ICR_VMMB)
5316                 igb_msg_task(adapter);
5317
5318         if (icr & E1000_ICR_LSC) {
5319                 hw->mac.get_link_status = 1;
5320                 /* guard against interrupt when we're going down */
5321                 if (!test_bit(__IGB_DOWN, &adapter->state))
5322                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
5323         }
5324
5325         /* Check for MDD event */
5326         if (icr & E1000_ICR_MDDET)
5327                 igb_process_mdd_event(adapter);
5328
5329         E1000_WRITE_REG(hw, E1000_EIMS, adapter->eims_other);
5330
5331         return IRQ_HANDLED;
5332 }
5333
5334 static void igb_write_itr(struct igb_q_vector *q_vector)
5335 {
5336         struct igb_adapter *adapter = q_vector->adapter;
5337         u32 itr_val = q_vector->itr_val & 0x7FFC;
5338
5339         if (!q_vector->set_itr)
5340                 return;
5341
5342         if (!itr_val)
5343                 itr_val = 0x4;
5344
5345         if (adapter->hw.mac.type == e1000_82575)
5346                 itr_val |= itr_val << 16;
5347         else
5348                 itr_val |= E1000_EITR_CNT_IGNR;
5349
5350         writel(itr_val, q_vector->itr_register);
5351         q_vector->set_itr = 0;
5352 }
5353
5354 static irqreturn_t igb_msix_ring(int irq, void *data)
5355 {
5356         struct igb_q_vector *q_vector = data;
5357
5358         /* Write the ITR value calculated from the previous interrupt. */
5359         igb_write_itr(q_vector);
5360
5361         napi_schedule(&q_vector->napi);
5362
5363         return IRQ_HANDLED;
5364 }
5365
5366 #ifdef IGB_DCA
5367 static void igb_update_dca(struct igb_q_vector *q_vector)
5368 {
5369         struct igb_adapter *adapter = q_vector->adapter;
5370         struct e1000_hw *hw = &adapter->hw;
5371         int cpu = get_cpu();
5372
5373         if (q_vector->cpu == cpu)
5374                 goto out_no_update;
5375
5376         if (q_vector->tx.ring) {
5377                 int q = q_vector->tx.ring->reg_idx;
5378                 u32 dca_txctrl = E1000_READ_REG(hw, E1000_DCA_TXCTRL(q));
5379                 if (hw->mac.type == e1000_82575) {
5380                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
5381                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
5382                 } else {
5383                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
5384                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
5385                                       E1000_DCA_TXCTRL_CPUID_SHIFT_82576;
5386                 }
5387                 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
5388                 E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(q), dca_txctrl);
5389         }
5390         if (q_vector->rx.ring) {
5391                 int q = q_vector->rx.ring->reg_idx;
5392                 u32 dca_rxctrl = E1000_READ_REG(hw, E1000_DCA_RXCTRL(q));
5393                 if (hw->mac.type == e1000_82575) {
5394                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
5395                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
5396                 } else {
5397                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
5398                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
5399                                       E1000_DCA_RXCTRL_CPUID_SHIFT_82576;
5400                 }
5401                 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
5402                 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
5403                 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
5404                 E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(q), dca_rxctrl);
5405         }
5406         q_vector->cpu = cpu;
5407 out_no_update:
5408         put_cpu();
5409 }
5410
5411 static void igb_setup_dca(struct igb_adapter *adapter)
5412 {
5413         struct e1000_hw *hw = &adapter->hw;
5414         int i;
5415
5416         if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
5417                 return;
5418
5419         /* Always use CB2 mode, difference is masked in the CB driver. */
5420         E1000_WRITE_REG(hw, E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
5421
5422         for (i = 0; i < adapter->num_q_vectors; i++) {
5423                 adapter->q_vector[i]->cpu = -1;
5424                 igb_update_dca(adapter->q_vector[i]);
5425         }
5426 }
5427
5428 static int __igb_notify_dca(struct device *dev, void *data)
5429 {
5430         struct net_device *netdev = dev_get_drvdata(dev);
5431         struct igb_adapter *adapter = netdev_priv(netdev);
5432         struct pci_dev *pdev = adapter->pdev;
5433         struct e1000_hw *hw = &adapter->hw;
5434         unsigned long event = *(unsigned long *)data;
5435
5436         switch (event) {
5437         case DCA_PROVIDER_ADD:
5438                 /* if already enabled, don't do it again */
5439                 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
5440                         break;
5441                 if (dca_add_requester(dev) == E1000_SUCCESS) {
5442                         adapter->flags |= IGB_FLAG_DCA_ENABLED;
5443                         dev_info(pci_dev_to_dev(pdev), "DCA enabled\n");
5444                         igb_setup_dca(adapter);
5445                         break;
5446                 }
5447                 /* Fall Through since DCA is disabled. */
5448         case DCA_PROVIDER_REMOVE:
5449                 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
5450                         /* without this a class_device is left
5451                          * hanging around in the sysfs model */
5452                         dca_remove_requester(dev);
5453                         dev_info(pci_dev_to_dev(pdev), "DCA disabled\n");
5454                         adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
5455                         E1000_WRITE_REG(hw, E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_DISABLE);
5456                 }
5457                 break;
5458         }
5459
5460         return E1000_SUCCESS;
5461 }
5462
5463 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
5464                           void *p)
5465 {
5466         int ret_val;
5467
5468         ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
5469                                          __igb_notify_dca);
5470
5471         return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
5472 }
5473 #endif /* IGB_DCA */
5474
5475 static int igb_vf_configure(struct igb_adapter *adapter, int vf)
5476 {
5477         unsigned char mac_addr[ETH_ALEN];
5478 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
5479         struct pci_dev *pdev = adapter->pdev;
5480         struct e1000_hw *hw = &adapter->hw;
5481         struct pci_dev *pvfdev;
5482         unsigned int device_id;
5483         u16 thisvf_devfn;
5484 #endif
5485
5486         random_ether_addr(mac_addr);
5487         igb_set_vf_mac(adapter, vf, mac_addr);
5488
5489 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
5490         switch (adapter->hw.mac.type) {
5491         case e1000_82576:
5492                 device_id = IGB_82576_VF_DEV_ID;
5493                 /* VF Stride for 82576 is 2 */
5494                 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 1)) |
5495                         (pdev->devfn & 1);
5496                 break;
5497         case e1000_i350:
5498                 device_id = IGB_I350_VF_DEV_ID;
5499                 /* VF Stride for I350 is 4 */
5500                 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 2)) |
5501                                 (pdev->devfn & 3);
5502                 break;
5503         default:
5504                 device_id = 0;
5505                 thisvf_devfn = 0;
5506                 break;
5507         }
5508
5509         pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5510         while (pvfdev) {
5511                 if (pvfdev->devfn == thisvf_devfn)
5512                         break;
5513                 pvfdev = pci_get_device(hw->vendor_id,
5514                                         device_id, pvfdev);
5515         }
5516
5517         if (pvfdev)
5518                 adapter->vf_data[vf].vfdev = pvfdev;
5519         else
5520                 dev_err(&pdev->dev,
5521                         "Couldn't find pci dev ptr for VF %4.4x\n",
5522                         thisvf_devfn);
5523         return pvfdev != NULL;
5524 #else
5525         return true;
5526 #endif
5527 }
5528
5529 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
5530 static int igb_find_enabled_vfs(struct igb_adapter *adapter)
5531 {
5532         struct e1000_hw *hw = &adapter->hw;
5533         struct pci_dev *pdev = adapter->pdev;
5534         struct pci_dev *pvfdev;
5535         u16 vf_devfn = 0;
5536         u16 vf_stride;
5537         unsigned int device_id;
5538         int vfs_found = 0;
5539
5540         switch (adapter->hw.mac.type) {
5541         case e1000_82576:
5542                 device_id = IGB_82576_VF_DEV_ID;
5543                 /* VF Stride for 82576 is 2 */
5544                 vf_stride = 2;
5545                 break;
5546         case e1000_i350:
5547                 device_id = IGB_I350_VF_DEV_ID;
5548                 /* VF Stride for I350 is 4 */
5549                 vf_stride = 4;
5550                 break;
5551         default:
5552                 device_id = 0;
5553                 vf_stride = 0;
5554                 break;
5555         }
5556
5557         vf_devfn = pdev->devfn + 0x80;
5558         pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5559         while (pvfdev) {
5560                 if (pvfdev->devfn == vf_devfn)
5561                         vfs_found++;
5562                 vf_devfn += vf_stride;
5563                 pvfdev = pci_get_device(hw->vendor_id,
5564                                         device_id, pvfdev);
5565         }
5566
5567         return vfs_found;
5568 }
5569 #endif
5570
5571 static int igb_check_vf_assignment(struct igb_adapter *adapter)
5572 {
5573 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
5574         int i;
5575         for (i = 0; i < adapter->vfs_allocated_count; i++) {
5576                 if (adapter->vf_data[i].vfdev) {
5577                         if (adapter->vf_data[i].vfdev->dev_flags &
5578                             PCI_DEV_FLAGS_ASSIGNED)
5579                                 return true;
5580                 }
5581         }
5582 #endif
5583         return false;
5584 }
5585
5586 static void igb_ping_all_vfs(struct igb_adapter *adapter)
5587 {
5588         struct e1000_hw *hw = &adapter->hw;
5589         u32 ping;
5590         int i;
5591
5592         for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
5593                 ping = E1000_PF_CONTROL_MSG;
5594                 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
5595                         ping |= E1000_VT_MSGTYPE_CTS;
5596                 e1000_write_mbx(hw, &ping, 1, i);
5597         }
5598 }
5599
5600 /**
5601  *  igb_mta_set_ - Set multicast filter table address
5602  *  @adapter: pointer to the adapter structure
5603  *  @hash_value: determines the MTA register and bit to set
5604  *
5605  *  The multicast table address is a register array of 32-bit registers.
5606  *  The hash_value is used to determine what register the bit is in, the
5607  *  current value is read, the new bit is OR'd in and the new value is
5608  *  written back into the register.
5609  **/
5610 void igb_mta_set(struct igb_adapter *adapter, u32 hash_value)
5611 {
5612         struct e1000_hw *hw = &adapter->hw;
5613         u32 hash_bit, hash_reg, mta;
5614
5615         /*
5616          * The MTA is a register array of 32-bit registers. It is
5617          * treated like an array of (32*mta_reg_count) bits.  We want to
5618          * set bit BitArray[hash_value]. So we figure out what register
5619          * the bit is in, read it, OR in the new bit, then write
5620          * back the new value.  The (hw->mac.mta_reg_count - 1) serves as a
5621          * mask to bits 31:5 of the hash value which gives us the
5622          * register we're modifying.  The hash bit within that register
5623          * is determined by the lower 5 bits of the hash value.
5624          */
5625         hash_reg = (hash_value >> 5) & (hw->mac.mta_reg_count - 1);
5626         hash_bit = hash_value & 0x1F;
5627
5628         mta = E1000_READ_REG_ARRAY(hw, E1000_MTA, hash_reg);
5629
5630         mta |= (1 << hash_bit);
5631
5632         E1000_WRITE_REG_ARRAY(hw, E1000_MTA, hash_reg, mta);
5633         E1000_WRITE_FLUSH(hw);
5634 }
5635
5636 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5637 {
5638
5639         struct e1000_hw *hw = &adapter->hw;
5640         u32 vmolr = E1000_READ_REG(hw, E1000_VMOLR(vf));
5641         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5642
5643         vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
5644                             IGB_VF_FLAG_MULTI_PROMISC);
5645         vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5646
5647 #ifdef IGB_ENABLE_VF_PROMISC
5648         if (*msgbuf & E1000_VF_SET_PROMISC_UNICAST) {
5649                 vmolr |= E1000_VMOLR_ROPE;
5650                 vf_data->flags |= IGB_VF_FLAG_UNI_PROMISC;
5651                 *msgbuf &= ~E1000_VF_SET_PROMISC_UNICAST;
5652         }
5653 #endif
5654         if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
5655                 vmolr |= E1000_VMOLR_MPME;
5656                 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
5657                 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
5658         } else {
5659                 /*
5660                  * if we have hashes and we are clearing a multicast promisc
5661                  * flag we need to write the hashes to the MTA as this step
5662                  * was previously skipped
5663                  */
5664                 if (vf_data->num_vf_mc_hashes > 30) {
5665                         vmolr |= E1000_VMOLR_MPME;
5666                 } else if (vf_data->num_vf_mc_hashes) {
5667                         int j;
5668                         vmolr |= E1000_VMOLR_ROMPE;
5669                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5670                                 igb_mta_set(adapter, vf_data->vf_mc_hashes[j]);
5671                 }
5672         }
5673
5674         E1000_WRITE_REG(hw, E1000_VMOLR(vf), vmolr);
5675
5676         /* there are flags left unprocessed, likely not supported */
5677         if (*msgbuf & E1000_VT_MSGINFO_MASK)
5678                 return -EINVAL;
5679
5680         return 0;
5681
5682 }
5683
5684 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
5685                                   u32 *msgbuf, u32 vf)
5686 {
5687         int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5688         u16 *hash_list = (u16 *)&msgbuf[1];
5689         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5690         int i;
5691
5692         /* salt away the number of multicast addresses assigned
5693          * to this VF for later use to restore when the PF multi cast
5694          * list changes
5695          */
5696         vf_data->num_vf_mc_hashes = n;
5697
5698         /* only up to 30 hash values supported */
5699         if (n > 30)
5700                 n = 30;
5701
5702         /* store the hashes for later use */
5703         for (i = 0; i < n; i++)
5704                 vf_data->vf_mc_hashes[i] = hash_list[i];
5705
5706         /* Flush and reset the mta with the new values */
5707         igb_set_rx_mode(adapter->netdev);
5708
5709         return 0;
5710 }
5711
5712 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
5713 {
5714         struct e1000_hw *hw = &adapter->hw;
5715         struct vf_data_storage *vf_data;
5716         int i, j;
5717
5718         for (i = 0; i < adapter->vfs_allocated_count; i++) {
5719                 u32 vmolr = E1000_READ_REG(hw, E1000_VMOLR(i));
5720                 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5721
5722                 vf_data = &adapter->vf_data[i];
5723
5724                 if ((vf_data->num_vf_mc_hashes > 30) ||
5725                     (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5726                         vmolr |= E1000_VMOLR_MPME;
5727                 } else if (vf_data->num_vf_mc_hashes) {
5728                         vmolr |= E1000_VMOLR_ROMPE;
5729                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5730                                 igb_mta_set(adapter, vf_data->vf_mc_hashes[j]);
5731                 }
5732                 E1000_WRITE_REG(hw, E1000_VMOLR(i), vmolr);
5733         }
5734 }
5735
5736 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5737 {
5738         struct e1000_hw *hw = &adapter->hw;
5739         u32 pool_mask, reg, vid;
5740         u16 vlan_default;
5741         int i;
5742
5743         pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5744
5745         /* Find the vlan filter for this id */
5746         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5747                 reg = E1000_READ_REG(hw, E1000_VLVF(i));
5748
5749                 /* remove the vf from the pool */
5750                 reg &= ~pool_mask;
5751
5752                 /* if pool is empty then remove entry from vfta */
5753                 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5754                     (reg & E1000_VLVF_VLANID_ENABLE)) {
5755                         reg = 0;
5756                         vid = reg & E1000_VLVF_VLANID_MASK;
5757                         igb_vfta_set(adapter, vid, FALSE);
5758                 }
5759
5760                 E1000_WRITE_REG(hw, E1000_VLVF(i), reg);
5761         }
5762
5763         adapter->vf_data[vf].vlans_enabled = 0;
5764
5765         vlan_default = adapter->vf_data[vf].default_vf_vlan_id;
5766         if (vlan_default)
5767                 igb_vlvf_set(adapter, vlan_default, true, vf);
5768 }
5769
5770 s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5771 {
5772         struct e1000_hw *hw = &adapter->hw;
5773         u32 reg, i;
5774
5775         /* The vlvf table only exists on 82576 hardware and newer */
5776         if (hw->mac.type < e1000_82576)
5777                 return -1;
5778
5779         /* we only need to do this if VMDq is enabled */
5780         if (!adapter->vmdq_pools)
5781                 return -1;
5782
5783         /* Find the vlan filter for this id */
5784         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5785                 reg = E1000_READ_REG(hw, E1000_VLVF(i));
5786                 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5787                     vid == (reg & E1000_VLVF_VLANID_MASK))
5788                         break;
5789         }
5790
5791         if (add) {
5792                 if (i == E1000_VLVF_ARRAY_SIZE) {
5793                         /* Did not find a matching VLAN ID entry that was
5794                          * enabled.  Search for a free filter entry, i.e.
5795                          * one without the enable bit set
5796                          */
5797                         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5798                                 reg = E1000_READ_REG(hw, E1000_VLVF(i));
5799                                 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5800                                         break;
5801                         }
5802                 }
5803                 if (i < E1000_VLVF_ARRAY_SIZE) {
5804                         /* Found an enabled/available entry */
5805                         reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5806
5807                         /* if !enabled we need to set this up in vfta */
5808                         if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
5809                                 /* add VID to filter table */
5810                                 igb_vfta_set(adapter, vid, TRUE);
5811                                 reg |= E1000_VLVF_VLANID_ENABLE;
5812                         }
5813                         reg &= ~E1000_VLVF_VLANID_MASK;
5814                         reg |= vid;
5815                         E1000_WRITE_REG(hw, E1000_VLVF(i), reg);
5816
5817                         /* do not modify RLPML for PF devices */
5818                         if (vf >= adapter->vfs_allocated_count)
5819                                 return E1000_SUCCESS;
5820
5821                         if (!adapter->vf_data[vf].vlans_enabled) {
5822                                 u32 size;
5823                                 reg = E1000_READ_REG(hw, E1000_VMOLR(vf));
5824                                 size = reg & E1000_VMOLR_RLPML_MASK;
5825                                 size += 4;
5826                                 reg &= ~E1000_VMOLR_RLPML_MASK;
5827                                 reg |= size;
5828                                 E1000_WRITE_REG(hw, E1000_VMOLR(vf), reg);
5829                         }
5830
5831                         adapter->vf_data[vf].vlans_enabled++;
5832                 }
5833         } else {
5834                 if (i < E1000_VLVF_ARRAY_SIZE) {
5835                         /* remove vf from the pool */
5836                         reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5837                         /* if pool is empty then remove entry from vfta */
5838                         if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5839                                 reg = 0;
5840                                 igb_vfta_set(adapter, vid, FALSE);
5841                         }
5842                         E1000_WRITE_REG(hw, E1000_VLVF(i), reg);
5843
5844                         /* do not modify RLPML for PF devices */
5845                         if (vf >= adapter->vfs_allocated_count)
5846                                 return E1000_SUCCESS;
5847
5848                         adapter->vf_data[vf].vlans_enabled--;
5849                         if (!adapter->vf_data[vf].vlans_enabled) {
5850                                 u32 size;
5851                                 reg = E1000_READ_REG(hw, E1000_VMOLR(vf));
5852                                 size = reg & E1000_VMOLR_RLPML_MASK;
5853                                 size -= 4;
5854                                 reg &= ~E1000_VMOLR_RLPML_MASK;
5855                                 reg |= size;
5856                                 E1000_WRITE_REG(hw, E1000_VMOLR(vf), reg);
5857                         }
5858                 }
5859         }
5860         return E1000_SUCCESS;
5861 }
5862
5863 #ifdef IFLA_VF_MAX
5864 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5865 {
5866         struct e1000_hw *hw = &adapter->hw;
5867
5868         if (vid)
5869                 E1000_WRITE_REG(hw, E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5870         else
5871                 E1000_WRITE_REG(hw, E1000_VMVIR(vf), 0);
5872 }
5873
5874 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5875                                int vf, u16 vlan, u8 qos)
5876 {
5877         int err = 0;
5878         struct igb_adapter *adapter = netdev_priv(netdev);
5879
5880         /* VLAN IDs accepted range 0-4094 */
5881         if ((vf >= adapter->vfs_allocated_count) || (vlan > VLAN_VID_MASK-1) || (qos > 7))
5882                 return -EINVAL;
5883         if (vlan || qos) {
5884                 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5885                 if (err)
5886                         goto out;
5887                 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5888                 igb_set_vmolr(adapter, vf, !vlan);
5889                 adapter->vf_data[vf].pf_vlan = vlan;
5890                 adapter->vf_data[vf].pf_qos = qos;
5891                 igb_set_vf_vlan_strip(adapter, vf, true);
5892                 dev_info(&adapter->pdev->dev,
5893                          "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5894                 if (test_bit(__IGB_DOWN, &adapter->state)) {
5895                         dev_warn(&adapter->pdev->dev,
5896                                  "The VF VLAN has been set,"
5897                                  " but the PF device is not up.\n");
5898                         dev_warn(&adapter->pdev->dev,
5899                                  "Bring the PF device up before"
5900                                  " attempting to use the VF device.\n");
5901                 }
5902         } else {
5903                 if (adapter->vf_data[vf].pf_vlan)
5904                         dev_info(&adapter->pdev->dev,
5905                                  "Clearing VLAN on VF %d\n", vf);
5906                 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5907                                    false, vf);
5908                 igb_set_vmvir(adapter, vlan, vf);
5909                 igb_set_vmolr(adapter, vf, true);
5910                 igb_set_vf_vlan_strip(adapter, vf, false);
5911                 adapter->vf_data[vf].pf_vlan = 0;
5912                 adapter->vf_data[vf].pf_qos = 0;
5913        }
5914 out:
5915        return err;
5916 }
5917 #endif
5918
5919 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5920 {
5921         int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5922         int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5923
5924         if (vid)
5925                 igb_set_vf_vlan_strip(adapter, vf, true);
5926         else
5927                 igb_set_vf_vlan_strip(adapter, vf, false);
5928
5929         return igb_vlvf_set(adapter, vid, add, vf);
5930 }
5931
5932 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
5933 {
5934         struct e1000_hw *hw = &adapter->hw;
5935
5936         /* clear flags except flag that the PF has set the MAC */
5937         adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
5938         adapter->vf_data[vf].last_nack = jiffies;
5939
5940         /* reset offloads to defaults */
5941         igb_set_vmolr(adapter, vf, true);
5942
5943         /* reset vlans for device */
5944         igb_clear_vf_vfta(adapter, vf);
5945 #ifdef IFLA_VF_MAX
5946         if (adapter->vf_data[vf].pf_vlan)
5947                 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5948                                     adapter->vf_data[vf].pf_vlan,
5949                                     adapter->vf_data[vf].pf_qos);
5950         else
5951                 igb_clear_vf_vfta(adapter, vf);
5952 #endif
5953
5954         /* reset multicast table array for vf */
5955         adapter->vf_data[vf].num_vf_mc_hashes = 0;
5956
5957         /* Flush and reset the mta with the new values */
5958         igb_set_rx_mode(adapter->netdev);
5959
5960         /*
5961          * Reset the VFs TDWBAL and TDWBAH registers which are not
5962          * cleared by a VFLR
5963          */
5964         E1000_WRITE_REG(hw, E1000_TDWBAH(vf), 0);
5965         E1000_WRITE_REG(hw, E1000_TDWBAL(vf), 0);
5966         if (hw->mac.type == e1000_82576) {
5967                 E1000_WRITE_REG(hw, E1000_TDWBAH(IGB_MAX_VF_FUNCTIONS + vf), 0);
5968                 E1000_WRITE_REG(hw, E1000_TDWBAL(IGB_MAX_VF_FUNCTIONS + vf), 0);
5969         }
5970 }
5971
5972 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5973 {
5974         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5975
5976         /* generate a new mac address as we were hotplug removed/added */
5977         if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5978                 random_ether_addr(vf_mac);
5979
5980         /* process remaining reset events */
5981         igb_vf_reset(adapter, vf);
5982 }
5983
5984 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
5985 {
5986         struct e1000_hw *hw = &adapter->hw;
5987         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5988         u32 reg, msgbuf[3];
5989         u8 *addr = (u8 *)(&msgbuf[1]);
5990
5991         /* process all the same items cleared in a function level reset */
5992         igb_vf_reset(adapter, vf);
5993
5994         /* set vf mac address */
5995         igb_del_mac_filter(adapter, vf_mac, vf);
5996         igb_add_mac_filter(adapter, vf_mac, vf);
5997
5998         /* enable transmit and receive for vf */
5999         reg = E1000_READ_REG(hw, E1000_VFTE);
6000         E1000_WRITE_REG(hw, E1000_VFTE, reg | (1 << vf));
6001         reg = E1000_READ_REG(hw, E1000_VFRE);
6002         E1000_WRITE_REG(hw, E1000_VFRE, reg | (1 << vf));
6003
6004         adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
6005
6006         /* reply to reset with ack and vf mac address */
6007         msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
6008         memcpy(addr, vf_mac, 6);
6009         e1000_write_mbx(hw, msgbuf, 3, vf);
6010 }
6011
6012 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
6013 {
6014         /*
6015          * The VF MAC Address is stored in a packed array of bytes
6016          * starting at the second 32 bit word of the msg array
6017          */
6018         unsigned char *addr = (unsigned char *)&msg[1];
6019         int err = -1;
6020
6021         if (is_valid_ether_addr(addr))
6022                 err = igb_set_vf_mac(adapter, vf, addr);
6023
6024         return err;
6025 }
6026
6027 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
6028 {
6029         struct e1000_hw *hw = &adapter->hw;
6030         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6031         u32 msg = E1000_VT_MSGTYPE_NACK;
6032
6033         /* if device isn't clear to send it shouldn't be reading either */
6034         if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
6035             time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
6036                 e1000_write_mbx(hw, &msg, 1, vf);
6037                 vf_data->last_nack = jiffies;
6038         }
6039 }
6040
6041 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
6042 {
6043         struct pci_dev *pdev = adapter->pdev;
6044         u32 msgbuf[E1000_VFMAILBOX_SIZE];
6045         struct e1000_hw *hw = &adapter->hw;
6046         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6047         s32 retval;
6048
6049         retval = e1000_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
6050
6051         if (retval) {
6052                 dev_err(pci_dev_to_dev(pdev), "Error receiving message from VF\n");
6053                 return;
6054         }
6055
6056         /* this is a message we already processed, do nothing */
6057         if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
6058                 return;
6059
6060         /*
6061          * until the vf completes a reset it should not be
6062          * allowed to start any configuration.
6063          */
6064
6065         if (msgbuf[0] == E1000_VF_RESET) {
6066                 igb_vf_reset_msg(adapter, vf);
6067                 return;
6068         }
6069
6070         if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
6071                 msgbuf[0] = E1000_VT_MSGTYPE_NACK;
6072                 if (time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
6073                         e1000_write_mbx(hw, msgbuf, 1, vf);
6074                         vf_data->last_nack = jiffies;
6075                 }
6076                 return;
6077         }
6078
6079         switch ((msgbuf[0] & 0xFFFF)) {
6080         case E1000_VF_SET_MAC_ADDR:
6081                 retval = -EINVAL;
6082 #ifndef IGB_DISABLE_VF_MAC_SET
6083                 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
6084                         retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
6085                 else
6086                         DPRINTK(DRV, INFO,
6087                                 "VF %d attempted to override administratively "
6088                                 "set MAC address\nReload the VF driver to "
6089                                 "resume operations\n", vf);
6090 #endif
6091                 break;
6092         case E1000_VF_SET_PROMISC:
6093                 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
6094                 break;
6095         case E1000_VF_SET_MULTICAST:
6096                 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
6097                 break;
6098         case E1000_VF_SET_LPE:
6099                 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
6100                 break;
6101         case E1000_VF_SET_VLAN:
6102                 retval = -1;
6103 #ifdef IFLA_VF_MAX
6104                 if (vf_data->pf_vlan)
6105                         DPRINTK(DRV, INFO,
6106                                 "VF %d attempted to override administratively "
6107                                 "set VLAN tag\nReload the VF driver to "
6108                                 "resume operations\n", vf);
6109                 else
6110 #endif
6111                         retval = igb_set_vf_vlan(adapter, msgbuf, vf);
6112                 break;
6113         default:
6114                 dev_err(pci_dev_to_dev(pdev), "Unhandled Msg %08x\n", msgbuf[0]);
6115                 retval = -E1000_ERR_MBX;
6116                 break;
6117         }
6118
6119         /* notify the VF of the results of what it sent us */
6120         if (retval)
6121                 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
6122         else
6123                 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
6124
6125         msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
6126
6127         e1000_write_mbx(hw, msgbuf, 1, vf);
6128 }
6129
6130 static void igb_msg_task(struct igb_adapter *adapter)
6131 {
6132         struct e1000_hw *hw = &adapter->hw;
6133         u32 vf;
6134
6135         for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
6136                 /* process any reset requests */
6137                 if (!e1000_check_for_rst(hw, vf))
6138                         igb_vf_reset_event(adapter, vf);
6139
6140                 /* process any messages pending */
6141                 if (!e1000_check_for_msg(hw, vf))
6142                         igb_rcv_msg_from_vf(adapter, vf);
6143
6144                 /* process any acks */
6145                 if (!e1000_check_for_ack(hw, vf))
6146                         igb_rcv_ack_from_vf(adapter, vf);
6147         }
6148 }
6149
6150 /**
6151  *  igb_set_uta - Set unicast filter table address
6152  *  @adapter: board private structure
6153  *
6154  *  The unicast table address is a register array of 32-bit registers.
6155  *  The table is meant to be used in a way similar to how the MTA is used
6156  *  however due to certain limitations in the hardware it is necessary to
6157  *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
6158  *  enable bit to allow vlan tag stripping when promiscuous mode is enabled
6159  **/
6160 static void igb_set_uta(struct igb_adapter *adapter)
6161 {
6162         struct e1000_hw *hw = &adapter->hw;
6163         int i;
6164
6165         /* The UTA table only exists on 82576 hardware and newer */
6166         if (hw->mac.type < e1000_82576)
6167                 return;
6168
6169         /* we only need to do this if VMDq is enabled */
6170         if (!adapter->vmdq_pools)
6171                 return;
6172
6173         for (i = 0; i < hw->mac.uta_reg_count; i++)
6174                 E1000_WRITE_REG_ARRAY(hw, E1000_UTA, i, ~0);
6175 }
6176
6177 /**
6178  * igb_intr_msi - Interrupt Handler
6179  * @irq: interrupt number
6180  * @data: pointer to a network interface device structure
6181  **/
6182 static irqreturn_t igb_intr_msi(int irq, void *data)
6183 {
6184         struct igb_adapter *adapter = data;
6185         struct igb_q_vector *q_vector = adapter->q_vector[0];
6186         struct e1000_hw *hw = &adapter->hw;
6187         /* read ICR disables interrupts using IAM */
6188         u32 icr = E1000_READ_REG(hw, E1000_ICR);
6189
6190         igb_write_itr(q_vector);
6191
6192         if (icr & E1000_ICR_DRSTA)
6193                 schedule_work(&adapter->reset_task);
6194
6195         if (icr & E1000_ICR_DOUTSYNC) {
6196                 /* HW is reporting DMA is out of sync */
6197                 adapter->stats.doosync++;
6198         }
6199
6200         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
6201                 hw->mac.get_link_status = 1;
6202                 if (!test_bit(__IGB_DOWN, &adapter->state))
6203                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
6204         }
6205
6206         napi_schedule(&q_vector->napi);
6207
6208         return IRQ_HANDLED;
6209 }
6210
6211 /**
6212  * igb_intr - Legacy Interrupt Handler
6213  * @irq: interrupt number
6214  * @data: pointer to a network interface device structure
6215  **/
6216 static irqreturn_t igb_intr(int irq, void *data)
6217 {
6218         struct igb_adapter *adapter = data;
6219         struct igb_q_vector *q_vector = adapter->q_vector[0];
6220         struct e1000_hw *hw = &adapter->hw;
6221         /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
6222          * need for the IMC write */
6223         u32 icr = E1000_READ_REG(hw, E1000_ICR);
6224
6225         /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
6226          * not set, then the adapter didn't send an interrupt */
6227         if (!(icr & E1000_ICR_INT_ASSERTED))
6228                 return IRQ_NONE;
6229
6230         igb_write_itr(q_vector);
6231
6232         if (icr & E1000_ICR_DRSTA)
6233                 schedule_work(&adapter->reset_task);
6234
6235         if (icr & E1000_ICR_DOUTSYNC) {
6236                 /* HW is reporting DMA is out of sync */
6237                 adapter->stats.doosync++;
6238         }
6239
6240         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
6241                 hw->mac.get_link_status = 1;
6242                 /* guard against interrupt when we're going down */
6243                 if (!test_bit(__IGB_DOWN, &adapter->state))
6244                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
6245         }
6246
6247         napi_schedule(&q_vector->napi);
6248
6249         return IRQ_HANDLED;
6250 }
6251
6252 void igb_ring_irq_enable(struct igb_q_vector *q_vector)
6253 {
6254         struct igb_adapter *adapter = q_vector->adapter;
6255         struct e1000_hw *hw = &adapter->hw;
6256
6257         if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
6258             (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
6259                 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
6260                         igb_set_itr(q_vector);
6261                 else
6262                         igb_update_ring_itr(q_vector);
6263         }
6264
6265         if (!test_bit(__IGB_DOWN, &adapter->state)) {
6266                 if (adapter->msix_entries)
6267                         E1000_WRITE_REG(hw, E1000_EIMS, q_vector->eims_value);
6268                 else
6269                         igb_irq_enable(adapter);
6270         }
6271 }
6272
6273 /**
6274  * igb_poll - NAPI Rx polling callback
6275  * @napi: napi polling structure
6276  * @budget: count of how many packets we should handle
6277  **/
6278 static int igb_poll(struct napi_struct *napi, int budget)
6279 {
6280         struct igb_q_vector *q_vector = container_of(napi, struct igb_q_vector, napi);
6281         bool clean_complete = true;
6282
6283 #ifdef IGB_DCA
6284         if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
6285                 igb_update_dca(q_vector);
6286 #endif
6287         if (q_vector->tx.ring)
6288                 clean_complete = igb_clean_tx_irq(q_vector);
6289
6290         if (q_vector->rx.ring)
6291                 clean_complete &= igb_clean_rx_irq(q_vector, budget);
6292
6293 #ifndef HAVE_NETDEV_NAPI_LIST
6294         /* if netdev is disabled we need to stop polling */
6295         if (!netif_running(q_vector->adapter->netdev))
6296                 clean_complete = true;
6297
6298 #endif
6299         /* If all work not completed, return budget and keep polling */
6300         if (!clean_complete)
6301                 return budget;
6302
6303         /* If not enough Rx work done, exit the polling mode */
6304         napi_complete(napi);
6305         igb_ring_irq_enable(q_vector);
6306
6307         return 0;
6308 }
6309
6310 #ifdef HAVE_HW_TIME_STAMP
6311 /**
6312  * igb_systim_to_hwtstamp - convert system time value to hw timestamp
6313  * @adapter: board private structure
6314  * @shhwtstamps: timestamp structure to update
6315  * @regval: unsigned 64bit system time value.
6316  *
6317  * We need to convert the system time value stored in the RX/TXSTMP registers
6318  * into a hwtstamp which can be used by the upper level timestamping functions
6319  */
6320 static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
6321                                    struct skb_shared_hwtstamps *shhwtstamps,
6322                                    u64 regval)
6323 {
6324         u64 ns;
6325
6326         /*
6327          * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
6328          * 24 to match clock shift we setup earlier.
6329          */
6330         if (adapter->hw.mac.type >= e1000_82580)
6331                 regval <<= IGB_82580_TSYNC_SHIFT;
6332
6333         ns = timecounter_cyc2time(&adapter->clock, regval);
6334
6335         /*
6336          * force a timecompare_update here (even if less than a second
6337          * has passed) in order to prevent the case when ptpd or other
6338          * software jumps the clock offset. othwerise there is a small
6339          * window when the timestamp would be based on previous skew
6340          * and invalid results would be pushed to the network stack.
6341          */
6342         timecompare_update(&adapter->compare, 0);
6343         memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
6344         shhwtstamps->hwtstamp = ns_to_ktime(ns);
6345         shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
6346 }
6347
6348 /**
6349  * igb_tx_hwtstamp - utility function which checks for TX time stamp
6350  * @q_vector: pointer to q_vector containing needed info
6351  * @buffer: pointer to igb_tx_buffer structure
6352  *
6353  * If we were asked to do hardware stamping and such a time stamp is
6354  * available, then it must have been for this skb here because we only
6355  * allow only one such packet into the queue.
6356  */
6357 static void igb_tx_hwtstamp(struct igb_q_vector *q_vector,
6358                             struct igb_tx_buffer *buffer_info)
6359 {
6360         struct igb_adapter *adapter = q_vector->adapter;
6361         struct e1000_hw *hw = &adapter->hw;
6362         struct skb_shared_hwtstamps shhwtstamps;
6363         u64 regval;
6364
6365         /* if skb does not support hw timestamp or TX stamp not valid exit */
6366         if (likely(!(buffer_info->tx_flags & IGB_TX_FLAGS_TSTAMP)) ||
6367             !(E1000_READ_REG(hw, E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
6368                 return;
6369
6370         regval = E1000_READ_REG(hw, E1000_TXSTMPL);
6371         regval |= (u64)E1000_READ_REG(hw, E1000_TXSTMPH) << 32;
6372
6373         igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
6374         skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
6375 }
6376
6377 #endif
6378 /**
6379  * igb_clean_tx_irq - Reclaim resources after transmit completes
6380  * @q_vector: pointer to q_vector containing needed info
6381  * returns TRUE if ring is completely cleaned
6382  **/
6383 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
6384 {
6385         struct igb_adapter *adapter = q_vector->adapter;
6386         struct igb_ring *tx_ring = q_vector->tx.ring;
6387         struct igb_tx_buffer *tx_buffer;
6388         union e1000_adv_tx_desc *tx_desc, *eop_desc;
6389         unsigned int total_bytes = 0, total_packets = 0;
6390         unsigned int budget = q_vector->tx.work_limit;
6391         unsigned int i = tx_ring->next_to_clean;
6392
6393         if (test_bit(__IGB_DOWN, &adapter->state))
6394                 return true;
6395
6396         tx_buffer = &tx_ring->tx_buffer_info[i];
6397         tx_desc = IGB_TX_DESC(tx_ring, i);
6398         i -= tx_ring->count;
6399
6400         for (; budget; budget--) {
6401                 eop_desc = tx_buffer->next_to_watch;
6402
6403                 /* prevent any other reads prior to eop_desc */
6404                 rmb();
6405
6406                 /* if next_to_watch is not set then there is no work pending */
6407                 if (!eop_desc)
6408                         break;
6409
6410                 /* if DD is not set pending work has not been completed */
6411                 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
6412                         break;
6413
6414                 /* clear next_to_watch to prevent false hangs */
6415                 tx_buffer->next_to_watch = NULL;
6416
6417                 /* update the statistics for this packet */
6418                 total_bytes += tx_buffer->bytecount;
6419                 total_packets += tx_buffer->gso_segs;
6420
6421 #ifdef HAVE_HW_TIME_STAMP
6422                 /* retrieve hardware timestamp */
6423                 igb_tx_hwtstamp(q_vector, tx_buffer);
6424
6425 #endif
6426                 /* free the skb */
6427                 dev_kfree_skb_any(tx_buffer->skb);
6428
6429                 /* unmap skb header data */
6430                 dma_unmap_single(tx_ring->dev,
6431                                  dma_unmap_addr(tx_buffer, dma),
6432                                  dma_unmap_len(tx_buffer, len),
6433                                  DMA_TO_DEVICE);
6434
6435                 /* clear tx_buffer data */
6436                 tx_buffer->skb = NULL;
6437                 dma_unmap_len_set(tx_buffer, len, 0);
6438
6439                 /* clear last DMA location and unmap remaining buffers */
6440                 while (tx_desc != eop_desc) {
6441                         tx_buffer++;
6442                         tx_desc++;
6443                         i++;
6444                         if (unlikely(!i)) {
6445                                 i -= tx_ring->count;
6446                                 tx_buffer = tx_ring->tx_buffer_info;
6447                                 tx_desc = IGB_TX_DESC(tx_ring, 0);
6448                         }
6449
6450                         /* unmap any remaining paged data */
6451                         if (dma_unmap_len(tx_buffer, len)) {
6452                                 dma_unmap_page(tx_ring->dev,
6453                                                dma_unmap_addr(tx_buffer, dma),
6454                                                dma_unmap_len(tx_buffer, len),
6455                                                DMA_TO_DEVICE);
6456                                 dma_unmap_len_set(tx_buffer, len, 0);
6457                         }
6458                 }
6459
6460                 /* move us one more past the eop_desc for start of next pkt */
6461                 tx_buffer++;
6462                 tx_desc++;
6463                 i++;
6464                 if (unlikely(!i)) {
6465                         i -= tx_ring->count;
6466                         tx_buffer = tx_ring->tx_buffer_info;
6467                         tx_desc = IGB_TX_DESC(tx_ring, 0);
6468                 }
6469         }
6470
6471 #ifdef CONFIG_BQL
6472         netdev_tx_completed_queue(txring_txq(tx_ring),
6473                                   total_packets, total_bytes);
6474 #endif /* CONFIG_BQL */
6475
6476         i += tx_ring->count;
6477         tx_ring->next_to_clean = i;
6478         tx_ring->tx_stats.bytes += total_bytes;
6479         tx_ring->tx_stats.packets += total_packets;
6480         q_vector->tx.total_bytes += total_bytes;
6481         q_vector->tx.total_packets += total_packets;
6482
6483         if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
6484                 struct e1000_hw *hw = &adapter->hw;
6485
6486                 eop_desc = tx_buffer->next_to_watch;
6487
6488                 /* Detect a transmit hang in hardware, this serializes the
6489                  * check with the clearing of time_stamp and movement of i */
6490                 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
6491                 if (eop_desc &&
6492                     time_after(jiffies, tx_buffer->time_stamp +
6493                                (adapter->tx_timeout_factor * HZ))
6494                     && !(E1000_READ_REG(hw, E1000_STATUS) &
6495                          E1000_STATUS_TXOFF)) {
6496
6497                         /* detected Tx unit hang */
6498                         dev_err(tx_ring->dev,
6499                                 "Detected Tx Unit Hang\n"
6500                                 "  Tx Queue             <%d>\n"
6501                                 "  TDH                  <%x>\n"
6502                                 "  TDT                  <%x>\n"
6503                                 "  next_to_use          <%x>\n"
6504                                 "  next_to_clean        <%x>\n"
6505                                 "buffer_info[next_to_clean]\n"
6506                                 "  time_stamp           <%lx>\n"
6507                                 "  next_to_watch        <%p>\n"
6508                                 "  jiffies              <%lx>\n"
6509                                 "  desc.status          <%x>\n",
6510                                 tx_ring->queue_index,
6511                                 E1000_READ_REG(hw, E1000_TDH(tx_ring->reg_idx)),
6512                                 readl(tx_ring->tail),
6513                                 tx_ring->next_to_use,
6514                                 tx_ring->next_to_clean,
6515                                 tx_buffer->time_stamp,
6516                                 eop_desc,
6517                                 jiffies,
6518                                 eop_desc->wb.status);
6519                         if (netif_is_multiqueue(netdev_ring(tx_ring)))
6520                                 netif_stop_subqueue(netdev_ring(tx_ring),
6521                                                     ring_queue_index(tx_ring));
6522                         else
6523                                 netif_stop_queue(netdev_ring(tx_ring));
6524
6525                         /* we are about to reset, no point in enabling stuff */
6526                         return true;
6527                 }
6528         }
6529
6530         if (unlikely(total_packets &&
6531                      netif_carrier_ok(netdev_ring(tx_ring)) &&
6532                      igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
6533                 /* Make sure that anybody stopping the queue after this
6534                  * sees the new next_to_clean.
6535                  */
6536                 smp_mb();
6537                 if (netif_is_multiqueue(netdev_ring(tx_ring))) {
6538                         if (__netif_subqueue_stopped(netdev_ring(tx_ring),
6539                                                      ring_queue_index(tx_ring)) &&
6540                             !(test_bit(__IGB_DOWN, &adapter->state))) {
6541                                 netif_wake_subqueue(netdev_ring(tx_ring),
6542                                                     ring_queue_index(tx_ring));
6543                                 tx_ring->tx_stats.restart_queue++;
6544                         }
6545                 } else {
6546                         if (netif_queue_stopped(netdev_ring(tx_ring)) &&
6547                             !(test_bit(__IGB_DOWN, &adapter->state))) {
6548                                 netif_wake_queue(netdev_ring(tx_ring));
6549                                 tx_ring->tx_stats.restart_queue++;
6550                         }
6551                 }
6552         }
6553
6554         return !!budget;
6555 }
6556
6557 #ifdef HAVE_VLAN_RX_REGISTER
6558 /**
6559  * igb_receive_skb - helper function to handle rx indications
6560  * @q_vector: structure containing interrupt and ring information
6561  * @skb: packet to send up
6562  **/
6563 static void igb_receive_skb(struct igb_q_vector *q_vector,
6564                             struct sk_buff *skb)
6565 {
6566         struct vlan_group **vlgrp = netdev_priv(skb->dev);
6567
6568         if (IGB_CB(skb)->vid) {
6569                 if (*vlgrp) {
6570                         vlan_gro_receive(&q_vector->napi, *vlgrp,
6571                                          IGB_CB(skb)->vid, skb);
6572                 } else {
6573                         dev_kfree_skb_any(skb);
6574                 }
6575         } else {
6576                 napi_gro_receive(&q_vector->napi, skb);
6577         }
6578 }
6579
6580 #endif /* HAVE_VLAN_RX_REGISTER */
6581 static inline void igb_rx_checksum(struct igb_ring *ring,
6582                                    union e1000_adv_rx_desc *rx_desc,
6583                                    struct sk_buff *skb)
6584 {
6585         skb_checksum_none_assert(skb);
6586
6587         /* Ignore Checksum bit is set */
6588         if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
6589                 return;
6590
6591         /* Rx checksum disabled via ethtool */
6592 #ifdef HAVE_NDO_SET_FEATURES
6593         if (!(netdev_ring(ring)->features & NETIF_F_RXCSUM))
6594 #else
6595         if (!test_bit(IGB_RING_FLAG_RX_CSUM, &ring->flags))
6596 #endif
6597                 return;
6598
6599         /* TCP/UDP checksum error bit is set */
6600         if (igb_test_staterr(rx_desc,
6601                              E1000_RXDEXT_STATERR_TCPE |
6602                              E1000_RXDEXT_STATERR_IPE)) {
6603                 /*
6604                  * work around errata with sctp packets where the TCPE aka
6605                  * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
6606                  * packets, (aka let the stack check the crc32c)
6607                  */
6608                 if (!((skb->len == 60) &&
6609                       test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags)))
6610                         ring->rx_stats.csum_err++;
6611
6612                 /* let the stack verify checksum errors */
6613                 return;
6614         }
6615         /* It must be a TCP or UDP packet with a valid checksum */
6616         if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
6617                                       E1000_RXD_STAT_UDPCS))
6618                 skb->ip_summed = CHECKSUM_UNNECESSARY;
6619 }
6620
6621 #ifdef NETIF_F_RXHASH
6622 static inline void igb_rx_hash(struct igb_ring *ring,
6623                                union e1000_adv_rx_desc *rx_desc,
6624                                struct sk_buff *skb)
6625 {
6626         if (netdev_ring(ring)->features & NETIF_F_RXHASH)
6627                 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
6628 }
6629
6630 #endif
6631 #ifdef HAVE_HW_TIME_STAMP
6632 static void igb_rx_hwtstamp(struct igb_q_vector *q_vector,
6633                             union e1000_adv_rx_desc *rx_desc,
6634                             struct sk_buff *skb)
6635 {
6636         struct igb_adapter *adapter = q_vector->adapter;
6637         struct e1000_hw *hw = &adapter->hw;
6638         u64 regval;
6639
6640         if (!igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP |
6641                                        E1000_RXDADV_STAT_TS))
6642                 return;
6643
6644         /*
6645          * If this bit is set, then the RX registers contain the time stamp. No
6646          * other packet will be time stamped until we read these registers, so
6647          * read the registers to make them available again. Because only one
6648          * packet can be time stamped at a time, we know that the register
6649          * values must belong to this one here and therefore we don't need to
6650          * compare any of the additional attributes stored for it.
6651          *
6652          * If nothing went wrong, then it should have a skb_shared_tx that we
6653          * can turn into a skb_shared_hwtstamps.
6654          */
6655         if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
6656                 u32 *stamp = (u32 *)skb->data;
6657                 regval = le32_to_cpu(*(stamp + 2));
6658                 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
6659                 skb_pull(skb, IGB_TS_HDR_LEN);
6660         } else {
6661                 if(!(E1000_READ_REG(hw, E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
6662                         return;
6663
6664                 regval = E1000_READ_REG(hw, E1000_RXSTMPL);
6665                 regval |= (u64)E1000_READ_REG(hw, E1000_RXSTMPH) << 32;
6666         }
6667
6668         igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
6669 }
6670 #endif
6671 static void igb_rx_vlan(struct igb_ring *ring,
6672                         union e1000_adv_rx_desc *rx_desc,
6673                         struct sk_buff *skb)
6674 {
6675         if (igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
6676                 u16 vid = 0;
6677                 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
6678                     test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags))
6679                         vid = be16_to_cpu(rx_desc->wb.upper.vlan);
6680                 else
6681                         vid = le16_to_cpu(rx_desc->wb.upper.vlan);
6682 #ifdef HAVE_VLAN_RX_REGISTER
6683                 IGB_CB(skb)->vid = vid;
6684         } else {
6685                 IGB_CB(skb)->vid = 0;
6686 #else
6687
6688 #ifdef HAVE_VLAN_PROTOCOL
6689                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
6690 #else
6691                 __vlan_hwaccel_put_tag(skb, vid);
6692 #endif
6693
6694
6695 #endif
6696         }
6697 }
6698
6699 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
6700 static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
6701 {
6702         /* HW will not DMA in data larger than the given buffer, even if it
6703          * parses the (NFS, of course) header to be larger.  In that case, it
6704          * fills the header buffer and spills the rest into the page.
6705          */
6706         u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info) &
6707                    E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
6708         if (hlen > IGB_RX_HDR_LEN)
6709                 hlen = IGB_RX_HDR_LEN;
6710         return hlen;
6711 }
6712
6713 #endif
6714 #ifndef IGB_NO_LRO
6715 /**
6716  * igb_merge_active_tail - merge active tail into lro skb
6717  * @tail: pointer to active tail in frag_list
6718  *
6719  * This function merges the length and data of an active tail into the
6720  * skb containing the frag_list.  It resets the tail's pointer to the head,
6721  * but it leaves the heads pointer to tail intact.
6722  **/
6723 static inline struct sk_buff *igb_merge_active_tail(struct sk_buff *tail)
6724 {
6725         struct sk_buff *head = IGB_CB(tail)->head;
6726
6727         if (!head)
6728                 return tail;
6729
6730         head->len += tail->len;
6731         head->data_len += tail->len;
6732         head->truesize += tail->len;
6733
6734         IGB_CB(tail)->head = NULL;
6735
6736         return head;
6737 }
6738
6739 /**
6740  * igb_add_active_tail - adds an active tail into the skb frag_list
6741  * @head: pointer to the start of the skb
6742  * @tail: pointer to active tail to add to frag_list
6743  *
6744  * This function adds an active tail to the end of the frag list.  This tail
6745  * will still be receiving data so we cannot yet ad it's stats to the main
6746  * skb.  That is done via igb_merge_active_tail.
6747  **/
6748 static inline void igb_add_active_tail(struct sk_buff *head, struct sk_buff *tail)
6749 {
6750         struct sk_buff *old_tail = IGB_CB(head)->tail;
6751
6752         if (old_tail) {
6753                 igb_merge_active_tail(old_tail);
6754                 old_tail->next = tail;
6755         } else {
6756                 skb_shinfo(head)->frag_list = tail;
6757         }
6758
6759         IGB_CB(tail)->head = head;
6760         IGB_CB(head)->tail = tail;
6761
6762         IGB_CB(head)->append_cnt++;
6763 }
6764
6765 /**
6766  * igb_close_active_frag_list - cleanup pointers on a frag_list skb
6767  * @head: pointer to head of an active frag list
6768  *
6769  * This function will clear the frag_tail_tracker pointer on an active
6770  * frag_list and returns true if the pointer was actually set
6771  **/
6772 static inline bool igb_close_active_frag_list(struct sk_buff *head)
6773 {
6774         struct sk_buff *tail = IGB_CB(head)->tail;
6775
6776         if (!tail)
6777                 return false;
6778
6779         igb_merge_active_tail(tail);
6780
6781         IGB_CB(head)->tail = NULL;
6782
6783         return true;
6784 }
6785
6786 /**
6787  * igb_can_lro - returns true if packet is TCP/IPV4 and LRO is enabled
6788  * @adapter: board private structure
6789  * @rx_desc: pointer to the rx descriptor
6790  * @skb: pointer to the skb to be merged
6791  *
6792  **/
6793 static inline bool igb_can_lro(struct igb_ring *rx_ring,
6794                                union e1000_adv_rx_desc *rx_desc,
6795                                struct sk_buff *skb)
6796 {
6797         struct iphdr *iph = (struct iphdr *)skb->data;
6798         __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
6799
6800         /* verify LRO is enabled */
6801         if (!(netdev_ring(rx_ring)->features & NETIF_F_LRO))
6802                 return false;
6803
6804         /* verify hardware indicates this is IPv4/TCP */
6805         if((!(pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_TCP)) ||
6806             !(pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_IPV4))))
6807                 return false;
6808
6809         /* verify the header is large enough for us to read IP/TCP fields */
6810         if (!pskb_may_pull(skb, sizeof(struct igb_lrohdr)))
6811                 return false;
6812
6813         /* verify there are no VLANs on packet */
6814         if (skb->protocol != __constant_htons(ETH_P_IP))
6815                 return false;
6816
6817         /* ensure we are version 4 with no options */
6818         if (*(u8 *)iph != 0x45)
6819                 return false;
6820
6821         /* .. and the packet is not fragmented */
6822         if (iph->frag_off & htons(IP_MF | IP_OFFSET))
6823                 return false;
6824
6825         /* .. and that next header is TCP */
6826         if (iph->protocol != IPPROTO_TCP)
6827                 return false;
6828
6829         return true;
6830 }
6831
6832 static inline struct igb_lrohdr *igb_lro_hdr(struct sk_buff *skb)
6833 {
6834         return (struct igb_lrohdr *)skb->data;
6835 }
6836
6837 /**
6838  * igb_lro_flush - Indicate packets to upper layer.
6839  *
6840  * Update IP and TCP header part of head skb if more than one
6841  * skb's chained and indicate packets to upper layer.
6842  **/
6843 static void igb_lro_flush(struct igb_q_vector *q_vector,
6844                           struct sk_buff *skb)
6845 {
6846         struct igb_lro_list *lrolist = q_vector->lrolist;
6847
6848         __skb_unlink(skb, &lrolist->active);
6849
6850         if (IGB_CB(skb)->append_cnt) {
6851                 struct igb_lrohdr *lroh = igb_lro_hdr(skb);
6852
6853                 /* close any active lro contexts */
6854                 igb_close_active_frag_list(skb);
6855
6856                 /* incorporate ip header and re-calculate checksum */
6857                 lroh->iph.tot_len = ntohs(skb->len);
6858                 lroh->iph.check = 0;
6859
6860                 /* header length is 5 since we know no options exist */
6861                 lroh->iph.check = ip_fast_csum((u8 *)lroh, 5);
6862
6863                 /* clear TCP checksum to indicate we are an LRO frame */
6864                 lroh->th.check = 0;
6865
6866                 /* incorporate latest timestamp into the tcp header */
6867                 if (IGB_CB(skb)->tsecr) {
6868                         lroh->ts[2] = IGB_CB(skb)->tsecr;
6869                         lroh->ts[1] = htonl(IGB_CB(skb)->tsval);
6870                 }
6871 #ifdef NETIF_F_TSO
6872
6873                 skb_shinfo(skb)->gso_size = IGB_CB(skb)->mss;
6874 #endif
6875         }
6876
6877 #ifdef HAVE_VLAN_RX_REGISTER
6878         igb_receive_skb(q_vector, skb);
6879 #else
6880         napi_gro_receive(&q_vector->napi, skb);
6881 #endif
6882         lrolist->stats.flushed++;
6883 }
6884
6885 static void igb_lro_flush_all(struct igb_q_vector *q_vector)
6886 {
6887         struct igb_lro_list *lrolist = q_vector->lrolist;
6888         struct sk_buff *skb, *tmp;
6889
6890         skb_queue_reverse_walk_safe(&lrolist->active, skb, tmp)
6891                 igb_lro_flush(q_vector, skb);
6892 }
6893
6894 /*
6895  * igb_lro_header_ok - Main LRO function.
6896  **/
6897 static void igb_lro_header_ok(struct sk_buff *skb)
6898 {
6899         struct igb_lrohdr *lroh = igb_lro_hdr(skb);
6900         u16 opt_bytes, data_len;
6901
6902         IGB_CB(skb)->tail = NULL;
6903         IGB_CB(skb)->tsecr = 0;
6904         IGB_CB(skb)->append_cnt = 0;
6905         IGB_CB(skb)->mss = 0;
6906
6907         /* ensure that the checksum is valid */
6908         if (skb->ip_summed != CHECKSUM_UNNECESSARY)
6909                 return;
6910
6911         /* If we see CE codepoint in IP header, packet is not mergeable */
6912         if (INET_ECN_is_ce(ipv4_get_dsfield(&lroh->iph)))
6913                 return;
6914
6915         /* ensure no bits set besides ack or psh */
6916         if (lroh->th.fin || lroh->th.syn || lroh->th.rst ||
6917             lroh->th.urg || lroh->th.ece || lroh->th.cwr ||
6918             !lroh->th.ack)
6919                 return;
6920
6921         /* store the total packet length */
6922         data_len = ntohs(lroh->iph.tot_len);
6923
6924         /* remove any padding from the end of the skb */
6925         __pskb_trim(skb, data_len);
6926
6927         /* remove header length from data length */
6928         data_len -= sizeof(struct igb_lrohdr);
6929
6930         /*
6931          * check for timestamps. Since the only option we handle are timestamps,
6932          * we only have to handle the simple case of aligned timestamps
6933          */
6934         opt_bytes = (lroh->th.doff << 2) - sizeof(struct tcphdr);
6935         if (opt_bytes != 0) {
6936                 if ((opt_bytes != TCPOLEN_TSTAMP_ALIGNED) ||
6937                     !pskb_may_pull(skb, sizeof(struct igb_lrohdr) +
6938                                         TCPOLEN_TSTAMP_ALIGNED) ||
6939                     (lroh->ts[0] != htonl((TCPOPT_NOP << 24) |
6940                                              (TCPOPT_NOP << 16) |
6941                                              (TCPOPT_TIMESTAMP << 8) |
6942                                               TCPOLEN_TIMESTAMP)) ||
6943                     (lroh->ts[2] == 0)) {
6944                         return;
6945                 }
6946
6947                 IGB_CB(skb)->tsval = ntohl(lroh->ts[1]);
6948                 IGB_CB(skb)->tsecr = lroh->ts[2];
6949
6950                 data_len -= TCPOLEN_TSTAMP_ALIGNED;
6951         }
6952
6953         /* record data_len as mss for the packet */
6954         IGB_CB(skb)->mss = data_len;
6955         IGB_CB(skb)->next_seq = ntohl(lroh->th.seq);
6956 }
6957
6958 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
6959 static bool igb_merge_frags(struct sk_buff *lro_skb, struct sk_buff *new_skb)
6960 {
6961         struct sk_buff *tail;
6962         struct skb_shared_info *tail_info;
6963         struct skb_shared_info *new_skb_info;
6964         u16 data_len;
6965
6966         /* header must be empty to pull frags into current skb */
6967         if (skb_headlen(new_skb))
6968                 return false;
6969
6970         if (IGB_CB(lro_skb)->tail)
6971                 tail = IGB_CB(lro_skb)->tail;
6972         else
6973                 tail = lro_skb;
6974
6975         tail_info = skb_shinfo(tail);
6976         new_skb_info = skb_shinfo(new_skb);
6977
6978         /* make sure we have room in frags list */
6979         if (new_skb_info->nr_frags >= (MAX_SKB_FRAGS - tail_info->nr_frags))
6980                 return false;
6981
6982         /* bump append count */
6983         IGB_CB(lro_skb)->append_cnt++;
6984
6985         /* copy frags into the last skb */
6986         memcpy(tail_info->frags + tail_info->nr_frags,
6987                new_skb_info->frags,
6988                new_skb_info->nr_frags * sizeof(skb_frag_t));
6989
6990         /* copy size data over */
6991         tail_info->nr_frags += new_skb_info->nr_frags;
6992         data_len = IGB_CB(new_skb)->mss;
6993         tail->len += data_len;
6994         tail->data_len += data_len;
6995         tail->truesize += data_len;
6996
6997         /* wipe record of data from new_skb */
6998         new_skb_info->nr_frags = 0;
6999         new_skb->len = new_skb->data_len = 0;
7000         new_skb->truesize -= data_len;
7001         new_skb->data = new_skb->head + NET_SKB_PAD + NET_IP_ALIGN;
7002         skb_reset_tail_pointer(new_skb);
7003         new_skb->protocol = 0;
7004         new_skb->ip_summed = CHECKSUM_NONE;
7005 #ifdef HAVE_VLAN_RX_REGISTER
7006         IGB_CB(new_skb)->vid = 0;
7007 #else
7008         new_skb->vlan_tci = 0;
7009 #endif
7010
7011         return true;
7012 }
7013
7014 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7015 /**
7016  * igb_lro_queue - if able, queue skb into lro chain
7017  * @q_vector: structure containing interrupt and ring information
7018  * @new_skb: pointer to current skb being checked
7019  *
7020  * Checks whether the skb given is eligible for LRO and if that's
7021  * fine chains it to the existing lro_skb based on flowid. If an LRO for
7022  * the flow doesn't exist create one.
7023  **/
7024 static struct sk_buff *igb_lro_queue(struct igb_q_vector *q_vector,
7025                                        struct sk_buff *new_skb)
7026 {
7027         struct sk_buff *lro_skb;
7028         struct igb_lro_list *lrolist = q_vector->lrolist;
7029         struct igb_lrohdr *lroh = igb_lro_hdr(new_skb);
7030         __be32 saddr = lroh->iph.saddr;
7031         __be32 daddr = lroh->iph.daddr;
7032         __be32 tcp_ports = *(__be32 *)&lroh->th;
7033         u16 data_len;
7034 #ifdef HAVE_VLAN_RX_REGISTER
7035         u16 vid = IGB_CB(new_skb)->vid;
7036 #else
7037         u16 vid = new_skb->vlan_tci;
7038 #endif
7039
7040         igb_lro_header_ok(new_skb);
7041
7042         /*
7043          * we have a packet that might be eligible for LRO,
7044          * so see if it matches anything we might expect
7045          */
7046         skb_queue_walk(&lrolist->active, lro_skb) {
7047                 if (*(__be32 *)&igb_lro_hdr(lro_skb)->th != tcp_ports ||
7048                     igb_lro_hdr(lro_skb)->iph.saddr != saddr ||
7049                     igb_lro_hdr(lro_skb)->iph.daddr != daddr)
7050                         continue;
7051
7052 #ifdef HAVE_VLAN_RX_REGISTER
7053                 if (IGB_CB(lro_skb)->vid != vid)
7054 #else
7055                 if (lro_skb->vlan_tci != vid)
7056 #endif
7057                         continue;
7058
7059                 /* out of order packet */
7060                 if (IGB_CB(lro_skb)->next_seq != IGB_CB(new_skb)->next_seq) {
7061                         igb_lro_flush(q_vector, lro_skb);
7062                         IGB_CB(new_skb)->mss = 0;
7063                         break;
7064                 }
7065
7066                 /* TCP timestamp options have changed */
7067                 if (!IGB_CB(lro_skb)->tsecr != !IGB_CB(new_skb)->tsecr) {
7068                         igb_lro_flush(q_vector, lro_skb);
7069                         break;
7070                 }
7071
7072                 /* make sure timestamp values are increasing */
7073                 if (IGB_CB(lro_skb)->tsecr &&
7074                     IGB_CB(lro_skb)->tsval > IGB_CB(new_skb)->tsval) {
7075                         igb_lro_flush(q_vector, lro_skb);
7076                         IGB_CB(new_skb)->mss = 0;
7077                         break;
7078                 }
7079
7080                 data_len = IGB_CB(new_skb)->mss;
7081
7082                 /*
7083                  * malformed header, no tcp data, resultant packet would
7084                  * be too large, or new skb is larger than our current mss.
7085                  */
7086                 if (data_len == 0 ||
7087                     data_len > IGB_CB(lro_skb)->mss ||
7088                     data_len > IGB_CB(lro_skb)->free) {
7089                         igb_lro_flush(q_vector, lro_skb);
7090                         break;
7091                 }
7092
7093                 /* ack sequence numbers or window size has changed */
7094                 if (igb_lro_hdr(lro_skb)->th.ack_seq != lroh->th.ack_seq ||
7095                     igb_lro_hdr(lro_skb)->th.window != lroh->th.window) {
7096                         igb_lro_flush(q_vector, lro_skb);
7097                         break;
7098                 }
7099
7100                 /* Remove IP and TCP header*/
7101                 skb_pull(new_skb, new_skb->len - data_len);
7102
7103                 /* update timestamp and timestamp echo response */
7104                 IGB_CB(lro_skb)->tsval = IGB_CB(new_skb)->tsval;
7105                 IGB_CB(lro_skb)->tsecr = IGB_CB(new_skb)->tsecr;
7106
7107                 /* update sequence and free space */
7108                 IGB_CB(lro_skb)->next_seq += data_len;
7109                 IGB_CB(lro_skb)->free -= data_len;
7110
7111 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7112                 /* if header is empty pull pages into current skb */
7113                 if (igb_merge_frags(lro_skb, new_skb)) {
7114                         lrolist->stats.recycled++;
7115                 } else {
7116 #endif
7117                         /* chain this new skb in frag_list */
7118                         igb_add_active_tail(lro_skb, new_skb);
7119                         new_skb = NULL;
7120 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7121                 }
7122 #endif
7123
7124                 if ((data_len < IGB_CB(lro_skb)->mss) || lroh->th.psh) {
7125                         igb_lro_hdr(lro_skb)->th.psh |= lroh->th.psh;
7126                         igb_lro_flush(q_vector, lro_skb);
7127                 }
7128
7129                 lrolist->stats.coal++;
7130                 return new_skb;
7131         }
7132
7133         if (IGB_CB(new_skb)->mss && !lroh->th.psh) {
7134                 /* if we are at capacity flush the tail */
7135                 if (skb_queue_len(&lrolist->active) >= IGB_LRO_MAX) {
7136                         lro_skb = skb_peek_tail(&lrolist->active);
7137                         if (lro_skb)
7138                                 igb_lro_flush(q_vector, lro_skb);
7139                 }
7140
7141                 /* update sequence and free space */
7142                 IGB_CB(new_skb)->next_seq += IGB_CB(new_skb)->mss;
7143                 IGB_CB(new_skb)->free = 65521 - new_skb->len;
7144
7145                 /* .. and insert at the front of the active list */
7146                 __skb_queue_head(&lrolist->active, new_skb);
7147
7148                 lrolist->stats.coal++;
7149                 return NULL;
7150         }
7151
7152         /* packet not handled by any of the above, pass it to the stack */
7153 #ifdef HAVE_VLAN_RX_REGISTER
7154         igb_receive_skb(q_vector, new_skb);
7155 #else
7156         napi_gro_receive(&q_vector->napi, new_skb);
7157 #endif
7158         return NULL;
7159 }
7160
7161 #endif /* IGB_NO_LRO */
7162 static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
7163 {
7164         struct igb_ring *rx_ring = q_vector->rx.ring;
7165         union e1000_adv_rx_desc *rx_desc;
7166 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7167         const int current_node = numa_node_id();
7168 #endif
7169         unsigned int total_bytes = 0, total_packets = 0;
7170         u16 cleaned_count = igb_desc_unused(rx_ring);
7171         u16 i = rx_ring->next_to_clean;
7172
7173         rx_desc = IGB_RX_DESC(rx_ring, i);
7174
7175         while (igb_test_staterr(rx_desc, E1000_RXD_STAT_DD)) {
7176                 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
7177                 struct sk_buff *skb = buffer_info->skb;
7178                 union e1000_adv_rx_desc *next_rxd;
7179
7180                 buffer_info->skb = NULL;
7181                 prefetch(skb->data);
7182
7183                 i++;
7184                 if (i == rx_ring->count)
7185                         i = 0;
7186
7187                 next_rxd = IGB_RX_DESC(rx_ring, i);
7188                 prefetch(next_rxd);
7189
7190                 /*
7191                  * This memory barrier is needed to keep us from reading
7192                  * any other fields out of the rx_desc until we know the
7193                  * RXD_STAT_DD bit is set
7194                  */
7195                 rmb();
7196
7197 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7198                 __skb_put(skb, le16_to_cpu(rx_desc->wb.upper.length));
7199                 dma_unmap_single(rx_ring->dev, buffer_info->dma,
7200                                  rx_ring->rx_buffer_len,
7201                                  DMA_FROM_DEVICE);
7202                 buffer_info->dma = 0;
7203
7204 #else
7205                 if (!skb_is_nonlinear(skb)) {
7206                         __skb_put(skb, igb_get_hlen(rx_desc));
7207                         dma_unmap_single(rx_ring->dev, buffer_info->dma,
7208                                          IGB_RX_HDR_LEN,
7209                                          DMA_FROM_DEVICE);
7210                         buffer_info->dma = 0;
7211                 }
7212
7213                 if (rx_desc->wb.upper.length) {
7214                         u16 length = le16_to_cpu(rx_desc->wb.upper.length);
7215
7216                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
7217                                            buffer_info->page,
7218                                            buffer_info->page_offset,
7219                                            length);
7220
7221                         skb->len += length;
7222                         skb->data_len += length;
7223                         skb->truesize += length;
7224
7225                         if ((page_count(buffer_info->page) != 1) ||
7226                             (page_to_nid(buffer_info->page) != current_node))
7227                                 buffer_info->page = NULL;
7228                         else
7229                                 get_page(buffer_info->page);
7230
7231                         dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
7232                                        PAGE_SIZE / 2, DMA_FROM_DEVICE);
7233                         buffer_info->page_dma = 0;
7234                 }
7235
7236                 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)) {
7237                         struct igb_rx_buffer *next_buffer;
7238                         next_buffer = &rx_ring->rx_buffer_info[i];
7239                         buffer_info->skb = next_buffer->skb;
7240                         buffer_info->dma = next_buffer->dma;
7241                         next_buffer->skb = skb;
7242                         next_buffer->dma = 0;
7243                         goto next_desc;
7244                 }
7245
7246 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7247                 if (igb_test_staterr(rx_desc,
7248                                      E1000_RXDEXT_ERR_FRAME_ERR_MASK)) {
7249                         dev_kfree_skb_any(skb);
7250                         goto next_desc;
7251                 }
7252
7253 #ifdef HAVE_HW_TIME_STAMP
7254                 igb_rx_hwtstamp(q_vector, rx_desc, skb);
7255 #endif
7256 #ifdef NETIF_F_RXHASH
7257                 igb_rx_hash(rx_ring, rx_desc, skb);
7258 #endif
7259                 igb_rx_checksum(rx_ring, rx_desc, skb);
7260                 igb_rx_vlan(rx_ring, rx_desc, skb);
7261
7262                 total_bytes += skb->len;
7263                 total_packets++;
7264
7265                 skb->protocol = eth_type_trans(skb, netdev_ring(rx_ring));
7266
7267 #ifndef IGB_NO_LRO
7268                 if (igb_can_lro(rx_ring, rx_desc, skb))
7269                         buffer_info->skb = igb_lro_queue(q_vector, skb);
7270                 else
7271 #endif
7272 #ifdef HAVE_VLAN_RX_REGISTER
7273                         igb_receive_skb(q_vector, skb);
7274 #else
7275                         napi_gro_receive(&q_vector->napi, skb);
7276 #endif
7277
7278 #ifndef NETIF_F_GRO
7279                 netdev_ring(rx_ring)->last_rx = jiffies;
7280
7281 #endif
7282                 budget--;
7283 next_desc:
7284                 cleaned_count++;
7285
7286                 if (!budget)
7287                         break;
7288
7289                 /* return some buffers to hardware, one at a time is too slow */
7290                 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
7291                         igb_alloc_rx_buffers(rx_ring, cleaned_count);
7292                         cleaned_count = 0;
7293                 }
7294
7295                 /* use prefetched values */
7296                 rx_desc = next_rxd;
7297         }
7298
7299         rx_ring->next_to_clean = i;
7300         rx_ring->rx_stats.packets += total_packets;
7301         rx_ring->rx_stats.bytes += total_bytes;
7302         q_vector->rx.total_packets += total_packets;
7303         q_vector->rx.total_bytes += total_bytes;
7304
7305         if (cleaned_count)
7306                 igb_alloc_rx_buffers(rx_ring, cleaned_count);
7307
7308 #ifndef IGB_NO_LRO
7309         if (netdev_ring(rx_ring)->features & NETIF_F_LRO)
7310                 igb_lro_flush_all(q_vector);
7311
7312 #endif /* IGB_NO_LRO */
7313         return !!budget;
7314 }
7315
7316 static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
7317                                  struct igb_rx_buffer *bi)
7318 {
7319         struct sk_buff *skb = bi->skb;
7320         dma_addr_t dma = bi->dma;
7321
7322         if (dma)
7323                 return true;
7324
7325         if (likely(!skb)) {
7326 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7327                 skb = netdev_alloc_skb_ip_align(netdev_ring(rx_ring),
7328                                                 rx_ring->rx_buffer_len);
7329 #else
7330                 skb = netdev_alloc_skb_ip_align(netdev_ring(rx_ring),
7331                                                 IGB_RX_HDR_LEN);
7332 #endif
7333                 bi->skb = skb;
7334                 if (!skb) {
7335                         rx_ring->rx_stats.alloc_failed++;
7336                         return false;
7337                 }
7338
7339                 /* initialize skb for ring */
7340                 skb_record_rx_queue(skb, ring_queue_index(rx_ring));
7341         }
7342
7343 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7344         dma = dma_map_single(rx_ring->dev, skb->data,
7345                              rx_ring->rx_buffer_len, DMA_FROM_DEVICE);
7346 #else
7347         dma = dma_map_single(rx_ring->dev, skb->data,
7348                              IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
7349 #endif
7350
7351         if (dma_mapping_error(rx_ring->dev, dma)) {
7352                 rx_ring->rx_stats.alloc_failed++;
7353                 return false;
7354         }
7355
7356         bi->dma = dma;
7357         return true;
7358 }
7359
7360 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7361 static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
7362                                   struct igb_rx_buffer *bi)
7363 {
7364         struct page *page = bi->page;
7365         dma_addr_t page_dma = bi->page_dma;
7366         unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
7367
7368         if (page_dma)
7369                 return true;
7370
7371         if (!page) {
7372                 page = alloc_page(GFP_ATOMIC | __GFP_COLD);
7373                 bi->page = page;
7374                 if (unlikely(!page)) {
7375                         rx_ring->rx_stats.alloc_failed++;
7376                         return false;
7377                 }
7378         }
7379
7380         page_dma = dma_map_page(rx_ring->dev, page,
7381                                 page_offset, PAGE_SIZE / 2,
7382                                 DMA_FROM_DEVICE);
7383
7384         if (dma_mapping_error(rx_ring->dev, page_dma)) {
7385                 rx_ring->rx_stats.alloc_failed++;
7386                 return false;
7387         }
7388
7389         bi->page_dma = page_dma;
7390         bi->page_offset = page_offset;
7391         return true;
7392 }
7393
7394 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7395 /**
7396  * igb_alloc_rx_buffers - Replace used receive buffers; packet split
7397  * @adapter: address of board private structure
7398  **/
7399 void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
7400 {
7401         union e1000_adv_rx_desc *rx_desc;
7402         struct igb_rx_buffer *bi;
7403         u16 i = rx_ring->next_to_use;
7404
7405         rx_desc = IGB_RX_DESC(rx_ring, i);
7406         bi = &rx_ring->rx_buffer_info[i];
7407         i -= rx_ring->count;
7408
7409         while (cleaned_count--) {
7410                 if (!igb_alloc_mapped_skb(rx_ring, bi))
7411                         break;
7412
7413                 /* Refresh the desc even if buffer_addrs didn't change
7414                  * because each write-back erases this info. */
7415 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7416                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
7417 #else
7418                 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
7419
7420                 if (!igb_alloc_mapped_page(rx_ring, bi))
7421                         break;
7422
7423                 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
7424
7425 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7426                 rx_desc++;
7427                 bi++;
7428                 i++;
7429                 if (unlikely(!i)) {
7430                         rx_desc = IGB_RX_DESC(rx_ring, 0);
7431                         bi = rx_ring->rx_buffer_info;
7432                         i -= rx_ring->count;
7433                 }
7434
7435                 /* clear the hdr_addr for the next_to_use descriptor */
7436                 rx_desc->read.hdr_addr = 0;
7437         }
7438
7439         i += rx_ring->count;
7440
7441         if (rx_ring->next_to_use != i) {
7442                 rx_ring->next_to_use = i;
7443
7444                 /* Force memory writes to complete before letting h/w
7445                  * know there are new descriptors to fetch.  (Only
7446                  * applicable for weak-ordered memory model archs,
7447                  * such as IA-64). */
7448                 wmb();
7449                 writel(i, rx_ring->tail);
7450         }
7451 }
7452
7453 #ifdef SIOCGMIIPHY
7454 /**
7455  * igb_mii_ioctl -
7456  * @netdev:
7457  * @ifreq:
7458  * @cmd:
7459  **/
7460 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
7461 {
7462         struct igb_adapter *adapter = netdev_priv(netdev);
7463         struct mii_ioctl_data *data = if_mii(ifr);
7464
7465         if (adapter->hw.phy.media_type != e1000_media_type_copper)
7466                 return -EOPNOTSUPP;
7467
7468         switch (cmd) {
7469         case SIOCGMIIPHY:
7470                 data->phy_id = adapter->hw.phy.addr;
7471                 break;
7472         case SIOCGMIIREG:
7473                 if (!capable(CAP_NET_ADMIN))
7474                         return -EPERM;
7475                 if (e1000_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
7476                                    &data->val_out))
7477                         return -EIO;
7478                 break;
7479         case SIOCSMIIREG:
7480         default:
7481                 return -EOPNOTSUPP;
7482         }
7483         return E1000_SUCCESS;
7484 }
7485
7486 #endif
7487 #ifdef HAVE_HW_TIME_STAMP
7488 /**
7489  * igb_hwtstamp_ioctl - control hardware time stamping
7490  * @netdev:
7491  * @ifreq:
7492  * @cmd:
7493  *
7494  * Outgoing time stamping can be enabled and disabled. Play nice and
7495  * disable it when requested, although it shouldn't case any overhead
7496  * when no packet needs it. At most one packet in the queue may be
7497  * marked for time stamping, otherwise it would be impossible to tell
7498  * for sure to which packet the hardware time stamp belongs.
7499  *
7500  * Incoming time stamping has to be configured via the hardware
7501  * filters. Not all combinations are supported, in particular event
7502  * type has to be specified. Matching the kind of event packet is
7503  * not supported, with the exception of "all V2 events regardless of
7504  * level 2 or 4".
7505  *
7506  **/
7507 static int igb_hwtstamp_ioctl(struct net_device *netdev,
7508                               struct ifreq *ifr, int cmd)
7509 {
7510         struct igb_adapter *adapter = netdev_priv(netdev);
7511         struct e1000_hw *hw = &adapter->hw;
7512         struct hwtstamp_config config;
7513         u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
7514         u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
7515         u32 tsync_rx_cfg = 0;
7516         bool is_l4 = false;
7517         bool is_l2 = false;
7518         u32 regval;
7519
7520         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
7521                 return -EFAULT;
7522
7523         /* reserved for future extensions */
7524         if (config.flags)
7525                 return -EINVAL;
7526
7527         switch (config.tx_type) {
7528         case HWTSTAMP_TX_OFF:
7529                 tsync_tx_ctl = 0;
7530         case HWTSTAMP_TX_ON:
7531                 break;
7532         default:
7533                 return -ERANGE;
7534         }
7535
7536         switch (config.rx_filter) {
7537         case HWTSTAMP_FILTER_NONE:
7538                 tsync_rx_ctl = 0;
7539                 break;
7540         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
7541         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
7542         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
7543         case HWTSTAMP_FILTER_ALL:
7544                 /*
7545                  * register TSYNCRXCFG must be set, therefore it is not
7546                  * possible to time stamp both Sync and Delay_Req messages
7547                  * => fall back to time stamping all packets
7548                  */
7549                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
7550                 config.rx_filter = HWTSTAMP_FILTER_ALL;
7551                 break;
7552         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
7553                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
7554                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
7555                 is_l4 = true;
7556                 break;
7557         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
7558                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
7559                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
7560                 is_l4 = true;
7561                 break;
7562         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
7563         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
7564                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
7565                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
7566                 is_l2 = true;
7567                 is_l4 = true;
7568                 config.rx_filter = HWTSTAMP_FILTER_SOME;
7569                 break;
7570         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
7571         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
7572                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
7573                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
7574                 is_l2 = true;
7575                 is_l4 = true;
7576                 config.rx_filter = HWTSTAMP_FILTER_SOME;
7577                 break;
7578         case HWTSTAMP_FILTER_PTP_V2_EVENT:
7579         case HWTSTAMP_FILTER_PTP_V2_SYNC:
7580         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
7581                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
7582                 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
7583                 is_l2 = true;
7584                 is_l4 = true;
7585                 break;
7586         default:
7587                 return -ERANGE;
7588         }
7589
7590         if (hw->mac.type == e1000_82575) {
7591                 if (tsync_rx_ctl | tsync_tx_ctl)
7592                         return -EINVAL;
7593                 return 0;
7594         }
7595
7596 #ifdef IGB_PER_PKT_TIMESTAMP
7597         /*
7598          * Per-packet timestamping only works if all packets are
7599          * timestamped, so enable timestamping in all packets as
7600          * long as one rx filter was configured.
7601          */
7602         if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) {
7603                 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
7604                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
7605         }
7606 #endif
7607
7608         /* enable/disable TX */
7609         regval = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
7610         regval &= ~E1000_TSYNCTXCTL_ENABLED;
7611         regval |= tsync_tx_ctl;
7612         E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, regval);
7613
7614         /* enable/disable RX */
7615         regval = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
7616         regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
7617         regval |= tsync_rx_ctl;
7618         E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, regval);
7619
7620         /* define which PTP packets are time stamped */
7621         E1000_WRITE_REG(hw, E1000_TSYNCRXCFG, tsync_rx_cfg);
7622
7623         /* define ethertype filter for timestamped packets */
7624         if (is_l2)
7625                 E1000_WRITE_REG(hw, E1000_ETQF(3),
7626                                 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
7627                                  E1000_ETQF_1588 | /* enable timestamping */
7628                                  ETH_P_1588));     /* 1588 eth protocol type */
7629         else
7630                 E1000_WRITE_REG(hw, E1000_ETQF(3), 0);
7631
7632 #define PTP_PORT 319
7633         /* L4 Queue Filter[3]: filter by destination port and protocol */
7634         if (is_l4) {
7635                 u32 ftqf = (IPPROTO_UDP /* UDP */
7636                         | E1000_FTQF_VF_BP /* VF not compared */
7637                         | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
7638                         | E1000_FTQF_MASK); /* mask all inputs */
7639                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
7640
7641                 E1000_WRITE_REG(hw, E1000_IMIR(3), htons(PTP_PORT));
7642                 E1000_WRITE_REG(hw, E1000_IMIREXT(3),
7643                                 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
7644                 if (hw->mac.type == e1000_82576) {
7645                         /* enable source port check */
7646                         E1000_WRITE_REG(hw, E1000_SPQF(3), htons(PTP_PORT));
7647                         ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
7648                 }
7649                 E1000_WRITE_REG(hw, E1000_FTQF(3), ftqf);
7650         } else {
7651                 E1000_WRITE_REG(hw, E1000_FTQF(3), E1000_FTQF_MASK);
7652         }
7653         E1000_WRITE_FLUSH(hw);
7654
7655         adapter->hwtstamp_config = config;
7656
7657         /* clear TX/RX time stamp registers, just to be sure */
7658         regval = E1000_READ_REG(hw, E1000_TXSTMPH);
7659         regval = E1000_READ_REG(hw, E1000_RXSTMPH);
7660
7661         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
7662                 -EFAULT : 0;
7663 }
7664
7665 #endif
7666 /**
7667  * igb_ioctl -
7668  * @netdev:
7669  * @ifreq:
7670  * @cmd:
7671  **/
7672 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
7673 {
7674         switch (cmd) {
7675 #ifdef SIOCGMIIPHY
7676         case SIOCGMIIPHY:
7677         case SIOCGMIIREG:
7678         case SIOCSMIIREG:
7679                 return igb_mii_ioctl(netdev, ifr, cmd);
7680 #endif
7681 #ifdef HAVE_HW_TIME_STAMP
7682         case SIOCSHWTSTAMP:
7683                 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
7684 #endif
7685 #ifdef ETHTOOL_OPS_COMPAT
7686         case SIOCETHTOOL:
7687                 return ethtool_ioctl(ifr);
7688 #endif
7689         default:
7690                 return -EOPNOTSUPP;
7691         }
7692 }
7693
7694 s32 e1000_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
7695 {
7696         struct igb_adapter *adapter = hw->back;
7697         u16 cap_offset;
7698
7699         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
7700         if (!cap_offset)
7701                 return -E1000_ERR_CONFIG;
7702
7703         pci_read_config_word(adapter->pdev, cap_offset + reg, value);
7704
7705         return E1000_SUCCESS;
7706 }
7707
7708 s32 e1000_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
7709 {
7710         struct igb_adapter *adapter = hw->back;
7711         u16 cap_offset;
7712
7713         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
7714         if (!cap_offset)
7715                 return -E1000_ERR_CONFIG;
7716
7717         pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
7718
7719         return E1000_SUCCESS;
7720 }
7721
7722 #ifdef HAVE_VLAN_RX_REGISTER
7723 static void igb_vlan_mode(struct net_device *netdev, struct vlan_group *vlgrp)
7724 #else
7725 void igb_vlan_mode(struct net_device *netdev, u32 features)
7726 #endif
7727 {
7728         struct igb_adapter *adapter = netdev_priv(netdev);
7729         struct e1000_hw *hw = &adapter->hw;
7730         u32 ctrl, rctl;
7731         int i;
7732 #ifdef HAVE_VLAN_RX_REGISTER
7733         bool enable = !!vlgrp;
7734
7735         igb_irq_disable(adapter);
7736
7737         adapter->vlgrp = vlgrp;
7738
7739         if (!test_bit(__IGB_DOWN, &adapter->state))
7740                 igb_irq_enable(adapter);
7741 #else
7742         bool enable = !!(features & NETIF_F_HW_VLAN_RX);
7743 #endif
7744
7745         if (enable) {
7746                 /* enable VLAN tag insert/strip */
7747                 ctrl = E1000_READ_REG(hw, E1000_CTRL);
7748                 ctrl |= E1000_CTRL_VME;
7749                 E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
7750
7751                 /* Disable CFI check */
7752                 rctl = E1000_READ_REG(hw, E1000_RCTL);
7753                 rctl &= ~E1000_RCTL_CFIEN;
7754                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
7755         } else {
7756                 /* disable VLAN tag insert/strip */
7757                 ctrl = E1000_READ_REG(hw, E1000_CTRL);
7758                 ctrl &= ~E1000_CTRL_VME;
7759                 E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
7760         }
7761
7762 #ifndef CONFIG_IGB_VMDQ_NETDEV
7763         for (i = 0; i < adapter->vmdq_pools; i++) {
7764                 igb_set_vf_vlan_strip(adapter,
7765                                       adapter->vfs_allocated_count + i,
7766                                       enable);
7767         }
7768
7769 #else
7770         igb_set_vf_vlan_strip(adapter,
7771                               adapter->vfs_allocated_count,
7772                               enable);
7773
7774         for (i = 1; i < adapter->vmdq_pools; i++) {
7775 #ifdef HAVE_VLAN_RX_REGISTER
7776                 struct igb_vmdq_adapter *vadapter;
7777                 vadapter = netdev_priv(adapter->vmdq_netdev[i-1]);
7778                 enable = !!vadapter->vlgrp;
7779 #else
7780                 struct net_device *vnetdev;
7781                 vnetdev = adapter->vmdq_netdev[i-1];
7782                 enable = !!(vnetdev->features & NETIF_F_HW_VLAN_RX);
7783 #endif
7784                 igb_set_vf_vlan_strip(adapter,
7785                                       adapter->vfs_allocated_count + i,
7786                                       enable);
7787         }
7788
7789 #endif
7790         igb_rlpml_set(adapter);
7791 }
7792
7793 #ifdef HAVE_VLAN_PROTOCOL
7794 static int igb_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
7795 #elif defined HAVE_INT_NDO_VLAN_RX_ADD_VID
7796 static int igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
7797 #else
7798 static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
7799 #endif
7800 {
7801         struct igb_adapter *adapter = netdev_priv(netdev);
7802         int pf_id = adapter->vfs_allocated_count;
7803
7804         /* attempt to add filter to vlvf array */
7805         igb_vlvf_set(adapter, vid, TRUE, pf_id);
7806
7807         /* add the filter since PF can receive vlans w/o entry in vlvf */
7808         igb_vfta_set(adapter, vid, TRUE);
7809 #ifndef HAVE_NETDEV_VLAN_FEATURES
7810
7811         /* Copy feature flags from netdev to the vlan netdev for this vid.
7812          * This allows things like TSO to bubble down to our vlan device.
7813          * There is no need to update netdev for vlan 0 (DCB), since it
7814          * wouldn't has v_netdev.
7815          */
7816         if (adapter->vlgrp) {
7817                 struct vlan_group *vlgrp = adapter->vlgrp;
7818                 struct net_device *v_netdev = vlan_group_get_device(vlgrp, vid);
7819                 if (v_netdev) {
7820                         v_netdev->features |= netdev->features;
7821                         vlan_group_set_device(vlgrp, vid, v_netdev);
7822                 }
7823         }
7824 #endif
7825 #ifndef HAVE_VLAN_RX_REGISTER
7826
7827         set_bit(vid, adapter->active_vlans);
7828 #endif
7829 #ifdef HAVE_INT_NDO_VLAN_RX_ADD_VID
7830         return 0;
7831 #endif
7832 }
7833
7834 #ifdef HAVE_VLAN_PROTOCOL
7835 static int igb_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
7836 #elif defined HAVE_INT_NDO_VLAN_RX_ADD_VID
7837 static int igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
7838 #else
7839 static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
7840 #endif
7841 {
7842         struct igb_adapter *adapter = netdev_priv(netdev);
7843         int pf_id = adapter->vfs_allocated_count;
7844         s32 err;
7845
7846 #ifdef HAVE_VLAN_RX_REGISTER
7847         igb_irq_disable(adapter);
7848
7849         vlan_group_set_device(adapter->vlgrp, vid, NULL);
7850
7851         if (!test_bit(__IGB_DOWN, &adapter->state))
7852                 igb_irq_enable(adapter);
7853
7854 #endif /* HAVE_VLAN_RX_REGISTER */
7855         /* remove vlan from VLVF table array */
7856         err = igb_vlvf_set(adapter, vid, FALSE, pf_id);
7857
7858         /* if vid was not present in VLVF just remove it from table */
7859         if (err)
7860                 igb_vfta_set(adapter, vid, FALSE);
7861 #ifndef HAVE_VLAN_RX_REGISTER
7862
7863         clear_bit(vid, adapter->active_vlans);
7864 #endif
7865 #ifdef HAVE_INT_NDO_VLAN_RX_ADD_VID
7866         return 0;
7867 #endif
7868 }
7869
7870 static void igb_restore_vlan(struct igb_adapter *adapter)
7871 {
7872 #ifdef HAVE_VLAN_RX_REGISTER
7873         igb_vlan_mode(adapter->netdev, adapter->vlgrp);
7874
7875         if (adapter->vlgrp) {
7876                 u16 vid;
7877                 for (vid = 0; vid < VLAN_N_VID; vid++) {
7878                         if (!vlan_group_get_device(adapter->vlgrp, vid))
7879                                 continue;
7880                         igb_vlan_rx_add_vid(adapter->netdev, vid);
7881                 }
7882         }
7883 #else
7884         u16 vid;
7885
7886         igb_vlan_mode(adapter->netdev, adapter->netdev->features);
7887
7888         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
7889 #ifdef HAVE_VLAN_PROTOCOL
7890                 igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
7891 #else
7892                 igb_vlan_rx_add_vid(adapter->netdev, vid);
7893 #endif
7894
7895 #endif
7896 }
7897
7898 int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
7899 {
7900         struct pci_dev *pdev = adapter->pdev;
7901         struct e1000_mac_info *mac = &adapter->hw.mac;
7902
7903         mac->autoneg = 0;
7904
7905         /* Fiber NIC's only allow 1000 gbps Full duplex */
7906         if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes ) &&
7907                 spddplx != (SPEED_1000 + DUPLEX_FULL)) {
7908                 dev_err(pci_dev_to_dev(pdev),
7909                         "Unsupported Speed/Duplex configuration\n");
7910                 return -EINVAL;
7911         }
7912
7913         switch (spddplx) {
7914         case SPEED_10 + DUPLEX_HALF:
7915                 mac->forced_speed_duplex = ADVERTISE_10_HALF;
7916                 break;
7917         case SPEED_10 + DUPLEX_FULL:
7918                 mac->forced_speed_duplex = ADVERTISE_10_FULL;
7919                 break;
7920         case SPEED_100 + DUPLEX_HALF:
7921                 mac->forced_speed_duplex = ADVERTISE_100_HALF;
7922                 break;
7923         case SPEED_100 + DUPLEX_FULL:
7924                 mac->forced_speed_duplex = ADVERTISE_100_FULL;
7925                 break;
7926         case SPEED_1000 + DUPLEX_FULL:
7927                 mac->autoneg = 1;
7928                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
7929                 break;
7930         case SPEED_1000 + DUPLEX_HALF: /* not supported */
7931         default:
7932                 dev_err(pci_dev_to_dev(pdev), "Unsupported Speed/Duplex configuration\n");
7933                 return -EINVAL;
7934         }
7935         return 0;
7936 }
7937
7938 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
7939                           bool runtime)
7940 {
7941         struct net_device *netdev = pci_get_drvdata(pdev);
7942         struct igb_adapter *adapter = netdev_priv(netdev);
7943         struct e1000_hw *hw = &adapter->hw;
7944         u32 ctrl, rctl, status;
7945         u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
7946 #ifdef CONFIG_PM
7947         int retval = 0;
7948 #endif
7949
7950         netif_device_detach(netdev);
7951
7952         if (netif_running(netdev))
7953                 __igb_close(netdev, true);
7954
7955         igb_clear_interrupt_scheme(adapter);
7956
7957 #ifdef CONFIG_PM
7958         retval = pci_save_state(pdev);
7959         if (retval)
7960                 return retval;
7961 #endif
7962
7963         status = E1000_READ_REG(hw, E1000_STATUS);
7964         if (status & E1000_STATUS_LU)
7965                 wufc &= ~E1000_WUFC_LNKC;
7966
7967         if (wufc) {
7968                 igb_setup_rctl(adapter);
7969                 igb_set_rx_mode(netdev);
7970
7971                 /* turn on all-multi mode if wake on multicast is enabled */
7972                 if (wufc & E1000_WUFC_MC) {
7973                         rctl = E1000_READ_REG(hw, E1000_RCTL);
7974                         rctl |= E1000_RCTL_MPE;
7975                         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
7976                 }
7977
7978                 ctrl = E1000_READ_REG(hw, E1000_CTRL);
7979                 /* phy power management enable */
7980                 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
7981                 ctrl |= E1000_CTRL_ADVD3WUC;
7982                 E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
7983
7984                 /* Allow time for pending master requests to run */
7985                 e1000_disable_pcie_master(hw);
7986
7987                 E1000_WRITE_REG(hw, E1000_WUC, E1000_WUC_PME_EN);
7988                 E1000_WRITE_REG(hw, E1000_WUFC, wufc);
7989         } else {
7990                 E1000_WRITE_REG(hw, E1000_WUC, 0);
7991                 E1000_WRITE_REG(hw, E1000_WUFC, 0);
7992         }
7993
7994         *enable_wake = wufc || adapter->en_mng_pt;
7995         if (!*enable_wake)
7996                 igb_power_down_link(adapter);
7997         else
7998                 igb_power_up_link(adapter);
7999
8000         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
8001          * would have already happened in close and is redundant. */
8002         igb_release_hw_control(adapter);
8003
8004         pci_disable_device(pdev);
8005
8006         return 0;
8007 }
8008
8009 #ifdef CONFIG_PM
8010 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
8011 static int igb_suspend(struct device *dev)
8012 {
8013         int retval;
8014         bool wake;
8015         struct pci_dev *pdev = to_pci_dev(dev);
8016
8017         retval = __igb_shutdown(pdev, &wake, 0);
8018         if (retval)
8019                 return retval;
8020
8021         if (wake) {
8022                 pci_prepare_to_sleep(pdev);
8023         } else {
8024                 pci_wake_from_d3(pdev, false);
8025                 pci_set_power_state(pdev, PCI_D3hot);
8026         }
8027
8028         return 0;
8029 }
8030
8031 static int igb_resume(struct device *dev)
8032 {
8033         struct pci_dev *pdev = to_pci_dev(dev);
8034         struct net_device *netdev = pci_get_drvdata(pdev);
8035         struct igb_adapter *adapter = netdev_priv(netdev);
8036         struct e1000_hw *hw = &adapter->hw;
8037         u32 err;
8038
8039         pci_set_power_state(pdev, PCI_D0);
8040         pci_restore_state(pdev);
8041         pci_save_state(pdev);
8042
8043         err = pci_enable_device_mem(pdev);
8044         if (err) {
8045                 dev_err(pci_dev_to_dev(pdev),
8046                         "igb: Cannot enable PCI device from suspend\n");
8047                 return err;
8048         }
8049         pci_set_master(pdev);
8050
8051         pci_enable_wake(pdev, PCI_D3hot, 0);
8052         pci_enable_wake(pdev, PCI_D3cold, 0);
8053
8054 #ifdef CONFIG_PM_RUNTIME
8055         if (!rtnl_is_locked()) {
8056                 /*
8057                  * shut up ASSERT_RTNL() warning in
8058                  * netif_set_real_num_tx/rx_queues.
8059                  */
8060                 rtnl_lock();
8061                 err = igb_init_interrupt_scheme(adapter);
8062                 rtnl_unlock();
8063         } else {
8064                 err = igb_init_interrupt_scheme(adapter);
8065         }
8066         if (err) {
8067 #else
8068         if (igb_init_interrupt_scheme(adapter)) {
8069 #endif /* CONFIG_PM_RUNTIME */
8070                 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for queues\n");
8071                 return -ENOMEM;
8072         }
8073
8074         igb_reset(adapter);
8075
8076         /* let the f/w know that the h/w is now under the control of the
8077          * driver. */
8078         igb_get_hw_control(adapter);
8079
8080         E1000_WRITE_REG(hw, E1000_WUS, ~0);
8081
8082         if (netdev->flags & IFF_UP) {
8083                 err = __igb_open(netdev, true);
8084                 if (err)
8085                         return err;
8086         }
8087
8088         netif_device_attach(netdev);
8089
8090         return 0;
8091 }
8092
8093 #ifdef CONFIG_PM_RUNTIME
8094 static int igb_runtime_idle(struct device *dev)
8095 {
8096         struct pci_dev *pdev = to_pci_dev(dev);
8097         struct net_device *netdev = pci_get_drvdata(pdev);
8098         struct igb_adapter *adapter = netdev_priv(netdev);
8099
8100         if (!igb_has_link(adapter))
8101                 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
8102
8103         return -EBUSY;
8104 }
8105
8106 static int igb_runtime_suspend(struct device *dev)
8107 {
8108         struct pci_dev *pdev = to_pci_dev(dev);
8109         int retval;
8110         bool wake;
8111
8112         retval = __igb_shutdown(pdev, &wake, 1);
8113         if (retval)
8114                 return retval;
8115
8116         if (wake) {
8117                 pci_prepare_to_sleep(pdev);
8118         } else {
8119                 pci_wake_from_d3(pdev, false);
8120                 pci_set_power_state(pdev, PCI_D3hot);
8121         }
8122
8123         return 0;
8124 }
8125
8126 static int igb_runtime_resume(struct device *dev)
8127 {
8128         return igb_resume(dev);
8129 }
8130 #endif /* CONFIG_PM_RUNTIME */
8131 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
8132 #endif /* CONFIG_PM */
8133
8134 #ifdef USE_REBOOT_NOTIFIER
8135 /* only want to do this for 2.4 kernels? */
8136 static int igb_notify_reboot(struct notifier_block *nb, unsigned long event,
8137                              void *p)
8138 {
8139         struct pci_dev *pdev = NULL;
8140         bool wake;
8141
8142         switch (event) {
8143         case SYS_DOWN:
8144         case SYS_HALT:
8145         case SYS_POWER_OFF:
8146                 while ((pdev = pci_find_device(PCI_ANY_ID, PCI_ANY_ID, pdev))) {
8147                         if (pci_dev_driver(pdev) == &igb_driver) {
8148                                 __igb_shutdown(pdev, &wake, 0);
8149                                 if (event == SYS_POWER_OFF) {
8150                                         pci_wake_from_d3(pdev, wake);
8151                                         pci_set_power_state(pdev, PCI_D3hot);
8152                                 }
8153                         }
8154                 }
8155         }
8156         return NOTIFY_DONE;
8157 }
8158 #else
8159 static void igb_shutdown(struct pci_dev *pdev)
8160 {
8161         bool wake = false;
8162
8163         __igb_shutdown(pdev, &wake, 0);
8164
8165         if (system_state == SYSTEM_POWER_OFF) {
8166                 pci_wake_from_d3(pdev, wake);
8167                 pci_set_power_state(pdev, PCI_D3hot);
8168         }
8169 }
8170 #endif /* USE_REBOOT_NOTIFIER */
8171
8172 #ifdef CONFIG_NET_POLL_CONTROLLER
8173 /*
8174  * Polling 'interrupt' - used by things like netconsole to send skbs
8175  * without having to re-enable interrupts. It's not called while
8176  * the interrupt routine is executing.
8177  */
8178 static void igb_netpoll(struct net_device *netdev)
8179 {
8180         struct igb_adapter *adapter = netdev_priv(netdev);
8181         struct e1000_hw *hw = &adapter->hw;
8182         struct igb_q_vector *q_vector;
8183         int i;
8184
8185         for (i = 0; i < adapter->num_q_vectors; i++) {
8186                 q_vector = adapter->q_vector[i];
8187                 if (adapter->msix_entries)
8188                         E1000_WRITE_REG(hw, E1000_EIMC, q_vector->eims_value);
8189                 else
8190                         igb_irq_disable(adapter);
8191                 napi_schedule(&q_vector->napi);
8192         }
8193 }
8194 #endif /* CONFIG_NET_POLL_CONTROLLER */
8195
8196 #ifdef HAVE_PCI_ERS
8197 #define E1000_DEV_ID_82576_VF 0x10CA
8198 /**
8199  * igb_io_error_detected - called when PCI error is detected
8200  * @pdev: Pointer to PCI device
8201  * @state: The current pci connection state
8202  *
8203  * This function is called after a PCI bus error affecting
8204  * this device has been detected.
8205  */
8206 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
8207                                               pci_channel_state_t state)
8208 {
8209         struct net_device *netdev = pci_get_drvdata(pdev);
8210         struct igb_adapter *adapter = netdev_priv(netdev);
8211
8212 #ifdef CONFIG_PCI_IOV__UNUSED
8213         struct pci_dev *bdev, *vfdev;
8214         u32 dw0, dw1, dw2, dw3;
8215         int vf, pos;
8216         u16 req_id, pf_func;
8217
8218         if (!(adapter->flags & IGB_FLAG_DETECT_BAD_DMA))
8219                 goto skip_bad_vf_detection;
8220
8221         bdev = pdev->bus->self;
8222         while (bdev && (bdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT))
8223                 bdev = bdev->bus->self;
8224
8225         if (!bdev)
8226                 goto skip_bad_vf_detection;
8227
8228         pos = pci_find_ext_capability(bdev, PCI_EXT_CAP_ID_ERR);
8229         if (!pos)
8230                 goto skip_bad_vf_detection;
8231
8232         pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG, &dw0);
8233         pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 4, &dw1);
8234         pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 8, &dw2);
8235         pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 12, &dw3);
8236
8237         req_id = dw1 >> 16;
8238         /* On the 82576 if bit 7 of the requestor ID is set then it's a VF */
8239         if (!(req_id & 0x0080))
8240                 goto skip_bad_vf_detection;
8241
8242         pf_func = req_id & 0x01;
8243         if ((pf_func & 1) == (pdev->devfn & 1)) {
8244
8245                 vf = (req_id & 0x7F) >> 1;
8246                 dev_err(pci_dev_to_dev(pdev),
8247                         "VF %d has caused a PCIe error\n", vf);
8248                 dev_err(pci_dev_to_dev(pdev),
8249                         "TLP: dw0: %8.8x\tdw1: %8.8x\tdw2: "
8250                         "%8.8x\tdw3: %8.8x\n",
8251                         dw0, dw1, dw2, dw3);
8252
8253                 /* Find the pci device of the offending VF */
8254                 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL,
8255                                        E1000_DEV_ID_82576_VF, NULL);
8256                 while (vfdev) {
8257                         if (vfdev->devfn == (req_id & 0xFF))
8258                                 break;
8259                         vfdev = pci_get_device(PCI_VENDOR_ID_INTEL,
8260                                                E1000_DEV_ID_82576_VF, vfdev);
8261                 }
8262                 /*
8263                  * There's a slim chance the VF could have been hot plugged,
8264                  * so if it is no longer present we don't need to issue the
8265                  * VFLR.  Just clean up the AER in that case.
8266                  */
8267                 if (vfdev) {
8268                         dev_err(pci_dev_to_dev(pdev),
8269                                 "Issuing VFLR to VF %d\n", vf);
8270                         pci_write_config_dword(vfdev, 0xA8, 0x00008000);
8271                 }
8272
8273                 pci_cleanup_aer_uncorrect_error_status(pdev);
8274         }
8275
8276         /*
8277          * Even though the error may have occurred on the other port
8278          * we still need to increment the vf error reference count for
8279          * both ports because the I/O resume function will be called
8280          * for both of them.
8281          */
8282         adapter->vferr_refcount++;
8283
8284         return PCI_ERS_RESULT_RECOVERED;
8285
8286 skip_bad_vf_detection:
8287 #endif /* CONFIG_PCI_IOV */
8288
8289         netif_device_detach(netdev);
8290
8291         if (state == pci_channel_io_perm_failure)
8292                 return PCI_ERS_RESULT_DISCONNECT;
8293
8294         if (netif_running(netdev))
8295                 igb_down(adapter);
8296         pci_disable_device(pdev);
8297
8298         /* Request a slot slot reset. */
8299         return PCI_ERS_RESULT_NEED_RESET;
8300 }
8301
8302 /**
8303  * igb_io_slot_reset - called after the pci bus has been reset.
8304  * @pdev: Pointer to PCI device
8305  *
8306  * Restart the card from scratch, as if from a cold-boot. Implementation
8307  * resembles the first-half of the igb_resume routine.
8308  */
8309 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
8310 {
8311         struct net_device *netdev = pci_get_drvdata(pdev);
8312         struct igb_adapter *adapter = netdev_priv(netdev);
8313         struct e1000_hw *hw = &adapter->hw;
8314         pci_ers_result_t result;
8315
8316         if (pci_enable_device_mem(pdev)) {
8317                 dev_err(pci_dev_to_dev(pdev),
8318                         "Cannot re-enable PCI device after reset.\n");
8319                 result = PCI_ERS_RESULT_DISCONNECT;
8320         } else {
8321                 pci_set_master(pdev);
8322                 pci_restore_state(pdev);
8323                 pci_save_state(pdev);
8324
8325                 pci_enable_wake(pdev, PCI_D3hot, 0);
8326                 pci_enable_wake(pdev, PCI_D3cold, 0);
8327
8328                 schedule_work(&adapter->reset_task);
8329                 E1000_WRITE_REG(hw, E1000_WUS, ~0);
8330                 result = PCI_ERS_RESULT_RECOVERED;
8331         }
8332
8333         pci_cleanup_aer_uncorrect_error_status(pdev);
8334
8335         return result;
8336 }
8337
8338 /**
8339  * igb_io_resume - called when traffic can start flowing again.
8340  * @pdev: Pointer to PCI device
8341  *
8342  * This callback is called when the error recovery driver tells us that
8343  * its OK to resume normal operation. Implementation resembles the
8344  * second-half of the igb_resume routine.
8345  */
8346 static void igb_io_resume(struct pci_dev *pdev)
8347 {
8348         struct net_device *netdev = pci_get_drvdata(pdev);
8349         struct igb_adapter *adapter = netdev_priv(netdev);
8350
8351         if (adapter->vferr_refcount) {
8352                 dev_info(pci_dev_to_dev(pdev), "Resuming after VF err\n");
8353                 adapter->vferr_refcount--;
8354                 return;
8355         }
8356
8357         if (netif_running(netdev)) {
8358                 if (igb_up(adapter)) {
8359                         dev_err(pci_dev_to_dev(pdev), "igb_up failed after reset\n");
8360                         return;
8361                 }
8362         }
8363
8364         netif_device_attach(netdev);
8365
8366         /* let the f/w know that the h/w is now under the control of the
8367          * driver. */
8368         igb_get_hw_control(adapter);
8369 }
8370
8371 #endif /* HAVE_PCI_ERS */
8372
8373 int igb_add_mac_filter(struct igb_adapter *adapter, u8 *addr, u16 queue)
8374 {
8375         struct e1000_hw *hw = &adapter->hw;
8376         int i;
8377
8378         if (is_zero_ether_addr(addr))
8379                 return 0;
8380
8381         for (i = 0; i < hw->mac.rar_entry_count; i++) {
8382                 if (adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE)
8383                         continue;
8384                 adapter->mac_table[i].state = (IGB_MAC_STATE_MODIFIED |
8385                                                    IGB_MAC_STATE_IN_USE);
8386                 memcpy(adapter->mac_table[i].addr, addr, ETH_ALEN);
8387                 adapter->mac_table[i].queue = queue;
8388                 igb_sync_mac_table(adapter);
8389                 return 0;
8390         }
8391         return -ENOMEM;
8392 }
8393 int igb_del_mac_filter(struct igb_adapter *adapter, u8* addr, u16 queue)
8394 {
8395         /* search table for addr, if found, set to 0 and sync */
8396         int i;
8397         struct e1000_hw *hw = &adapter->hw;
8398
8399         if (is_zero_ether_addr(addr))
8400                 return 0;
8401         for (i = 0; i < hw->mac.rar_entry_count; i++) {
8402                 if (!compare_ether_addr(addr, adapter->mac_table[i].addr) &&
8403                     adapter->mac_table[i].queue == queue) {
8404                         adapter->mac_table[i].state = IGB_MAC_STATE_MODIFIED;
8405                         memset(adapter->mac_table[i].addr, 0, ETH_ALEN);
8406                         adapter->mac_table[i].queue = 0;
8407                         igb_sync_mac_table(adapter);
8408                         return 0;
8409                 }
8410         }
8411         return -ENOMEM;
8412 }
8413 static int igb_set_vf_mac(struct igb_adapter *adapter,
8414                           int vf, unsigned char *mac_addr)
8415 {
8416         igb_del_mac_filter(adapter, adapter->vf_data[vf].vf_mac_addresses, vf);
8417         memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
8418
8419         igb_add_mac_filter(adapter, mac_addr, vf);
8420
8421         return 0;
8422 }
8423
8424 #ifdef IFLA_VF_MAX
8425 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
8426 {
8427         struct igb_adapter *adapter = netdev_priv(netdev);
8428         if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
8429                 return -EINVAL;
8430         adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
8431         dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
8432         dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
8433                                       " change effective.\n");
8434         if (test_bit(__IGB_DOWN, &adapter->state)) {
8435                 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
8436                          " but the PF device is not up.\n");
8437                 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
8438                          " attempting to use the VF device.\n");
8439         }
8440         return igb_set_vf_mac(adapter, vf, mac);
8441 }
8442
8443 static int igb_link_mbps(int internal_link_speed)
8444 {
8445         switch (internal_link_speed) {
8446         case SPEED_100:
8447                 return 100;
8448         case SPEED_1000:
8449                 return 1000;
8450         default:
8451                 return 0;
8452         }
8453 }
8454
8455 static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
8456                         int link_speed)
8457 {
8458         int rf_dec, rf_int;
8459         u32 bcnrc_val;
8460
8461         if (tx_rate != 0) {
8462                 /* Calculate the rate factor values to set */
8463                 rf_int = link_speed / tx_rate;
8464                 rf_dec = (link_speed - (rf_int * tx_rate));
8465                 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
8466
8467                 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
8468                 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
8469                                 E1000_RTTBCNRC_RF_INT_MASK);
8470                 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
8471         } else {
8472                 bcnrc_val = 0;
8473         }
8474
8475         E1000_WRITE_REG(hw, E1000_RTTDQSEL, vf); /* vf X uses queue X */
8476         /*
8477          * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
8478          * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
8479          */
8480         E1000_WRITE_REG(hw, E1000_RTTBCNRM(0), 0x14);
8481         E1000_WRITE_REG(hw, E1000_RTTBCNRC, bcnrc_val);
8482 }
8483
8484 static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
8485 {
8486         int actual_link_speed, i;
8487         bool reset_rate = false;
8488
8489         /* VF TX rate limit was not set */
8490         if ((adapter->vf_rate_link_speed == 0) ||
8491                 (adapter->hw.mac.type != e1000_82576))
8492                 return;
8493
8494         actual_link_speed = igb_link_mbps(adapter->link_speed);
8495         if (actual_link_speed != adapter->vf_rate_link_speed) {
8496                 reset_rate = true;
8497                 adapter->vf_rate_link_speed = 0;
8498                 dev_info(&adapter->pdev->dev,
8499                 "Link speed has been changed. VF Transmit rate is disabled\n");
8500         }
8501
8502         for (i = 0; i < adapter->vfs_allocated_count; i++) {
8503                 if (reset_rate)
8504                         adapter->vf_data[i].tx_rate = 0;
8505
8506                 igb_set_vf_rate_limit(&adapter->hw, i,
8507                         adapter->vf_data[i].tx_rate, actual_link_speed);
8508         }
8509 }
8510
8511 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
8512 {
8513         struct igb_adapter *adapter = netdev_priv(netdev);
8514         struct e1000_hw *hw = &adapter->hw;
8515         int actual_link_speed;
8516
8517         if (hw->mac.type != e1000_82576)
8518                 return -EOPNOTSUPP;
8519
8520         actual_link_speed = igb_link_mbps(adapter->link_speed);
8521         if ((vf >= adapter->vfs_allocated_count) ||
8522                 (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) ||
8523                 (tx_rate < 0) || (tx_rate > actual_link_speed))
8524                 return -EINVAL;
8525
8526         adapter->vf_rate_link_speed = actual_link_speed;
8527         adapter->vf_data[vf].tx_rate = (u16)tx_rate;
8528         igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
8529
8530         return 0;
8531 }
8532
8533 static int igb_ndo_get_vf_config(struct net_device *netdev,
8534                                  int vf, struct ifla_vf_info *ivi)
8535 {
8536         struct igb_adapter *adapter = netdev_priv(netdev);
8537         if (vf >= adapter->vfs_allocated_count)
8538                 return -EINVAL;
8539         ivi->vf = vf;
8540         memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
8541         ivi->tx_rate = adapter->vf_data[vf].tx_rate;
8542         ivi->vlan = adapter->vf_data[vf].pf_vlan;
8543         ivi->qos = adapter->vf_data[vf].pf_qos;
8544         return 0;
8545 }
8546 #endif
8547 static void igb_vmm_control(struct igb_adapter *adapter)
8548 {
8549         struct e1000_hw *hw = &adapter->hw;
8550         u32 reg;
8551
8552         switch (hw->mac.type) {
8553         case e1000_82575:
8554         default:
8555                 /* replication is not supported for 82575 */
8556                 return;
8557         case e1000_82576:
8558                 /* notify HW that the MAC is adding vlan tags */
8559                 reg = E1000_READ_REG(hw, E1000_DTXCTL);
8560                 reg |= (E1000_DTXCTL_VLAN_ADDED |
8561                         E1000_DTXCTL_SPOOF_INT);
8562                 E1000_WRITE_REG(hw, E1000_DTXCTL, reg);
8563         case e1000_82580:
8564                 /* enable replication vlan tag stripping */
8565                 reg = E1000_READ_REG(hw, E1000_RPLOLR);
8566                 reg |= E1000_RPLOLR_STRVLAN;
8567                 E1000_WRITE_REG(hw, E1000_RPLOLR, reg);
8568         case e1000_i350:
8569                 /* none of the above registers are supported by i350 */
8570                 break;
8571         }
8572
8573         /* Enable Malicious Driver Detection */
8574         if ((hw->mac.type == e1000_i350) && (adapter->vfs_allocated_count) &&
8575             (adapter->mdd))
8576                 igb_enable_mdd(adapter);
8577
8578         /* enable replication and loopback support */
8579         e1000_vmdq_set_loopback_pf(hw, adapter->vfs_allocated_count ||
8580                                    adapter->vmdq_pools);
8581
8582         e1000_vmdq_set_anti_spoofing_pf(hw, adapter->vfs_allocated_count ||
8583                                         adapter->vmdq_pools,
8584                                         adapter->vfs_allocated_count);
8585         e1000_vmdq_set_replication_pf(hw, adapter->vfs_allocated_count ||
8586                                       adapter->vmdq_pools);
8587 }
8588
8589 static void igb_init_fw(struct igb_adapter *adapter)
8590 {
8591         struct e1000_fw_drv_info fw_cmd;
8592         struct e1000_hw *hw = &adapter->hw;
8593         int i;
8594         u16 mask;
8595
8596         mask = E1000_SWFW_PHY0_SM;
8597
8598         if (!hw->mac.ops.acquire_swfw_sync(hw, mask)) {
8599                 for (i = 0; i <= FW_MAX_RETRIES; i++) {
8600                         E1000_WRITE_REG(hw, E1000_FWSTS, E1000_FWSTS_FWRI);
8601                         fw_cmd.hdr.cmd = FW_CMD_DRV_INFO;
8602                         fw_cmd.hdr.buf_len = FW_CMD_DRV_INFO_LEN;
8603                         fw_cmd.hdr.cmd_or_resp.cmd_resv = FW_CMD_RESERVED;
8604                         fw_cmd.port_num = hw->bus.func;
8605                         fw_cmd.drv_version = FW_FAMILY_DRV_VER;
8606                         fw_cmd.hdr.checksum = 0;
8607                         fw_cmd.hdr.checksum = e1000_calculate_checksum((u8 *)&fw_cmd,
8608                                                                    (FW_HDR_LEN +
8609                                                                     fw_cmd.hdr.buf_len));
8610                          e1000_host_interface_command(hw, (u8*)&fw_cmd,
8611                                                      sizeof(fw_cmd));
8612                         if (fw_cmd.hdr.cmd_or_resp.ret_status == FW_STATUS_SUCCESS)
8613                                 break;
8614                 }
8615         } else
8616                 dev_warn(pci_dev_to_dev(adapter->pdev),
8617                          "Unable to get semaphore, firmware init failed.\n");
8618         hw->mac.ops.release_swfw_sync(hw, mask);
8619 }
8620
8621 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
8622 {
8623         struct e1000_hw *hw = &adapter->hw;
8624         u32 dmac_thr;
8625         u16 hwm;
8626
8627         if (hw->mac.type > e1000_82580) {
8628                 if (adapter->dmac != IGB_DMAC_DISABLE) {
8629                         u32 reg;
8630
8631                         /* force threshold to 0.  */
8632                         E1000_WRITE_REG(hw, E1000_DMCTXTH, 0);
8633
8634                         /*
8635                          * DMA Coalescing high water mark needs to be greater
8636                          * than the Rx threshold. Set hwm to PBA - max frame
8637                          * size in 16B units, capping it at PBA - 6KB.
8638                          */
8639                         hwm = 64 * pba - adapter->max_frame_size / 16;
8640                         if (hwm < 64 * (pba - 6))
8641                                 hwm = 64 * (pba - 6);
8642                         reg = E1000_READ_REG(hw, E1000_FCRTC);
8643                         reg &= ~E1000_FCRTC_RTH_COAL_MASK;
8644                         reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
8645                                 & E1000_FCRTC_RTH_COAL_MASK);
8646                         E1000_WRITE_REG(hw, E1000_FCRTC, reg);
8647
8648                         /*
8649                          * Set the DMA Coalescing Rx threshold to PBA - 2 * max
8650                          * frame size, capping it at PBA - 10KB.
8651                          */
8652                         dmac_thr = pba - adapter->max_frame_size / 512;
8653                         if (dmac_thr < pba - 10)
8654                                 dmac_thr = pba - 10;
8655                         reg = E1000_READ_REG(hw, E1000_DMACR);
8656                         reg &= ~E1000_DMACR_DMACTHR_MASK;
8657                         reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
8658                                 & E1000_DMACR_DMACTHR_MASK);
8659
8660                         /* transition to L0x or L1 if available..*/
8661                         reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
8662
8663                         /* watchdog timer= msec values in 32usec intervals */
8664                         reg |= ((adapter->dmac) >> 5);
8665                         E1000_WRITE_REG(hw, E1000_DMACR, reg);
8666
8667                         /* no lower threshold to disable coalescing(smart fifb)-UTRESH=0*/
8668                         E1000_WRITE_REG(hw, E1000_DMCRTRH, 0);
8669
8670                         /*
8671                          * This sets the time to wait before requesting transition to
8672                          * low power state to number of usecs needed to receive 1 512
8673                          * byte frame at gigabit line rate
8674                          */
8675                         reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
8676
8677                         E1000_WRITE_REG(hw, E1000_DMCTLX, reg);
8678
8679                         /* free space in tx packet buffer to wake from DMA coal */
8680                         E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
8681                                 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
8682
8683                         /* make low power state decision controlled by DMA coal */
8684                         reg = E1000_READ_REG(hw, E1000_PCIEMISC);
8685                         reg &= ~E1000_PCIEMISC_LX_DECISION;
8686                         E1000_WRITE_REG(hw, E1000_PCIEMISC, reg);
8687                 } /* endif adapter->dmac is not disabled */
8688         } else if (hw->mac.type == e1000_82580) {
8689                 u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
8690                 E1000_WRITE_REG(hw, E1000_PCIEMISC,
8691                                 reg & ~E1000_PCIEMISC_LX_DECISION);
8692                 E1000_WRITE_REG(hw, E1000_DMACR, 0);
8693         }
8694 }
8695
8696 /* igb_main.c */
8697
8698
8699 /**
8700  * igb_probe - Device Initialization Routine
8701  * @pdev: PCI device information struct
8702  * @ent: entry in igb_pci_tbl
8703  *
8704  * Returns 0 on success, negative on failure
8705  *
8706  * igb_probe initializes an adapter identified by a pci_dev structure.
8707  * The OS initialization, configuring of the adapter private structure,
8708  * and a hardware reset occur.
8709  **/
8710 int igb_kni_probe(struct pci_dev *pdev,
8711                                struct net_device **lad_dev)
8712 {
8713         struct net_device *netdev;
8714         struct igb_adapter *adapter;
8715         struct e1000_hw *hw;
8716         u16 eeprom_data = 0;
8717         u8 pba_str[E1000_PBANUM_LENGTH];
8718         s32 ret_val;
8719         static int global_quad_port_a; /* global quad port a indication */
8720         int i, err, pci_using_dac = 0;
8721         static int cards_found;
8722
8723         err = pci_enable_device_mem(pdev);
8724         if (err)
8725                 return err;
8726
8727 #ifdef NO_KNI
8728         pci_using_dac = 0;
8729         err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
8730         if (!err) {
8731                 err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
8732                 if (!err)
8733                         pci_using_dac = 1;
8734         } else {
8735                 err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
8736                 if (err) {
8737                         err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
8738                         if (err) {
8739                                 IGB_ERR("No usable DMA configuration, "
8740                                         "aborting\n");
8741                                 goto err_dma;
8742                         }
8743                 }
8744         }
8745
8746 #ifndef HAVE_ASPM_QUIRKS
8747         /* 82575 requires that the pci-e link partner disable the L0s state */
8748         switch (pdev->device) {
8749         case E1000_DEV_ID_82575EB_COPPER:
8750         case E1000_DEV_ID_82575EB_FIBER_SERDES:
8751         case E1000_DEV_ID_82575GB_QUAD_COPPER:
8752                 pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S);
8753         default:
8754                 break;
8755         }
8756
8757 #endif /* HAVE_ASPM_QUIRKS */
8758         err = pci_request_selected_regions(pdev,
8759                                            pci_select_bars(pdev,
8760                                                            IORESOURCE_MEM),
8761                                            igb_driver_name);
8762         if (err)
8763                 goto err_pci_reg;
8764
8765         pci_enable_pcie_error_reporting(pdev);
8766
8767         pci_set_master(pdev);
8768 #endif /* NO_KNI */
8769         err = -ENOMEM;
8770 #ifdef HAVE_TX_MQ
8771         netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
8772                                    IGB_MAX_TX_QUEUES);
8773 #else
8774         netdev = alloc_etherdev(sizeof(struct igb_adapter));
8775 #endif /* HAVE_TX_MQ */
8776         if (!netdev)
8777                 goto err_alloc_etherdev;
8778
8779
8780         SET_MODULE_OWNER(netdev);
8781         SET_NETDEV_DEV(netdev, &pdev->dev);
8782
8783         //pci_set_drvdata(pdev, netdev);
8784
8785         adapter = netdev_priv(netdev);
8786         adapter->netdev = netdev;
8787         adapter->pdev = pdev;
8788         hw = &adapter->hw;
8789         hw->back = adapter;
8790         adapter->port_num = hw->bus.func;
8791         adapter->msg_enable = (1 << debug) - 1;
8792
8793 #ifdef HAVE_PCI_ERS
8794         err = pci_save_state(pdev);
8795         if (err)
8796                 goto err_ioremap;
8797 #endif
8798         err = -EIO;
8799         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
8800                               pci_resource_len(pdev, 0));
8801         if (!hw->hw_addr)
8802                 goto err_ioremap;
8803
8804 #ifdef HAVE_NET_DEVICE_OPS
8805         netdev->netdev_ops = &igb_netdev_ops;
8806 #else /* HAVE_NET_DEVICE_OPS */
8807         netdev->open = &igb_open;
8808         netdev->stop = &igb_close;
8809         netdev->get_stats = &igb_get_stats;
8810 #ifdef HAVE_SET_RX_MODE
8811         netdev->set_rx_mode = &igb_set_rx_mode;
8812 #endif
8813         netdev->set_multicast_list = &igb_set_rx_mode;
8814         netdev->set_mac_address = &igb_set_mac;
8815         netdev->change_mtu = &igb_change_mtu;
8816         netdev->do_ioctl = &igb_ioctl;
8817 #ifdef HAVE_TX_TIMEOUT
8818         netdev->tx_timeout = &igb_tx_timeout;
8819 #endif
8820         netdev->vlan_rx_register = igb_vlan_mode;
8821         netdev->vlan_rx_add_vid = igb_vlan_rx_add_vid;
8822         netdev->vlan_rx_kill_vid = igb_vlan_rx_kill_vid;
8823 #ifdef CONFIG_NET_POLL_CONTROLLER
8824         netdev->poll_controller = igb_netpoll;
8825 #endif
8826         netdev->hard_start_xmit = &igb_xmit_frame;
8827 #endif /* HAVE_NET_DEVICE_OPS */
8828         igb_set_ethtool_ops(netdev);
8829 #ifdef HAVE_TX_TIMEOUT
8830         netdev->watchdog_timeo = 5 * HZ;
8831 #endif
8832
8833         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
8834
8835         adapter->bd_number = cards_found;
8836
8837         /* setup the private structure */
8838         err = igb_sw_init(adapter);
8839         if (err)
8840                 goto err_sw_init;
8841
8842         e1000_get_bus_info(hw);
8843
8844         hw->phy.autoneg_wait_to_complete = FALSE;
8845         hw->mac.adaptive_ifs = FALSE;
8846
8847         /* Copper options */
8848         if (hw->phy.media_type == e1000_media_type_copper) {
8849 #ifdef ETH_TP_MDI_X
8850                 hw->phy.mdix = ETH_TP_MDI_INVALID;
8851 #else
8852                 hw->phy.mdix = AUTO_ALL_MODES;
8853 #endif /* ETH_TP_MDI_X */
8854                 hw->phy.disable_polarity_correction = FALSE;
8855                 hw->phy.ms_type = e1000_ms_hw_default;
8856         }
8857
8858         if (e1000_check_reset_block(hw))
8859                 dev_info(pci_dev_to_dev(pdev),
8860                         "PHY reset is blocked due to SOL/IDER session.\n");
8861
8862         /*
8863          * features is initialized to 0 in allocation, it might have bits
8864          * set by igb_sw_init so we should use an or instead of an
8865          * assignment.
8866          */
8867         netdev->features |= NETIF_F_SG |
8868                             NETIF_F_IP_CSUM |
8869 #ifdef NETIF_F_IPV6_CSUM
8870                             NETIF_F_IPV6_CSUM |
8871 #endif
8872 #ifdef NETIF_F_TSO
8873                             NETIF_F_TSO |
8874 #ifdef NETIF_F_TSO6
8875                             NETIF_F_TSO6 |
8876 #endif
8877 #endif /* NETIF_F_TSO */
8878 #ifdef NETIF_F_RXHASH
8879                             NETIF_F_RXHASH |
8880 #endif
8881 #ifdef HAVE_NDO_SET_FEATURES
8882                             NETIF_F_RXCSUM |
8883 #endif
8884                             NETIF_F_HW_VLAN_RX |
8885                             NETIF_F_HW_VLAN_TX;
8886
8887 #ifdef HAVE_NDO_SET_FEATURES
8888         /* copy netdev features into list of user selectable features */
8889         netdev->hw_features |= netdev->features;
8890 #ifndef IGB_NO_LRO
8891
8892         /* give us the option of enabling LRO later */
8893         netdev->hw_features |= NETIF_F_LRO;
8894 #endif
8895 #else
8896 #ifdef NETIF_F_GRO
8897
8898         /* this is only needed on kernels prior to 2.6.39 */
8899         netdev->features |= NETIF_F_GRO;
8900 #endif
8901 #endif
8902
8903         /* set this bit last since it cannot be part of hw_features */
8904         netdev->features |= NETIF_F_HW_VLAN_FILTER;
8905
8906 #ifdef HAVE_NETDEV_VLAN_FEATURES
8907         netdev->vlan_features |= NETIF_F_TSO |
8908                                  NETIF_F_TSO6 |
8909                                  NETIF_F_IP_CSUM |
8910                                  NETIF_F_IPV6_CSUM |
8911                                  NETIF_F_SG;
8912
8913 #endif
8914         if (pci_using_dac)
8915                 netdev->features |= NETIF_F_HIGHDMA;
8916
8917         if (hw->mac.type >= e1000_82576)
8918                 netdev->features |= NETIF_F_SCTP_CSUM;
8919
8920 #ifdef NO_KNI
8921         adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
8922
8923         /* before reading the NVM, reset the controller to put the device in a
8924          * known good starting state */
8925         e1000_reset_hw(hw);
8926 #endif
8927
8928         /* make sure the NVM is good */
8929         if (e1000_validate_nvm_checksum(hw) < 0) {
8930                 dev_err(pci_dev_to_dev(pdev), "The NVM Checksum Is Not"
8931                         " Valid\n");
8932                 err = -EIO;
8933                 goto err_eeprom;
8934         }
8935
8936         /* copy the MAC address out of the NVM */
8937         if (e1000_read_mac_addr(hw))
8938                 dev_err(pci_dev_to_dev(pdev), "NVM Read Error\n");
8939         memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
8940 #ifdef ETHTOOL_GPERMADDR
8941         memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
8942
8943         if (!is_valid_ether_addr(netdev->perm_addr)) {
8944 #else
8945         if (!is_valid_ether_addr(netdev->dev_addr)) {
8946 #endif
8947                 dev_err(pci_dev_to_dev(pdev), "Invalid MAC Address\n");
8948                 err = -EIO;
8949                 goto err_eeprom;
8950         }
8951
8952         memcpy(&adapter->mac_table[0].addr, hw->mac.addr, netdev->addr_len);
8953         adapter->mac_table[0].queue = adapter->vfs_allocated_count;
8954         adapter->mac_table[0].state = (IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE);
8955         igb_rar_set(adapter, 0);
8956
8957         /* get firmware version for ethtool -i */
8958         e1000_read_nvm(&adapter->hw, 5, 1, &adapter->fw_version);
8959 #ifdef NO_KNI
8960         setup_timer(&adapter->watchdog_timer, &igb_watchdog,
8961                     (unsigned long) adapter);
8962         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
8963                 setup_timer(&adapter->dma_err_timer, &igb_dma_err_timer,
8964                             (unsigned long) adapter);
8965         setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
8966                     (unsigned long) adapter);
8967
8968         INIT_WORK(&adapter->reset_task, igb_reset_task);
8969         INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
8970         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
8971                 INIT_WORK(&adapter->dma_err_task, igb_dma_err_task);
8972 #endif
8973
8974         /* Initialize link properties that are user-changeable */
8975         adapter->fc_autoneg = true;
8976         hw->mac.autoneg = true;
8977         hw->phy.autoneg_advertised = 0x2f;
8978
8979         hw->fc.requested_mode = e1000_fc_default;
8980         hw->fc.current_mode = e1000_fc_default;
8981
8982         e1000_validate_mdi_setting(hw);
8983
8984         /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
8985          * enable the ACPI Magic Packet filter
8986          */
8987
8988         if (hw->bus.func == 0)
8989                 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
8990         else if (hw->mac.type >= e1000_82580)
8991                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
8992                                  NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
8993                                  &eeprom_data);
8994         else if (hw->bus.func == 1)
8995                 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
8996
8997         if (eeprom_data & IGB_EEPROM_APME)
8998                 adapter->eeprom_wol |= E1000_WUFC_MAG;
8999
9000         /* now that we have the eeprom settings, apply the special cases where
9001          * the eeprom may be wrong or the board simply won't support wake on
9002          * lan on a particular port */
9003         switch (pdev->device) {
9004         case E1000_DEV_ID_82575GB_QUAD_COPPER:
9005                 adapter->eeprom_wol = 0;
9006                 break;
9007         case E1000_DEV_ID_82575EB_FIBER_SERDES:
9008         case E1000_DEV_ID_82576_FIBER:
9009         case E1000_DEV_ID_82576_SERDES:
9010                 /* Wake events only supported on port A for dual fiber
9011                  * regardless of eeprom setting */
9012                 if (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_FUNC_1)
9013                         adapter->eeprom_wol = 0;
9014                 break;
9015         case E1000_DEV_ID_82576_QUAD_COPPER:
9016         case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
9017                 /* if quad port adapter, disable WoL on all but port A */
9018                 if (global_quad_port_a != 0)
9019                         adapter->eeprom_wol = 0;
9020                 else
9021                         adapter->flags |= IGB_FLAG_QUAD_PORT_A;
9022                 /* Reset for multiple quad port adapters */
9023                 if (++global_quad_port_a == 4)
9024                         global_quad_port_a = 0;
9025                 break;
9026         }
9027
9028         /* initialize the wol settings based on the eeprom settings */
9029         adapter->wol = adapter->eeprom_wol;
9030 #ifdef NO_KNI
9031         device_set_wakeup_enable(pci_dev_to_dev(adapter->pdev), adapter->wol);
9032
9033         /* reset the hardware with the new settings */
9034         igb_reset(adapter);
9035
9036         /* let the f/w know that the h/w is now under the control of the
9037          * driver. */
9038         igb_get_hw_control(adapter);
9039
9040         strncpy(netdev->name, "eth%d", IFNAMSIZ);
9041         err = register_netdev(netdev);
9042         if (err)
9043                 goto err_register;
9044
9045 #ifdef CONFIG_IGB_VMDQ_NETDEV
9046         err = igb_init_vmdq_netdevs(adapter);
9047         if (err)
9048                 goto err_register;
9049 #endif
9050         /* carrier off reporting is important to ethtool even BEFORE open */
9051         netif_carrier_off(netdev);
9052
9053 #ifdef IGB_DCA
9054         if (dca_add_requester(&pdev->dev) == E1000_SUCCESS) {
9055                 adapter->flags |= IGB_FLAG_DCA_ENABLED;
9056                 dev_info(pci_dev_to_dev(pdev), "DCA enabled\n");
9057                 igb_setup_dca(adapter);
9058         }
9059
9060 #endif
9061 #ifdef HAVE_HW_TIME_STAMP
9062         /* do hw tstamp init after resetting */
9063         igb_init_hw_timer(adapter);
9064
9065 #endif
9066
9067 #endif /* NO_KNI */
9068         dev_info(pci_dev_to_dev(pdev), "Intel(R) Gigabit Ethernet Network Connection\n");
9069         /* print bus type/speed/width info */
9070         dev_info(pci_dev_to_dev(pdev), "%s: (PCIe:%s:%s) ",
9071                  netdev->name,
9072                  ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5GT/s" :
9073                   (hw->bus.speed == e1000_bus_speed_5000) ? "5.0GT/s" :
9074                                                             "unknown"),
9075                  ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4\n" :
9076                   (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2\n" :
9077                   (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1\n" :
9078                    "unknown"));
9079         dev_info(pci_dev_to_dev(pdev), "%s: MAC: ", netdev->name);
9080         for (i = 0; i < 6; i++)
9081                 printk("%2.2x%c", netdev->dev_addr[i], i == 5 ? '\n' : ':');
9082
9083         ret_val = e1000_read_pba_string(hw, pba_str, E1000_PBANUM_LENGTH);
9084         if (ret_val)
9085                 strncpy(pba_str, "Unknown", sizeof(pba_str) - 1);
9086         dev_info(pci_dev_to_dev(pdev), "%s: PBA No: %s\n", netdev->name,
9087                  pba_str);
9088
9089         /* Initialize the thermal sensor on i350 devices. */
9090         if (hw->mac.type == e1000_i350 && hw->bus.func == 0) {
9091                 u16 ets_word;
9092
9093                 /*
9094                  * Read the NVM to determine if this i350 device supports an
9095                  * external thermal sensor.
9096                  */
9097                 e1000_read_nvm(hw, NVM_ETS_CFG, 1, &ets_word);
9098                 if (ets_word != 0x0000 && ets_word != 0xFFFF)
9099                         adapter->ets = true;
9100                 else
9101                         adapter->ets = false;
9102 #ifdef NO_KNI
9103 #ifdef IGB_SYSFS
9104                 igb_sysfs_init(adapter);
9105 #else
9106 #ifdef IGB_PROCFS
9107                 igb_procfs_init(adapter);
9108 #endif /* IGB_PROCFS */
9109 #endif /* IGB_SYSFS */
9110 #endif /* NO_KNI */
9111         } else {
9112                 adapter->ets = false;
9113         }
9114
9115         switch (hw->mac.type) {
9116         case e1000_i350:
9117                 /* Enable EEE for internal copper PHY devices */
9118                 if (hw->phy.media_type == e1000_media_type_copper)
9119                         e1000_set_eee_i350(hw);
9120
9121                 /* send driver version info to firmware */
9122                 igb_init_fw(adapter);
9123                 break;
9124         default:
9125                 break;
9126         }
9127 #ifndef IGB_NO_LRO
9128         if (netdev->features & NETIF_F_LRO)
9129                 dev_info(pci_dev_to_dev(pdev), "Internal LRO is enabled \n");
9130         else
9131                 dev_info(pci_dev_to_dev(pdev), "LRO is disabled \n");
9132 #endif
9133         dev_info(pci_dev_to_dev(pdev),
9134                  "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
9135                  adapter->msix_entries ? "MSI-X" :
9136                  (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9137                  adapter->num_rx_queues, adapter->num_tx_queues);
9138
9139         cards_found++;
9140         *lad_dev = netdev;
9141
9142         pm_runtime_put_noidle(&pdev->dev);
9143         return 0;
9144
9145 //err_register:
9146         //igb_release_hw_control(adapter);
9147 err_eeprom:
9148         //if (!e1000_check_reset_block(hw))
9149         //      e1000_phy_hw_reset(hw);
9150
9151         if (hw->flash_address)
9152                 iounmap(hw->flash_address);
9153 err_sw_init:
9154         //igb_clear_interrupt_scheme(adapter);
9155         //igb_reset_sriov_capability(adapter);
9156         iounmap(hw->hw_addr);
9157 err_ioremap:
9158         free_netdev(netdev);
9159 err_alloc_etherdev:
9160         //pci_release_selected_regions(pdev,
9161         //                             pci_select_bars(pdev, IORESOURCE_MEM));
9162 //err_pci_reg:
9163 //err_dma:
9164         pci_disable_device(pdev);
9165         return err;
9166 }
9167
9168
9169 void igb_kni_remove(struct pci_dev *pdev)
9170 {
9171         pci_disable_device(pdev);
9172 }
9173