kernel/linux/kni/ethtool/igb/igb_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*******************************************************************************
   3
   4   Intel(R) Gigabit Ethernet Linux driver
   5   Copyright(c) 2007-2013 Intel Corporation.
   6
   7   Contact Information:
   8   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
   9   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  10
  11 *******************************************************************************/
  12
  13 #include <linux/module.h>
  14 #include <linux/types.h>
  15 #include <linux/init.h>
  16 #include <linux/vmalloc.h>
  17 #include <linux/pagemap.h>
  18 #include <linux/netdevice.h>
  19 #include <linux/tcp.h>
  20 #ifdef NETIF_F_TSO
  21 #include <net/checksum.h>
  22 #ifdef NETIF_F_TSO6
  23 #include <linux/ipv6.h>
  24 #include <net/ip6_checksum.h>
  25 #endif
  26 #endif
  27 #ifdef SIOCGMIIPHY
  28 #include <linux/mii.h>
  29 #endif
  30 #ifdef SIOCETHTOOL
  31 #include <linux/ethtool.h>
  32 #endif
  33 #include <linux/if_vlan.h>
  34 #ifdef CONFIG_PM_RUNTIME
  35 #include <linux/pm_runtime.h>
  36 #endif /* CONFIG_PM_RUNTIME */
  37
  38 #include <linux/if_bridge.h>
  39 #include "igb.h"
  40 #include "igb_vmdq.h"
  41
  42 #include <linux/uio_driver.h>
  43
  44 #if defined(DEBUG) || defined (DEBUG_DUMP) || defined (DEBUG_ICR) || defined(DEBUG_ITR)
  45 #define DRV_DEBUG "_debug"
  46 #else
  47 #define DRV_DEBUG
  48 #endif
  49 #define DRV_HW_PERF
  50 #define VERSION_SUFFIX
  51
  52 #define MAJ 5
  53 #define MIN 0
  54 #define BUILD 6
  55 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." __stringify(BUILD) VERSION_SUFFIX DRV_DEBUG DRV_HW_PERF
  56
  57 char igb_driver_name[] = "igb";
  58 char igb_driver_version[] = DRV_VERSION;
  59 static const char igb_driver_string[] =
  60                                 "Intel(R) Gigabit Ethernet Network Driver";
  61 static const char igb_copyright[] =
  62                                 "Copyright (c) 2007-2013 Intel Corporation.";
  63
  64 const struct pci_device_id igb_pci_tbl[] = {
  65         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) },
  66         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) },
  67         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) },
  68         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER) },
  69         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER) },
  70         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES) },
  71         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII) },
  72         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER_FLASHLESS) },
  73         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES_FLASHLESS) },
  74         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER) },
  75         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER) },
  76         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER) },
  77         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES) },
  78         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII) },
  79         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER) },
  80         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER) },
  81         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER) },
  82         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES) },
  83         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII) },
  84         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL) },
  85         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII) },
  86         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES) },
  87         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE) },
  88         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP) },
  89         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576) },
  90         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS) },
  91         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES) },
  92         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER) },
  93         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES) },
  94         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD) },
  95         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2) },
  96         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER) },
  97         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER) },
  98         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES) },
  99         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER) },
 100         /* required last entry */
 101         {0, }
 102 };
 103
 104 //MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
 105 static void igb_set_sriov_capability(struct igb_adapter *adapter) __attribute__((__unused__));
 106 void igb_reset(struct igb_adapter *);
 107 static int igb_setup_all_tx_resources(struct igb_adapter *);
 108 static int igb_setup_all_rx_resources(struct igb_adapter *);
 109 static void igb_free_all_tx_resources(struct igb_adapter *);
 110 static void igb_free_all_rx_resources(struct igb_adapter *);
 111 static void igb_setup_mrqc(struct igb_adapter *);
 112 void igb_update_stats(struct igb_adapter *);
 113 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
 114 static void __devexit igb_remove(struct pci_dev *pdev);
 115 static int igb_sw_init(struct igb_adapter *);
 116 static int igb_open(struct net_device *);
 117 static int igb_close(struct net_device *);
 118 static void igb_configure(struct igb_adapter *);
 119 static void igb_configure_tx(struct igb_adapter *);
 120 static void igb_configure_rx(struct igb_adapter *);
 121 static void igb_clean_all_tx_rings(struct igb_adapter *);
 122 static void igb_clean_all_rx_rings(struct igb_adapter *);
 123 static void igb_clean_tx_ring(struct igb_ring *);
 124 static void igb_set_rx_mode(struct net_device *);
 125 #ifdef HAVE_TIMER_SETUP
 126 static void igb_update_phy_info(struct timer_list *);
 127 static void igb_watchdog(struct timer_list *);
 128 #else
 129 static void igb_update_phy_info(unsigned long);
 130 static void igb_watchdog(unsigned long);
 131 #endif
 132 static void igb_watchdog_task(struct work_struct *);
 133 static void igb_dma_err_task(struct work_struct *);
 134 #ifdef HAVE_TIMER_SETUP
 135 static void igb_dma_err_timer(struct timer_list *);
 136 #else
 137 static void igb_dma_err_timer(unsigned long data);
 138 #endif
 139 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
 140 static struct net_device_stats *igb_get_stats(struct net_device *);
 141 static int igb_change_mtu(struct net_device *, int);
 142 void igb_full_sync_mac_table(struct igb_adapter *adapter);
 143 static int igb_set_mac(struct net_device *, void *);
 144 static void igb_set_uta(struct igb_adapter *adapter);
 145 static irqreturn_t igb_intr(int irq, void *);
 146 static irqreturn_t igb_intr_msi(int irq, void *);
 147 static irqreturn_t igb_msix_other(int irq, void *);
 148 static irqreturn_t igb_msix_ring(int irq, void *);
 149 #ifdef IGB_DCA
 150 static void igb_update_dca(struct igb_q_vector *);
 151 static void igb_setup_dca(struct igb_adapter *);
 152 #endif /* IGB_DCA */
 153 static int igb_poll(struct napi_struct *, int);
 154 static bool igb_clean_tx_irq(struct igb_q_vector *);
 155 static bool igb_clean_rx_irq(struct igb_q_vector *, int);
 156 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
 157 static void igb_tx_timeout(struct net_device *);
 158 static void igb_reset_task(struct work_struct *);
 159 #ifdef HAVE_VLAN_RX_REGISTER
 160 static void igb_vlan_mode(struct net_device *, struct vlan_group *);
 161 #endif
 162 #ifdef HAVE_VLAN_PROTOCOL
 163 static int igb_vlan_rx_add_vid(struct net_device *,
 164                                __be16 proto, u16);
 165 static int igb_vlan_rx_kill_vid(struct net_device *,
 166                                 __be16 proto, u16);
 167 #elif defined HAVE_INT_NDO_VLAN_RX_ADD_VID
 168 #ifdef NETIF_F_HW_VLAN_CTAG_RX
 169 static int igb_vlan_rx_add_vid(struct net_device *,
 170                                __always_unused __be16 proto, u16);
 171 static int igb_vlan_rx_kill_vid(struct net_device *,
 172                                 __always_unused __be16 proto, u16);
 173 #else
 174 static int igb_vlan_rx_add_vid(struct net_device *, u16);
 175 static int igb_vlan_rx_kill_vid(struct net_device *, u16);
 176 #endif
 177 #else
 178 static void igb_vlan_rx_add_vid(struct net_device *, u16);
 179 static void igb_vlan_rx_kill_vid(struct net_device *, u16);
 180 #endif
 181 static void igb_restore_vlan(struct igb_adapter *);
 182 void igb_rar_set(struct igb_adapter *adapter, u32 index);
 183 static void igb_ping_all_vfs(struct igb_adapter *);
 184 static void igb_msg_task(struct igb_adapter *);
 185 static void igb_vmm_control(struct igb_adapter *);
 186 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
 187 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
 188 static void igb_process_mdd_event(struct igb_adapter *);
 189 #ifdef IFLA_VF_MAX
 190 static int igb_ndo_set_vf_mac( struct net_device *netdev, int vf, u8 *mac);
 191 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
 192 #ifdef HAVE_VF_VLAN_PROTO
 193                                 int vf, u16 vlan, u8 qos, __be16 vlan_proto);
 194 #else
 195                                 int vf, u16 vlan, u8 qos);
 196 #endif
 197 #ifdef HAVE_VF_SPOOFCHK_CONFIGURE
 198 static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
 199                                 bool setting);
 200 #endif
 201 #ifdef HAVE_VF_MIN_MAX_TXRATE
 202 static int igb_ndo_set_vf_bw(struct net_device *, int, int, int);
 203 #else /* HAVE_VF_MIN_MAX_TXRATE */
 204 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
 205 #endif /* HAVE_VF_MIN_MAX_TXRATE */
 206 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
 207                                  struct ifla_vf_info *ivi);
 208 static void igb_check_vf_rate_limit(struct igb_adapter *);
 209 #endif
 210 static int igb_vf_configure(struct igb_adapter *adapter, int vf);
 211 #ifdef CONFIG_PM
 212 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
 213 static int igb_suspend(struct device *dev);
 214 static int igb_resume(struct device *dev);
 215 #ifdef CONFIG_PM_RUNTIME
 216 static int igb_runtime_suspend(struct device *dev);
 217 static int igb_runtime_resume(struct device *dev);
 218 static int igb_runtime_idle(struct device *dev);
 219 #endif /* CONFIG_PM_RUNTIME */
 220 static const struct dev_pm_ops igb_pm_ops = {
 221 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,34)
 222         .suspend = igb_suspend,
 223         .resume = igb_resume,
 224         .freeze = igb_suspend,
 225         .thaw = igb_resume,
 226         .poweroff = igb_suspend,
 227         .restore = igb_resume,
 228 #ifdef CONFIG_PM_RUNTIME
 229         .runtime_suspend = igb_runtime_suspend,
 230         .runtime_resume = igb_runtime_resume,
 231         .runtime_idle = igb_runtime_idle,
 232 #endif
 233 #else /* Linux >= 2.6.34 */
 234         SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
 235 #ifdef CONFIG_PM_RUNTIME
 236         SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
 237                         igb_runtime_idle)
 238 #endif /* CONFIG_PM_RUNTIME */
 239 #endif /* Linux version */
 240 };
 241 #else
 242 static int igb_suspend(struct pci_dev *pdev, pm_message_t state);
 243 static int igb_resume(struct pci_dev *pdev);
 244 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
 245 #endif /* CONFIG_PM */
 246 #ifndef USE_REBOOT_NOTIFIER
 247 static void igb_shutdown(struct pci_dev *);
 248 #else
 249 static int igb_notify_reboot(struct notifier_block *, unsigned long, void *);
 250 static struct notifier_block igb_notifier_reboot = {
 251         .notifier_call  = igb_notify_reboot,
 252         .next           = NULL,
 253         .priority       = 0
 254 };
 255 #endif
 256 #ifdef IGB_DCA
 257 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
 258 static struct notifier_block dca_notifier = {
 259         .notifier_call  = igb_notify_dca,
 260         .next           = NULL,
 261         .priority       = 0
 262 };
 263 #endif
 264 #ifdef CONFIG_NET_POLL_CONTROLLER
 265 /* for netdump / net console */
 266 static void igb_netpoll(struct net_device *);
 267 #endif
 268
 269 #ifdef HAVE_PCI_ERS
 270 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
 271                      pci_channel_state_t);
 272 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
 273 static void igb_io_resume(struct pci_dev *);
 274
 275 static struct pci_error_handlers igb_err_handler = {
 276         .error_detected = igb_io_error_detected,
 277         .slot_reset = igb_io_slot_reset,
 278         .resume = igb_io_resume,
 279 };
 280 #endif
 281
 282 static void igb_init_fw(struct igb_adapter *adapter);
 283 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
 284
 285 static struct pci_driver igb_driver = {
 286         .name     = igb_driver_name,
 287         .id_table = igb_pci_tbl,
 288         .probe    = igb_probe,
 289         .remove   = __devexit_p(igb_remove),
 290 #ifdef CONFIG_PM
 291 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
 292         .driver.pm = &igb_pm_ops,
 293 #else
 294         .suspend  = igb_suspend,
 295         .resume   = igb_resume,
 296 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
 297 #endif /* CONFIG_PM */
 298 #ifndef USE_REBOOT_NOTIFIER
 299         .shutdown = igb_shutdown,
 300 #endif
 301 #ifdef HAVE_PCI_ERS
 302         .err_handler = &igb_err_handler
 303 #endif
 304 };
 305
 306 //MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 307 //MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
 308 //MODULE_LICENSE("GPL");
 309 //MODULE_VERSION(DRV_VERSION);
 310
 311 static void igb_vfta_set(struct igb_adapter *adapter, u32 vid, bool add)
 312 {
 313         struct e1000_hw *hw = &adapter->hw;
 314         struct e1000_host_mng_dhcp_cookie *mng_cookie = &hw->mng_cookie;
 315         u32 index = (vid >> E1000_VFTA_ENTRY_SHIFT) & E1000_VFTA_ENTRY_MASK;
 316         u32 mask = 1 << (vid & E1000_VFTA_ENTRY_BIT_SHIFT_MASK);
 317         u32 vfta;
 318
 319         /*
 320          * if this is the management vlan the only option is to add it in so
 321          * that the management pass through will continue to work
 322          */
 323         if ((mng_cookie->status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
 324             (vid == mng_cookie->vlan_id))
 325                 add = TRUE;
 326
 327         vfta = adapter->shadow_vfta[index];
 328
 329         if (add)
 330                 vfta |= mask;
 331         else
 332                 vfta &= ~mask;
 333
 334         e1000_write_vfta(hw, index, vfta);
 335         adapter->shadow_vfta[index] = vfta;
 336 }
 337
 338 static int debug = NETIF_MSG_DRV | NETIF_MSG_PROBE;
 339 //module_param(debug, int, 0);
 340 //MODULE_PARM_DESC(debug, "Debug level (0=none, ..., 16=all)");
 341
 342 /**
 343  * igb_init_module - Driver Registration Routine
 344  *
 345  * igb_init_module is the first routine called when the driver is
 346  * loaded. All it does is register with the PCI subsystem.
 347  **/
 348 static int __init igb_init_module(void)
 349 {
 350         int ret;
 351
 352         printk(KERN_INFO "%s - version %s\n",
 353                igb_driver_string, igb_driver_version);
 354
 355         printk(KERN_INFO "%s\n", igb_copyright);
 356 #ifdef IGB_HWMON
 357 /* only use IGB_PROCFS if IGB_HWMON is not defined */
 358 #else
 359 #ifdef IGB_PROCFS
 360         if (igb_procfs_topdir_init())
 361                 printk(KERN_INFO "Procfs failed to initialize topdir\n");
 362 #endif /* IGB_PROCFS */
 363 #endif /* IGB_HWMON  */
 364
 365 #ifdef IGB_DCA
 366         dca_register_notify(&dca_notifier);
 367 #endif
 368         ret = pci_register_driver(&igb_driver);
 369 #ifdef USE_REBOOT_NOTIFIER
 370         if (ret >= 0) {
 371                 register_reboot_notifier(&igb_notifier_reboot);
 372         }
 373 #endif
 374         return ret;
 375 }
 376
 377 #undef module_init
 378 #define module_init(x) static int x(void)  __attribute__((__unused__));
 379 module_init(igb_init_module);
 380
 381 /**
 382  * igb_exit_module - Driver Exit Cleanup Routine
 383  *
 384  * igb_exit_module is called just before the driver is removed
 385  * from memory.
 386  **/
 387 static void __exit igb_exit_module(void)
 388 {
 389 #ifdef IGB_DCA
 390         dca_unregister_notify(&dca_notifier);
 391 #endif
 392 #ifdef USE_REBOOT_NOTIFIER
 393         unregister_reboot_notifier(&igb_notifier_reboot);
 394 #endif
 395         pci_unregister_driver(&igb_driver);
 396
 397 #ifdef IGB_HWMON
 398 /* only compile IGB_PROCFS if IGB_HWMON is not defined */
 399 #else
 400 #ifdef IGB_PROCFS
 401         igb_procfs_topdir_exit();
 402 #endif /* IGB_PROCFS */
 403 #endif /* IGB_HWMON */
 404 }
 405
 406 #undef module_exit
 407 #define module_exit(x) static void x(void)  __attribute__((__unused__));
 408 module_exit(igb_exit_module);
 409
 410 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
 411 /**
 412  * igb_cache_ring_register - Descriptor ring to register mapping
 413  * @adapter: board private structure to initialize
 414  *
 415  * Once we know the feature-set enabled for the device, we'll cache
 416  * the register offset the descriptor ring is assigned to.
 417  **/
 418 static void igb_cache_ring_register(struct igb_adapter *adapter)
 419 {
 420         int i = 0, j = 0;
 421         u32 rbase_offset = adapter->vfs_allocated_count;
 422
 423         switch (adapter->hw.mac.type) {
 424         case e1000_82576:
 425                 /* The queues are allocated for virtualization such that VF 0
 426                  * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
 427                  * In order to avoid collision we start at the first free queue
 428                  * and continue consuming queues in the same sequence
 429                  */
 430                 if ((adapter->rss_queues > 1) && adapter->vmdq_pools) {
 431                         for (; i < adapter->rss_queues; i++)
 432                                 adapter->rx_ring[i]->reg_idx = rbase_offset +
 433                                                                Q_IDX_82576(i);
 434                 }
 435         case e1000_82575:
 436         case e1000_82580:
 437         case e1000_i350:
 438         case e1000_i354:
 439         case e1000_i210:
 440         case e1000_i211:
 441         default:
 442                 for (; i < adapter->num_rx_queues; i++)
 443                         adapter->rx_ring[i]->reg_idx = rbase_offset + i;
 444                 for (; j < adapter->num_tx_queues; j++)
 445                         adapter->tx_ring[j]->reg_idx = rbase_offset + j;
 446                 break;
 447         }
 448 }
 449
 450 static void igb_configure_lli(struct igb_adapter *adapter)
 451 {
 452         struct e1000_hw *hw = &adapter->hw;
 453         u16 port;
 454
 455         /* LLI should only be enabled for MSI-X or MSI interrupts */
 456         if (!adapter->msix_entries && !(adapter->flags & IGB_FLAG_HAS_MSI))
 457                 return;
 458
 459         if (adapter->lli_port) {
 460                 /* use filter 0 for port */
 461                 port = htons((u16)adapter->lli_port);
 462                 E1000_WRITE_REG(hw, E1000_IMIR(0),
 463                         (port | E1000_IMIR_PORT_IM_EN));
 464                 E1000_WRITE_REG(hw, E1000_IMIREXT(0),
 465                         (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
 466         }
 467
 468         if (adapter->flags & IGB_FLAG_LLI_PUSH) {
 469                 /* use filter 1 for push flag */
 470                 E1000_WRITE_REG(hw, E1000_IMIR(1),
 471                         (E1000_IMIR_PORT_BP | E1000_IMIR_PORT_IM_EN));
 472                 E1000_WRITE_REG(hw, E1000_IMIREXT(1),
 473                         (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_PSH));
 474         }
 475
 476         if (adapter->lli_size) {
 477                 /* use filter 2 for size */
 478                 E1000_WRITE_REG(hw, E1000_IMIR(2),
 479                         (E1000_IMIR_PORT_BP | E1000_IMIR_PORT_IM_EN));
 480                 E1000_WRITE_REG(hw, E1000_IMIREXT(2),
 481                         (adapter->lli_size | E1000_IMIREXT_CTRL_BP));
 482         }
 483
 484 }
 485
 486 /**
 487  *  igb_write_ivar - configure ivar for given MSI-X vector
 488  *  @hw: pointer to the HW structure
 489  *  @msix_vector: vector number we are allocating to a given ring
 490  *  @index: row index of IVAR register to write within IVAR table
 491  *  @offset: column offset of in IVAR, should be multiple of 8
 492  *
 493  *  This function is intended to handle the writing of the IVAR register
 494  *  for adapters 82576 and newer.  The IVAR table consists of 2 columns,
 495  *  each containing an cause allocation for an Rx and Tx ring, and a
 496  *  variable number of rows depending on the number of queues supported.
 497  **/
 498 static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
 499                            int index, int offset)
 500 {
 501         u32 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
 502
 503         /* clear any bits that are currently set */
 504         ivar &= ~((u32)0xFF << offset);
 505
 506         /* write vector and valid bit */
 507         ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
 508
 509         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
 510 }
 511
 512 #define IGB_N0_QUEUE -1
 513 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
 514 {
 515         struct igb_adapter *adapter = q_vector->adapter;
 516         struct e1000_hw *hw = &adapter->hw;
 517         int rx_queue = IGB_N0_QUEUE;
 518         int tx_queue = IGB_N0_QUEUE;
 519         u32 msixbm = 0;
 520
 521         if (q_vector->rx.ring)
 522                 rx_queue = q_vector->rx.ring->reg_idx;
 523         if (q_vector->tx.ring)
 524                 tx_queue = q_vector->tx.ring->reg_idx;
 525
 526         switch (hw->mac.type) {
 527         case e1000_82575:
 528                 /* The 82575 assigns vectors using a bitmask, which matches the
 529                    bitmask for the EICR/EIMS/EIMC registers.  To assign one
 530                    or more queues to a vector, we write the appropriate bits
 531                    into the MSIXBM register for that vector. */
 532                 if (rx_queue > IGB_N0_QUEUE)
 533                         msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
 534                 if (tx_queue > IGB_N0_QUEUE)
 535                         msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
 536                 if (!adapter->msix_entries && msix_vector == 0)
 537                         msixbm |= E1000_EIMS_OTHER;
 538                 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), msix_vector, msixbm);
 539                 q_vector->eims_value = msixbm;
 540                 break;
 541         case e1000_82576:
 542                 /*
 543                  * 82576 uses a table that essentially consists of 2 columns
 544                  * with 8 rows.  The ordering is column-major so we use the
 545                  * lower 3 bits as the row index, and the 4th bit as the
 546                  * column offset.
 547                  */
 548                 if (rx_queue > IGB_N0_QUEUE)
 549                         igb_write_ivar(hw, msix_vector,
 550                                        rx_queue & 0x7,
 551                                        (rx_queue & 0x8) << 1);
 552                 if (tx_queue > IGB_N0_QUEUE)
 553                         igb_write_ivar(hw, msix_vector,
 554                                        tx_queue & 0x7,
 555                                        ((tx_queue & 0x8) << 1) + 8);
 556                 q_vector->eims_value = 1 << msix_vector;
 557                 break;
 558         case e1000_82580:
 559         case e1000_i350:
 560         case e1000_i354:
 561         case e1000_i210:
 562         case e1000_i211:
 563                 /*
 564                  * On 82580 and newer adapters the scheme is similar to 82576
 565                  * however instead of ordering column-major we have things
 566                  * ordered row-major.  So we traverse the table by using
 567                  * bit 0 as the column offset, and the remaining bits as the
 568                  * row index.
 569                  */
 570                 if (rx_queue > IGB_N0_QUEUE)
 571                         igb_write_ivar(hw, msix_vector,
 572                                        rx_queue >> 1,
 573                                        (rx_queue & 0x1) << 4);
 574                 if (tx_queue > IGB_N0_QUEUE)
 575                         igb_write_ivar(hw, msix_vector,
 576                                        tx_queue >> 1,
 577                                        ((tx_queue & 0x1) << 4) + 8);
 578                 q_vector->eims_value = 1 << msix_vector;
 579                 break;
 580         default:
 581                 BUG();
 582                 break;
 583         }
 584
 585         /* add q_vector eims value to global eims_enable_mask */
 586         adapter->eims_enable_mask |= q_vector->eims_value;
 587
 588         /* configure q_vector to set itr on first interrupt */
 589         q_vector->set_itr = 1;
 590 }
 591
 592 /**
 593  * igb_configure_msix - Configure MSI-X hardware
 594  *
 595  * igb_configure_msix sets up the hardware to properly
 596  * generate MSI-X interrupts.
 597  **/
 598 static void igb_configure_msix(struct igb_adapter *adapter)
 599 {
 600         u32 tmp;
 601         int i, vector = 0;
 602         struct e1000_hw *hw = &adapter->hw;
 603
 604         adapter->eims_enable_mask = 0;
 605
 606         /* set vector for other causes, i.e. link changes */
 607         switch (hw->mac.type) {
 608         case e1000_82575:
 609                 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
 610                 /* enable MSI-X PBA support*/
 611                 tmp |= E1000_CTRL_EXT_PBA_CLR;
 612
 613                 /* Auto-Mask interrupts upon ICR read. */
 614                 tmp |= E1000_CTRL_EXT_EIAME;
 615                 tmp |= E1000_CTRL_EXT_IRCA;
 616
 617                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
 618
 619                 /* enable msix_other interrupt */
 620                 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), vector++,
 621                                       E1000_EIMS_OTHER);
 622                 adapter->eims_other = E1000_EIMS_OTHER;
 623
 624                 break;
 625
 626         case e1000_82576:
 627         case e1000_82580:
 628         case e1000_i350:
 629         case e1000_i354:
 630         case e1000_i210:
 631         case e1000_i211:
 632                 /* Turn on MSI-X capability first, or our settings
 633                  * won't stick.  And it will take days to debug. */
 634                 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE |
 635                                 E1000_GPIE_PBA | E1000_GPIE_EIAME |
 636                                 E1000_GPIE_NSICR);
 637
 638                 /* enable msix_other interrupt */
 639                 adapter->eims_other = 1 << vector;
 640                 tmp = (vector++ | E1000_IVAR_VALID) << 8;
 641
 642                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmp);
 643                 break;
 644         default:
 645                 /* do nothing, since nothing else supports MSI-X */
 646                 break;
 647         } /* switch (hw->mac.type) */
 648
 649         adapter->eims_enable_mask |= adapter->eims_other;
 650
 651         for (i = 0; i < adapter->num_q_vectors; i++)
 652                 igb_assign_vector(adapter->q_vector[i], vector++);
 653
 654         E1000_WRITE_FLUSH(hw);
 655 }
 656
 657 /**
 658  * igb_request_msix - Initialize MSI-X interrupts
 659  *
 660  * igb_request_msix allocates MSI-X vectors and requests interrupts from the
 661  * kernel.
 662  **/
 663 static int igb_request_msix(struct igb_adapter *adapter)
 664 {
 665         struct net_device *netdev = adapter->netdev;
 666         struct e1000_hw *hw = &adapter->hw;
 667         int i, err = 0, vector = 0, free_vector = 0;
 668
 669         err = request_irq(adapter->msix_entries[vector].vector,
 670                           &igb_msix_other, 0, netdev->name, adapter);
 671         if (err)
 672                 goto err_out;
 673
 674         for (i = 0; i < adapter->num_q_vectors; i++) {
 675                 struct igb_q_vector *q_vector = adapter->q_vector[i];
 676
 677                 vector++;
 678
 679                 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
 680
 681                 if (q_vector->rx.ring && q_vector->tx.ring)
 682                         sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
 683                                 q_vector->rx.ring->queue_index);
 684                 else if (q_vector->tx.ring)
 685                         sprintf(q_vector->name, "%s-tx-%u", netdev->name,
 686                                 q_vector->tx.ring->queue_index);
 687                 else if (q_vector->rx.ring)
 688                         sprintf(q_vector->name, "%s-rx-%u", netdev->name,
 689                                 q_vector->rx.ring->queue_index);
 690                 else
 691                         sprintf(q_vector->name, "%s-unused", netdev->name);
 692
 693                 err = request_irq(adapter->msix_entries[vector].vector,
 694                                   igb_msix_ring, 0, q_vector->name,
 695                                   q_vector);
 696                 if (err)
 697                         goto err_free;
 698         }
 699
 700         igb_configure_msix(adapter);
 701         return 0;
 702
 703 err_free:
 704         /* free already assigned IRQs */
 705         free_irq(adapter->msix_entries[free_vector++].vector, adapter);
 706
 707         vector--;
 708         for (i = 0; i < vector; i++) {
 709                 free_irq(adapter->msix_entries[free_vector++].vector,
 710                          adapter->q_vector[i]);
 711         }
 712 err_out:
 713         return err;
 714 }
 715
 716 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
 717 {
 718         if (adapter->msix_entries) {
 719                 pci_disable_msix(adapter->pdev);
 720                 kfree(adapter->msix_entries);
 721                 adapter->msix_entries = NULL;
 722         } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
 723                 pci_disable_msi(adapter->pdev);
 724         }
 725 }
 726
 727 /**
 728  * igb_free_q_vector - Free memory allocated for specific interrupt vector
 729  * @adapter: board private structure to initialize
 730  * @v_idx: Index of vector to be freed
 731  *
 732  * This function frees the memory allocated to the q_vector.  In addition if
 733  * NAPI is enabled it will delete any references to the NAPI struct prior
 734  * to freeing the q_vector.
 735  **/
 736 static void igb_free_q_vector(struct igb_adapter *adapter, int v_idx)
 737 {
 738         struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
 739
 740         if (q_vector->tx.ring)
 741                 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
 742
 743         if (q_vector->rx.ring)
 744                 adapter->tx_ring[q_vector->rx.ring->queue_index] = NULL;
 745
 746         adapter->q_vector[v_idx] = NULL;
 747         netif_napi_del(&q_vector->napi);
 748 #ifndef IGB_NO_LRO
 749         __skb_queue_purge(&q_vector->lrolist.active);
 750 #endif
 751         kfree(q_vector);
 752 }
 753
 754 /**
 755  * igb_free_q_vectors - Free memory allocated for interrupt vectors
 756  * @adapter: board private structure to initialize
 757  *
 758  * This function frees the memory allocated to the q_vectors.  In addition if
 759  * NAPI is enabled it will delete any references to the NAPI struct prior
 760  * to freeing the q_vector.
 761  **/
 762 static void igb_free_q_vectors(struct igb_adapter *adapter)
 763 {
 764         int v_idx = adapter->num_q_vectors;
 765
 766         adapter->num_tx_queues = 0;
 767         adapter->num_rx_queues = 0;
 768         adapter->num_q_vectors = 0;
 769
 770         while (v_idx--)
 771                 igb_free_q_vector(adapter, v_idx);
 772 }
 773
 774 /**
 775  * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
 776  *
 777  * This function resets the device so that it has 0 rx queues, tx queues, and
 778  * MSI-X interrupts allocated.
 779  */
 780 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
 781 {
 782         igb_free_q_vectors(adapter);
 783         igb_reset_interrupt_capability(adapter);
 784 }
 785
 786 /**
 787  * igb_process_mdd_event
 788  * @adapter - board private structure
 789  *
 790  * Identify a malicious VF, disable the VF TX/RX queues and log a message.
 791  */
 792 static void igb_process_mdd_event(struct igb_adapter *adapter)
 793 {
 794         struct e1000_hw *hw = &adapter->hw;
 795         u32 lvmmc, vfte, vfre, mdfb;
 796         u8 vf_queue;
 797
 798         lvmmc = E1000_READ_REG(hw, E1000_LVMMC);
 799         vf_queue = lvmmc >> 29;
 800
 801         /* VF index cannot be bigger or equal to VFs allocated */
 802         if (vf_queue >= adapter->vfs_allocated_count)
 803                 return;
 804
 805         netdev_info(adapter->netdev,
 806                     "VF %d misbehaved. VF queues are disabled. "
 807                     "VM misbehavior code is 0x%x\n", vf_queue, lvmmc);
 808
 809         /* Disable VFTE and VFRE related bits */
 810         vfte = E1000_READ_REG(hw, E1000_VFTE);
 811         vfte &= ~(1 << vf_queue);
 812         E1000_WRITE_REG(hw, E1000_VFTE, vfte);
 813
 814         vfre = E1000_READ_REG(hw, E1000_VFRE);
 815         vfre &= ~(1 << vf_queue);
 816         E1000_WRITE_REG(hw, E1000_VFRE, vfre);
 817
 818         /* Disable MDFB related bit. Clear on write */
 819         mdfb = E1000_READ_REG(hw, E1000_MDFB);
 820         mdfb |= (1 << vf_queue);
 821         E1000_WRITE_REG(hw, E1000_MDFB, mdfb);
 822
 823         /* Reset the specific VF */
 824         E1000_WRITE_REG(hw, E1000_VTCTRL(vf_queue), E1000_VTCTRL_RST);
 825 }
 826
 827 /**
 828  * igb_disable_mdd
 829  * @adapter - board private structure
 830  *
 831  * Disable MDD behavior in the HW
 832  **/
 833 static void igb_disable_mdd(struct igb_adapter *adapter)
 834 {
 835         struct e1000_hw *hw = &adapter->hw;
 836         u32 reg;
 837
 838         if ((hw->mac.type != e1000_i350) ||
 839             (hw->mac.type != e1000_i354))
 840                 return;
 841
 842         reg = E1000_READ_REG(hw, E1000_DTXCTL);
 843         reg &= (~E1000_DTXCTL_MDP_EN);
 844         E1000_WRITE_REG(hw, E1000_DTXCTL, reg);
 845 }
 846
 847 /**
 848  * igb_enable_mdd
 849  * @adapter - board private structure
 850  *
 851  * Enable the HW to detect malicious driver and sends an interrupt to
 852  * the driver.
 853  **/
 854 static void igb_enable_mdd(struct igb_adapter *adapter)
 855 {
 856         struct e1000_hw *hw = &adapter->hw;
 857         u32 reg;
 858
 859         /* Only available on i350 device */
 860         if (hw->mac.type != e1000_i350)
 861                 return;
 862
 863         reg = E1000_READ_REG(hw, E1000_DTXCTL);
 864         reg |= E1000_DTXCTL_MDP_EN;
 865         E1000_WRITE_REG(hw, E1000_DTXCTL, reg);
 866 }
 867
 868 /**
 869  * igb_reset_sriov_capability - disable SR-IOV if enabled
 870  *
 871  * Attempt to disable single root IO virtualization capabilites present in the
 872  * kernel.
 873  **/
 874 static void igb_reset_sriov_capability(struct igb_adapter *adapter)
 875 {
 876         struct pci_dev *pdev = adapter->pdev;
 877         struct e1000_hw *hw = &adapter->hw;
 878
 879         /* reclaim resources allocated to VFs */
 880         if (adapter->vf_data) {
 881                 if (!pci_vfs_assigned(pdev)) {
 882                         /*
 883                          * disable iov and allow time for transactions to
 884                          * clear
 885                          */
 886                         pci_disable_sriov(pdev);
 887                         msleep(500);
 888
 889                         dev_info(pci_dev_to_dev(pdev), "IOV Disabled\n");
 890                 } else {
 891                         dev_info(pci_dev_to_dev(pdev), "IOV Not Disabled\n "
 892                                         "VF(s) are assigned to guests!\n");
 893                 }
 894                 /* Disable Malicious Driver Detection */
 895                 igb_disable_mdd(adapter);
 896
 897                 /* free vf data storage */
 898                 kfree(adapter->vf_data);
 899                 adapter->vf_data = NULL;
 900
 901                 /* switch rings back to PF ownership */
 902                 E1000_WRITE_REG(hw, E1000_IOVCTL,
 903                                 E1000_IOVCTL_REUSE_VFQ);
 904                 E1000_WRITE_FLUSH(hw);
 905                 msleep(100);
 906         }
 907
 908         adapter->vfs_allocated_count = 0;
 909 }
 910
 911 /**
 912  * igb_set_sriov_capability - setup SR-IOV if supported
 913  *
 914  * Attempt to enable single root IO virtualization capabilites present in the
 915  * kernel.
 916  **/
 917 static void igb_set_sriov_capability(struct igb_adapter *adapter)
 918 {
 919         struct pci_dev *pdev = adapter->pdev;
 920         int old_vfs = 0;
 921         int i;
 922
 923         old_vfs = pci_num_vf(pdev);
 924         if (old_vfs) {
 925                 dev_info(pci_dev_to_dev(pdev),
 926                                 "%d pre-allocated VFs found - override "
 927                                 "max_vfs setting of %d\n", old_vfs,
 928                                 adapter->vfs_allocated_count);
 929                 adapter->vfs_allocated_count = old_vfs;
 930         }
 931         /* no VFs requested, do nothing */
 932         if (!adapter->vfs_allocated_count)
 933                 return;
 934
 935         /* allocate vf data storage */
 936         adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
 937                                    sizeof(struct vf_data_storage),
 938                                    GFP_KERNEL);
 939
 940         if (adapter->vf_data) {
 941                 if (!old_vfs) {
 942                         if (pci_enable_sriov(pdev,
 943                                         adapter->vfs_allocated_count))
 944                                 goto err_out;
 945                 }
 946                 for (i = 0; i < adapter->vfs_allocated_count; i++)
 947                         igb_vf_configure(adapter, i);
 948
 949                 switch (adapter->hw.mac.type) {
 950                 case e1000_82576:
 951                 case e1000_i350:
 952                         /* Enable VM to VM loopback by default */
 953                         adapter->flags |= IGB_FLAG_LOOPBACK_ENABLE;
 954                         break;
 955                 default:
 956                         /* Currently no other hardware supports loopback */
 957                         break;
 958                 }
 959
 960                 /* DMA Coalescing is not supported in IOV mode. */
 961                 if (adapter->hw.mac.type >= e1000_i350)
 962                 adapter->dmac = IGB_DMAC_DISABLE;
 963                 if (adapter->hw.mac.type < e1000_i350)
 964                 adapter->flags |= IGB_FLAG_DETECT_BAD_DMA;
 965                 return;
 966
 967         }
 968
 969 err_out:
 970         kfree(adapter->vf_data);
 971         adapter->vf_data = NULL;
 972         adapter->vfs_allocated_count = 0;
 973         dev_warn(pci_dev_to_dev(pdev),
 974                         "Failed to initialize SR-IOV virtualization\n");
 975 }
 976
 977 /**
 978  * igb_set_interrupt_capability - set MSI or MSI-X if supported
 979  *
 980  * Attempt to configure interrupts using the best available
 981  * capabilities of the hardware and kernel.
 982  **/
 983 static void igb_set_interrupt_capability(struct igb_adapter *adapter, bool msix)
 984 {
 985         struct pci_dev *pdev = adapter->pdev;
 986         int err;
 987         int numvecs, i;
 988
 989         if (!msix)
 990                 adapter->int_mode = IGB_INT_MODE_MSI;
 991
 992         /* Number of supported queues. */
 993         adapter->num_rx_queues = adapter->rss_queues;
 994
 995         if (adapter->vmdq_pools > 1)
 996                 adapter->num_rx_queues += adapter->vmdq_pools - 1;
 997
 998 #ifdef HAVE_TX_MQ
 999         if (adapter->vmdq_pools)
1000                 adapter->num_tx_queues = adapter->vmdq_pools;
1001         else
1002                 adapter->num_tx_queues = adapter->num_rx_queues;
1003 #else
1004         adapter->num_tx_queues = max_t(u32, 1, adapter->vmdq_pools);
1005 #endif
1006
1007         switch (adapter->int_mode) {
1008         case IGB_INT_MODE_MSIX:
1009                 /* start with one vector for every rx queue */
1010                 numvecs = adapter->num_rx_queues;
1011
1012                 /* if tx handler is separate add 1 for every tx queue */
1013                 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1014                         numvecs += adapter->num_tx_queues;
1015
1016                 /* store the number of vectors reserved for queues */
1017                 adapter->num_q_vectors = numvecs;
1018
1019                 /* add 1 vector for link status interrupts */
1020                 numvecs++;
1021                 adapter->msix_entries = kcalloc(numvecs,
1022                                                 sizeof(struct msix_entry),
1023                                                 GFP_KERNEL);
1024                 if (adapter->msix_entries) {
1025                         for (i = 0; i < numvecs; i++)
1026                                 adapter->msix_entries[i].entry = i;
1027
1028 #ifdef HAVE_PCI_ENABLE_MSIX
1029                         err = pci_enable_msix(pdev,
1030                                               adapter->msix_entries, numvecs);
1031 #else
1032                         err = pci_enable_msix_range(pdev,
1033                                         adapter->msix_entries,
1034                                         numvecs,
1035                                         numvecs);
1036 #endif
1037                         if (err == 0)
1038                                 break;
1039                 }
1040                 /* MSI-X failed, so fall through and try MSI */
1041                 dev_warn(pci_dev_to_dev(pdev), "Failed to initialize MSI-X interrupts. "
1042                          "Falling back to MSI interrupts.\n");
1043                 igb_reset_interrupt_capability(adapter);
1044         case IGB_INT_MODE_MSI:
1045                 if (!pci_enable_msi(pdev))
1046                         adapter->flags |= IGB_FLAG_HAS_MSI;
1047                 else
1048                         dev_warn(pci_dev_to_dev(pdev), "Failed to initialize MSI "
1049                                  "interrupts.  Falling back to legacy "
1050                                  "interrupts.\n");
1051                 /* Fall through */
1052         case IGB_INT_MODE_LEGACY:
1053                 /* disable advanced features and set number of queues to 1 */
1054                 igb_reset_sriov_capability(adapter);
1055                 adapter->vmdq_pools = 0;
1056                 adapter->rss_queues = 1;
1057                 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1058                 adapter->num_rx_queues = 1;
1059                 adapter->num_tx_queues = 1;
1060                 adapter->num_q_vectors = 1;
1061                 /* Don't do anything; this is system default */
1062                 break;
1063         }
1064 }
1065
1066 static void igb_add_ring(struct igb_ring *ring,
1067                          struct igb_ring_container *head)
1068 {
1069         head->ring = ring;
1070         head->count++;
1071 }
1072
1073 /**
1074  * igb_alloc_q_vector - Allocate memory for a single interrupt vector
1075  * @adapter: board private structure to initialize
1076  * @v_count: q_vectors allocated on adapter, used for ring interleaving
1077  * @v_idx: index of vector in adapter struct
1078  * @txr_count: total number of Tx rings to allocate
1079  * @txr_idx: index of first Tx ring to allocate
1080  * @rxr_count: total number of Rx rings to allocate
1081  * @rxr_idx: index of first Rx ring to allocate
1082  *
1083  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
1084  **/
1085 static int igb_alloc_q_vector(struct igb_adapter *adapter,
1086                               unsigned int v_count, unsigned int v_idx,
1087                               unsigned int txr_count, unsigned int txr_idx,
1088                               unsigned int rxr_count, unsigned int rxr_idx)
1089 {
1090         struct igb_q_vector *q_vector;
1091         struct igb_ring *ring;
1092         int ring_count, size;
1093
1094         /* igb only supports 1 Tx and/or 1 Rx queue per vector */
1095         if (txr_count > 1 || rxr_count > 1)
1096                 return -ENOMEM;
1097
1098         ring_count = txr_count + rxr_count;
1099         size = sizeof(struct igb_q_vector) +
1100                (sizeof(struct igb_ring) * ring_count);
1101
1102         /* allocate q_vector and rings */
1103         q_vector = kzalloc(size, GFP_KERNEL);
1104         if (!q_vector)
1105                 return -ENOMEM;
1106
1107 #ifndef IGB_NO_LRO
1108         /* initialize LRO */
1109         __skb_queue_head_init(&q_vector->lrolist.active);
1110
1111 #endif
1112         /* initialize NAPI */
1113         netif_napi_add(adapter->netdev, &q_vector->napi,
1114                        igb_poll, 64);
1115
1116         /* tie q_vector and adapter together */
1117         adapter->q_vector[v_idx] = q_vector;
1118         q_vector->adapter = adapter;
1119
1120         /* initialize work limits */
1121         q_vector->tx.work_limit = adapter->tx_work_limit;
1122
1123         /* initialize ITR configuration */
1124         q_vector->itr_register = adapter->hw.hw_addr + E1000_EITR(0);
1125         q_vector->itr_val = IGB_START_ITR;
1126
1127         /* initialize pointer to rings */
1128         ring = q_vector->ring;
1129
1130         /* initialize ITR */
1131         if (rxr_count) {
1132                 /* rx or rx/tx vector */
1133                 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
1134                         q_vector->itr_val = adapter->rx_itr_setting;
1135         } else {
1136                 /* tx only vector */
1137                 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
1138                         q_vector->itr_val = adapter->tx_itr_setting;
1139         }
1140
1141         if (txr_count) {
1142                 /* assign generic ring traits */
1143                 ring->dev = &adapter->pdev->dev;
1144                 ring->netdev = adapter->netdev;
1145
1146                 /* configure backlink on ring */
1147                 ring->q_vector = q_vector;
1148
1149                 /* update q_vector Tx values */
1150                 igb_add_ring(ring, &q_vector->tx);
1151
1152                 /* For 82575, context index must be unique per ring. */
1153                 if (adapter->hw.mac.type == e1000_82575)
1154                         set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
1155
1156                 /* apply Tx specific ring traits */
1157                 ring->count = adapter->tx_ring_count;
1158                 ring->queue_index = txr_idx;
1159
1160                 /* assign ring to adapter */
1161                 adapter->tx_ring[txr_idx] = ring;
1162
1163                 /* push pointer to next ring */
1164                 ring++;
1165         }
1166
1167         if (rxr_count) {
1168                 /* assign generic ring traits */
1169                 ring->dev = &adapter->pdev->dev;
1170                 ring->netdev = adapter->netdev;
1171
1172                 /* configure backlink on ring */
1173                 ring->q_vector = q_vector;
1174
1175                 /* update q_vector Rx values */
1176                 igb_add_ring(ring, &q_vector->rx);
1177
1178 #ifndef HAVE_NDO_SET_FEATURES
1179                 /* enable rx checksum */
1180                 set_bit(IGB_RING_FLAG_RX_CSUM, &ring->flags);
1181
1182 #endif
1183                 /* set flag indicating ring supports SCTP checksum offload */
1184                 if (adapter->hw.mac.type >= e1000_82576)
1185                         set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
1186
1187                 if ((adapter->hw.mac.type == e1000_i350) ||
1188                     (adapter->hw.mac.type == e1000_i354))
1189                         set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
1190
1191                 /* apply Rx specific ring traits */
1192                 ring->count = adapter->rx_ring_count;
1193                 ring->queue_index = rxr_idx;
1194
1195                 /* assign ring to adapter */
1196                 adapter->rx_ring[rxr_idx] = ring;
1197         }
1198
1199         return 0;
1200 }
1201
1202 /**
1203  * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1204  * @adapter: board private structure to initialize
1205  *
1206  * We allocate one q_vector per queue interrupt.  If allocation fails we
1207  * return -ENOMEM.
1208  **/
1209 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1210 {
1211         int q_vectors = adapter->num_q_vectors;
1212         int rxr_remaining = adapter->num_rx_queues;
1213         int txr_remaining = adapter->num_tx_queues;
1214         int rxr_idx = 0, txr_idx = 0, v_idx = 0;
1215         int err;
1216
1217         if (q_vectors >= (rxr_remaining + txr_remaining)) {
1218                 for (; rxr_remaining; v_idx++) {
1219                         err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
1220                                                  0, 0, 1, rxr_idx);
1221
1222                         if (err)
1223                                 goto err_out;
1224
1225                         /* update counts and index */
1226                         rxr_remaining--;
1227                         rxr_idx++;
1228                 }
1229         }
1230
1231         for (; v_idx < q_vectors; v_idx++) {
1232                 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
1233                 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
1234                 err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
1235                                          tqpv, txr_idx, rqpv, rxr_idx);
1236
1237                 if (err)
1238                         goto err_out;
1239
1240                 /* update counts and index */
1241                 rxr_remaining -= rqpv;
1242                 txr_remaining -= tqpv;
1243                 rxr_idx++;
1244                 txr_idx++;
1245         }
1246
1247         return 0;
1248
1249 err_out:
1250         adapter->num_tx_queues = 0;
1251         adapter->num_rx_queues = 0;
1252         adapter->num_q_vectors = 0;
1253
1254         while (v_idx--)
1255                 igb_free_q_vector(adapter, v_idx);
1256
1257         return -ENOMEM;
1258 }
1259
1260 /**
1261  * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1262  *
1263  * This function initializes the interrupts and allocates all of the queues.
1264  **/
1265 static int igb_init_interrupt_scheme(struct igb_adapter *adapter, bool msix)
1266 {
1267         struct pci_dev *pdev = adapter->pdev;
1268         int err;
1269
1270         igb_set_interrupt_capability(adapter, msix);
1271
1272         err = igb_alloc_q_vectors(adapter);
1273         if (err) {
1274                 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for vectors\n");
1275                 goto err_alloc_q_vectors;
1276         }
1277
1278         igb_cache_ring_register(adapter);
1279
1280         return 0;
1281
1282 err_alloc_q_vectors:
1283         igb_reset_interrupt_capability(adapter);
1284         return err;
1285 }
1286
1287 /**
1288  * igb_request_irq - initialize interrupts
1289  *
1290  * Attempts to configure interrupts using the best available
1291  * capabilities of the hardware and kernel.
1292  **/
1293 static int igb_request_irq(struct igb_adapter *adapter)
1294 {
1295         struct net_device *netdev = adapter->netdev;
1296         struct pci_dev *pdev = adapter->pdev;
1297         int err = 0;
1298
1299         if (adapter->msix_entries) {
1300                 err = igb_request_msix(adapter);
1301                 if (!err)
1302                         goto request_done;
1303                 /* fall back to MSI */
1304                 igb_free_all_tx_resources(adapter);
1305                 igb_free_all_rx_resources(adapter);
1306
1307                 igb_clear_interrupt_scheme(adapter);
1308                 igb_reset_sriov_capability(adapter);
1309                 err = igb_init_interrupt_scheme(adapter, false);
1310                 if (err)
1311                         goto request_done;
1312                 igb_setup_all_tx_resources(adapter);
1313                 igb_setup_all_rx_resources(adapter);
1314                 igb_configure(adapter);
1315         }
1316
1317         igb_assign_vector(adapter->q_vector[0], 0);
1318
1319         if (adapter->flags & IGB_FLAG_HAS_MSI) {
1320                 err = request_irq(pdev->irq, &igb_intr_msi, 0,
1321                                   netdev->name, adapter);
1322                 if (!err)
1323                         goto request_done;
1324
1325                 /* fall back to legacy interrupts */
1326                 igb_reset_interrupt_capability(adapter);
1327                 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1328         }
1329
1330         err = request_irq(pdev->irq, &igb_intr, IRQF_SHARED,
1331                           netdev->name, adapter);
1332
1333         if (err)
1334                 dev_err(pci_dev_to_dev(pdev), "Error %d getting interrupt\n",
1335                         err);
1336
1337 request_done:
1338         return err;
1339 }
1340
1341 static void igb_free_irq(struct igb_adapter *adapter)
1342 {
1343         if (adapter->msix_entries) {
1344                 int vector = 0, i;
1345
1346                 free_irq(adapter->msix_entries[vector++].vector, adapter);
1347
1348                 for (i = 0; i < adapter->num_q_vectors; i++)
1349                         free_irq(adapter->msix_entries[vector++].vector,
1350                                  adapter->q_vector[i]);
1351         } else {
1352                 free_irq(adapter->pdev->irq, adapter);
1353         }
1354 }
1355
1356 /**
1357  * igb_irq_disable - Mask off interrupt generation on the NIC
1358  * @adapter: board private structure
1359  **/
1360 static void igb_irq_disable(struct igb_adapter *adapter)
1361 {
1362         struct e1000_hw *hw = &adapter->hw;
1363
1364         /*
1365          * we need to be careful when disabling interrupts.  The VFs are also
1366          * mapped into these registers and so clearing the bits can cause
1367          * issues on the VF drivers so we only need to clear what we set
1368          */
1369         if (adapter->msix_entries) {
1370                 u32 regval = E1000_READ_REG(hw, E1000_EIAM);
1371                 E1000_WRITE_REG(hw, E1000_EIAM, regval & ~adapter->eims_enable_mask);
1372                 E1000_WRITE_REG(hw, E1000_EIMC, adapter->eims_enable_mask);
1373                 regval = E1000_READ_REG(hw, E1000_EIAC);
1374                 E1000_WRITE_REG(hw, E1000_EIAC, regval & ~adapter->eims_enable_mask);
1375         }
1376
1377         E1000_WRITE_REG(hw, E1000_IAM, 0);
1378         E1000_WRITE_REG(hw, E1000_IMC, ~0);
1379         E1000_WRITE_FLUSH(hw);
1380
1381         if (adapter->msix_entries) {
1382                 int vector = 0, i;
1383
1384                 synchronize_irq(adapter->msix_entries[vector++].vector);
1385
1386                 for (i = 0; i < adapter->num_q_vectors; i++)
1387                         synchronize_irq(adapter->msix_entries[vector++].vector);
1388         } else {
1389                 synchronize_irq(adapter->pdev->irq);
1390         }
1391 }
1392
1393 /**
1394  * igb_irq_enable - Enable default interrupt generation settings
1395  * @adapter: board private structure
1396  **/
1397 static void igb_irq_enable(struct igb_adapter *adapter)
1398 {
1399         struct e1000_hw *hw = &adapter->hw;
1400
1401         if (adapter->msix_entries) {
1402                 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1403                 u32 regval = E1000_READ_REG(hw, E1000_EIAC);
1404                 E1000_WRITE_REG(hw, E1000_EIAC, regval | adapter->eims_enable_mask);
1405                 regval = E1000_READ_REG(hw, E1000_EIAM);
1406                 E1000_WRITE_REG(hw, E1000_EIAM, regval | adapter->eims_enable_mask);
1407                 E1000_WRITE_REG(hw, E1000_EIMS, adapter->eims_enable_mask);
1408                 if (adapter->vfs_allocated_count) {
1409                         E1000_WRITE_REG(hw, E1000_MBVFIMR, 0xFF);
1410                         ims |= E1000_IMS_VMMB;
1411                         if (adapter->mdd)
1412                                 if ((adapter->hw.mac.type == e1000_i350) ||
1413                                     (adapter->hw.mac.type == e1000_i354))
1414                                 ims |= E1000_IMS_MDDET;
1415                 }
1416                 E1000_WRITE_REG(hw, E1000_IMS, ims);
1417         } else {
1418                 E1000_WRITE_REG(hw, E1000_IMS, IMS_ENABLE_MASK |
1419                                 E1000_IMS_DRSTA);
1420                 E1000_WRITE_REG(hw, E1000_IAM, IMS_ENABLE_MASK |
1421                                 E1000_IMS_DRSTA);
1422         }
1423 }
1424
1425 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1426 {
1427         struct e1000_hw *hw = &adapter->hw;
1428         u16 vid = adapter->hw.mng_cookie.vlan_id;
1429         u16 old_vid = adapter->mng_vlan_id;
1430
1431         if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1432                 /* add VID to filter table */
1433                 igb_vfta_set(adapter, vid, TRUE);
1434                 adapter->mng_vlan_id = vid;
1435         } else {
1436                 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1437         }
1438
1439         if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1440             (vid != old_vid) &&
1441 #ifdef HAVE_VLAN_RX_REGISTER
1442             !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1443 #else
1444             !test_bit(old_vid, adapter->active_vlans)) {
1445 #endif
1446                 /* remove VID from filter table */
1447                 igb_vfta_set(adapter, old_vid, FALSE);
1448         }
1449 }
1450
1451 /**
1452  * igb_release_hw_control - release control of the h/w to f/w
1453  * @adapter: address of board private structure
1454  *
1455  * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1456  * For ASF and Pass Through versions of f/w this means that the
1457  * driver is no longer loaded.
1458  *
1459  **/
1460 static void igb_release_hw_control(struct igb_adapter *adapter)
1461 {
1462         struct e1000_hw *hw = &adapter->hw;
1463         u32 ctrl_ext;
1464
1465         /* Let firmware take over control of h/w */
1466         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1467         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1468                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1469 }
1470
1471 /**
1472  * igb_get_hw_control - get control of the h/w from f/w
1473  * @adapter: address of board private structure
1474  *
1475  * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1476  * For ASF and Pass Through versions of f/w this means that
1477  * the driver is loaded.
1478  *
1479  **/
1480 static void igb_get_hw_control(struct igb_adapter *adapter)
1481 {
1482         struct e1000_hw *hw = &adapter->hw;
1483         u32 ctrl_ext;
1484
1485         /* Let firmware know the driver has taken over */
1486         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1487         E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1488                         ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1489 }
1490
1491 /**
1492  * igb_configure - configure the hardware for RX and TX
1493  * @adapter: private board structure
1494  **/
1495 static void igb_configure(struct igb_adapter *adapter)
1496 {
1497         struct net_device *netdev = adapter->netdev;
1498         int i;
1499
1500         igb_get_hw_control(adapter);
1501         igb_set_rx_mode(netdev);
1502
1503         igb_restore_vlan(adapter);
1504
1505         igb_setup_tctl(adapter);
1506         igb_setup_mrqc(adapter);
1507         igb_setup_rctl(adapter);
1508
1509         igb_configure_tx(adapter);
1510         igb_configure_rx(adapter);
1511
1512         e1000_rx_fifo_flush_82575(&adapter->hw);
1513 #ifdef CONFIG_NETDEVICES_MULTIQUEUE
1514         if (adapter->num_tx_queues > 1)
1515                 netdev->features |= NETIF_F_MULTI_QUEUE;
1516         else
1517                 netdev->features &= ~NETIF_F_MULTI_QUEUE;
1518 #endif
1519
1520         /* call igb_desc_unused which always leaves
1521          * at least 1 descriptor unused to make sure
1522          * next_to_use != next_to_clean */
1523         for (i = 0; i < adapter->num_rx_queues; i++) {
1524                 struct igb_ring *ring = adapter->rx_ring[i];
1525                 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
1526         }
1527 }
1528
1529 /**
1530  * igb_power_up_link - Power up the phy/serdes link
1531  * @adapter: address of board private structure
1532  **/
1533 void igb_power_up_link(struct igb_adapter *adapter)
1534 {
1535         e1000_phy_hw_reset(&adapter->hw);
1536
1537         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1538                 e1000_power_up_phy(&adapter->hw);
1539         else
1540                 e1000_power_up_fiber_serdes_link(&adapter->hw);
1541 }
1542
1543 /**
1544  * igb_power_down_link - Power down the phy/serdes link
1545  * @adapter: address of board private structure
1546  */
1547 static void igb_power_down_link(struct igb_adapter *adapter)
1548 {
1549         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1550                 e1000_power_down_phy(&adapter->hw);
1551         else
1552                 e1000_shutdown_fiber_serdes_link(&adapter->hw);
1553 }
1554
1555 /* Detect and switch function for Media Auto Sense */
1556 static void igb_check_swap_media(struct igb_adapter *adapter)
1557 {
1558         struct e1000_hw *hw = &adapter->hw;
1559         u32 ctrl_ext, connsw;
1560         bool swap_now = false;
1561         bool link;
1562
1563         ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1564         connsw = E1000_READ_REG(hw, E1000_CONNSW);
1565         link = igb_has_link(adapter);
1566         (void) link;
1567
1568         /* need to live swap if current media is copper and we have fiber/serdes
1569          * to go to.
1570          */
1571
1572         if ((hw->phy.media_type == e1000_media_type_copper) &&
1573             (!(connsw & E1000_CONNSW_AUTOSENSE_EN))) {
1574                 swap_now = true;
1575         } else if (!(connsw & E1000_CONNSW_SERDESD)) {
1576                 /* copper signal takes time to appear */
1577                 if (adapter->copper_tries < 2) {
1578                         adapter->copper_tries++;
1579                         connsw |= E1000_CONNSW_AUTOSENSE_CONF;
1580                         E1000_WRITE_REG(hw, E1000_CONNSW, connsw);
1581                         return;
1582                 } else {
1583                         adapter->copper_tries = 0;
1584                         if ((connsw & E1000_CONNSW_PHYSD) &&
1585                             (!(connsw & E1000_CONNSW_PHY_PDN))) {
1586                                 swap_now = true;
1587                                 connsw &= ~E1000_CONNSW_AUTOSENSE_CONF;
1588                                 E1000_WRITE_REG(hw, E1000_CONNSW, connsw);
1589                         }
1590                 }
1591         }
1592
1593         if (swap_now) {
1594                 switch (hw->phy.media_type) {
1595                 case e1000_media_type_copper:
1596                         dev_info(pci_dev_to_dev(adapter->pdev),
1597                                  "%s:MAS: changing media to fiber/serdes\n",
1598                         adapter->netdev->name);
1599                         ctrl_ext |=
1600                                 E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
1601                         adapter->flags |= IGB_FLAG_MEDIA_RESET;
1602                         adapter->copper_tries = 0;
1603                         break;
1604                 case e1000_media_type_internal_serdes:
1605                 case e1000_media_type_fiber:
1606                         dev_info(pci_dev_to_dev(adapter->pdev),
1607                                  "%s:MAS: changing media to copper\n",
1608                                  adapter->netdev->name);
1609                         ctrl_ext &=
1610                                 ~E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
1611                         adapter->flags |= IGB_FLAG_MEDIA_RESET;
1612                         break;
1613                 default:
1614                         /* shouldn't get here during regular operation */
1615                         dev_err(pci_dev_to_dev(adapter->pdev),
1616                                 "%s:AMS: Invalid media type found, returning\n",
1617                                 adapter->netdev->name);
1618                         break;
1619                 }
1620                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext);
1621         }
1622 }
1623
1624 #ifdef HAVE_I2C_SUPPORT
1625 /*  igb_get_i2c_data - Reads the I2C SDA data bit
1626  *  @hw: pointer to hardware structure
1627  *  @i2cctl: Current value of I2CCTL register
1628  *
1629  *  Returns the I2C data bit value
1630  */
1631 static int igb_get_i2c_data(void *data)
1632 {
1633         struct igb_adapter *adapter = data;
1634         struct e1000_hw *hw = &adapter->hw;
1635         s32 i2cctl = E1000_READ_REG(hw, E1000_I2CPARAMS);
1636
1637         return (i2cctl & E1000_I2C_DATA_IN) != 0;
1638 }
1639
1640 /* igb_set_i2c_data - Sets the I2C data bit
1641  *  @data: pointer to hardware structure
1642  *  @state: I2C data value (0 or 1) to set
1643  *
1644  *  Sets the I2C data bit
1645  */
1646 static void igb_set_i2c_data(void *data, int state)
1647 {
1648         struct igb_adapter *adapter = data;
1649         struct e1000_hw *hw = &adapter->hw;
1650         s32 i2cctl = E1000_READ_REG(hw, E1000_I2CPARAMS);
1651
1652         if (state)
1653                 i2cctl |= E1000_I2C_DATA_OUT;
1654         else
1655                 i2cctl &= ~E1000_I2C_DATA_OUT;
1656
1657         i2cctl &= ~E1000_I2C_DATA_OE_N;
1658         i2cctl |= E1000_I2C_CLK_OE_N;
1659
1660         E1000_WRITE_REG(hw, E1000_I2CPARAMS, i2cctl);
1661         E1000_WRITE_FLUSH(hw);
1662
1663 }
1664
1665 /* igb_set_i2c_clk - Sets the I2C SCL clock
1666  *  @data: pointer to hardware structure
1667  *  @state: state to set clock
1668  *
1669  *  Sets the I2C clock line to state
1670  */
1671 static void igb_set_i2c_clk(void *data, int state)
1672 {
1673         struct igb_adapter *adapter = data;
1674         struct e1000_hw *hw = &adapter->hw;
1675         s32 i2cctl = E1000_READ_REG(hw, E1000_I2CPARAMS);
1676
1677         if (state) {
1678                 i2cctl |= E1000_I2C_CLK_OUT;
1679                 i2cctl &= ~E1000_I2C_CLK_OE_N;
1680         } else {
1681                 i2cctl &= ~E1000_I2C_CLK_OUT;
1682                 i2cctl &= ~E1000_I2C_CLK_OE_N;
1683         }
1684         E1000_WRITE_REG(hw, E1000_I2CPARAMS, i2cctl);
1685         E1000_WRITE_FLUSH(hw);
1686 }
1687
1688 /* igb_get_i2c_clk - Gets the I2C SCL clock state
1689  *  @data: pointer to hardware structure
1690  *
1691  *  Gets the I2C clock state
1692  */
1693 static int igb_get_i2c_clk(void *data)
1694 {
1695         struct igb_adapter *adapter = data;
1696         struct e1000_hw *hw = &adapter->hw;
1697         s32 i2cctl = E1000_READ_REG(hw, E1000_I2CPARAMS);
1698
1699         return (i2cctl & E1000_I2C_CLK_IN) != 0;
1700 }
1701
1702 static const struct i2c_algo_bit_data igb_i2c_algo = {
1703         .setsda         = igb_set_i2c_data,
1704         .setscl         = igb_set_i2c_clk,
1705         .getsda         = igb_get_i2c_data,
1706         .getscl         = igb_get_i2c_clk,
1707         .udelay         = 5,
1708         .timeout        = 20,
1709 };
1710
1711 /*  igb_init_i2c - Init I2C interface
1712  *  @adapter: pointer to adapter structure
1713  *
1714  */
1715 static s32 igb_init_i2c(struct igb_adapter *adapter)
1716 {
1717         s32 status = E1000_SUCCESS;
1718
1719         /* I2C interface supported on i350 devices */
1720         if (adapter->hw.mac.type != e1000_i350)
1721                 return E1000_SUCCESS;
1722
1723         /* Initialize the i2c bus which is controlled by the registers.
1724          * This bus will use the i2c_algo_bit structue that implements
1725          * the protocol through toggling of the 4 bits in the register.
1726          */
1727         adapter->i2c_adap.owner = THIS_MODULE;
1728         adapter->i2c_algo = igb_i2c_algo;
1729         adapter->i2c_algo.data = adapter;
1730         adapter->i2c_adap.algo_data = &adapter->i2c_algo;
1731         adapter->i2c_adap.dev.parent = &adapter->pdev->dev;
1732         strlcpy(adapter->i2c_adap.name, "igb BB",
1733                 sizeof(adapter->i2c_adap.name));
1734         status = i2c_bit_add_bus(&adapter->i2c_adap);
1735         return status;
1736 }
1737
1738 #endif /* HAVE_I2C_SUPPORT */
1739 /**
1740  * igb_up - Open the interface and prepare it to handle traffic
1741  * @adapter: board private structure
1742  **/
1743 int igb_up(struct igb_adapter *adapter)
1744 {
1745         struct e1000_hw *hw = &adapter->hw;
1746         int i;
1747
1748         /* hardware has been reset, we need to reload some things */
1749         igb_configure(adapter);
1750
1751         clear_bit(__IGB_DOWN, &adapter->state);
1752
1753         for (i = 0; i < adapter->num_q_vectors; i++)
1754                 napi_enable(&(adapter->q_vector[i]->napi));
1755
1756         if (adapter->msix_entries)
1757                 igb_configure_msix(adapter);
1758         else
1759                 igb_assign_vector(adapter->q_vector[0], 0);
1760
1761         igb_configure_lli(adapter);
1762
1763         /* Clear any pending interrupts. */
1764         E1000_READ_REG(hw, E1000_ICR);
1765         igb_irq_enable(adapter);
1766
1767         /* notify VFs that reset has been completed */
1768         if (adapter->vfs_allocated_count) {
1769                 u32 reg_data = E1000_READ_REG(hw, E1000_CTRL_EXT);
1770                 reg_data |= E1000_CTRL_EXT_PFRSTD;
1771                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg_data);
1772         }
1773
1774         netif_tx_start_all_queues(adapter->netdev);
1775
1776         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
1777                 schedule_work(&adapter->dma_err_task);
1778         /* start the watchdog. */
1779         hw->mac.get_link_status = 1;
1780         schedule_work(&adapter->watchdog_task);
1781
1782         if ((adapter->flags & IGB_FLAG_EEE) &&
1783             (!hw->dev_spec._82575.eee_disable))
1784                 adapter->eee_advert = MDIO_EEE_100TX | MDIO_EEE_1000T;
1785
1786         return 0;
1787 }
1788
1789 void igb_down(struct igb_adapter *adapter)
1790 {
1791         struct net_device *netdev = adapter->netdev;
1792         struct e1000_hw *hw = &adapter->hw;
1793         u32 tctl, rctl;
1794         int i;
1795
1796         /* signal that we're down so the interrupt handler does not
1797          * reschedule our watchdog timer */
1798         set_bit(__IGB_DOWN, &adapter->state);
1799
1800         /* disable receives in the hardware */
1801         rctl = E1000_READ_REG(hw, E1000_RCTL);
1802         E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
1803         /* flush and sleep below */
1804
1805         netif_tx_stop_all_queues(netdev);
1806
1807         /* disable transmits in the hardware */
1808         tctl = E1000_READ_REG(hw, E1000_TCTL);
1809         tctl &= ~E1000_TCTL_EN;
1810         E1000_WRITE_REG(hw, E1000_TCTL, tctl);
1811         /* flush both disables and wait for them to finish */
1812         E1000_WRITE_FLUSH(hw);
1813         usleep_range(10000, 20000);
1814
1815         for (i = 0; i < adapter->num_q_vectors; i++)
1816                 napi_disable(&(adapter->q_vector[i]->napi));
1817
1818         igb_irq_disable(adapter);
1819
1820         adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
1821
1822         del_timer_sync(&adapter->watchdog_timer);
1823         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
1824                 del_timer_sync(&adapter->dma_err_timer);
1825         del_timer_sync(&adapter->phy_info_timer);
1826
1827         netif_carrier_off(netdev);
1828
1829         /* record the stats before reset*/
1830         igb_update_stats(adapter);
1831
1832         adapter->link_speed = 0;
1833         adapter->link_duplex = 0;
1834
1835 #ifdef HAVE_PCI_ERS
1836         if (!pci_channel_offline(adapter->pdev))
1837                 igb_reset(adapter);
1838 #else
1839         igb_reset(adapter);
1840 #endif
1841         igb_clean_all_tx_rings(adapter);
1842         igb_clean_all_rx_rings(adapter);
1843 #ifdef IGB_DCA
1844         /* since we reset the hardware DCA settings were cleared */
1845         igb_setup_dca(adapter);
1846 #endif
1847 }
1848
1849 void igb_reinit_locked(struct igb_adapter *adapter)
1850 {
1851         WARN_ON(in_interrupt());
1852         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1853                 usleep_range(1000, 2000);
1854         igb_down(adapter);
1855         igb_up(adapter);
1856         clear_bit(__IGB_RESETTING, &adapter->state);
1857 }
1858
1859 /**
1860  * igb_enable_mas - Media Autosense re-enable after swap
1861  *
1862  * @adapter: adapter struct
1863  **/
1864 static s32  igb_enable_mas(struct igb_adapter *adapter)
1865 {
1866         struct e1000_hw *hw = &adapter->hw;
1867         u32 connsw;
1868         s32 ret_val = E1000_SUCCESS;
1869
1870         connsw = E1000_READ_REG(hw, E1000_CONNSW);
1871         if (hw->phy.media_type == e1000_media_type_copper) {
1872                 /* configure for SerDes media detect */
1873                 if (!(connsw & E1000_CONNSW_SERDESD)) {
1874                         connsw |= E1000_CONNSW_ENRGSRC;
1875                         connsw |= E1000_CONNSW_AUTOSENSE_EN;
1876                         E1000_WRITE_REG(hw, E1000_CONNSW, connsw);
1877                         E1000_WRITE_FLUSH(hw);
1878                 } else if (connsw & E1000_CONNSW_SERDESD) {
1879                         /* already SerDes, no need to enable anything */
1880                         return ret_val;
1881                 } else {
1882                         dev_info(pci_dev_to_dev(adapter->pdev),
1883                         "%s:MAS: Unable to configure feature, disabling..\n",
1884                         adapter->netdev->name);
1885                         adapter->flags &= ~IGB_FLAG_MAS_ENABLE;
1886                 }
1887         }
1888         return ret_val;
1889 }
1890
1891 void igb_reset(struct igb_adapter *adapter)
1892 {
1893         struct pci_dev *pdev = adapter->pdev;
1894         struct e1000_hw *hw = &adapter->hw;
1895         struct e1000_mac_info *mac = &hw->mac;
1896         struct e1000_fc_info *fc = &hw->fc;
1897         u32 pba = 0, tx_space, min_tx_space, min_rx_space, hwm;
1898
1899         /* Repartition Pba for greater than 9k mtu
1900          * To take effect CTRL.RST is required.
1901          */
1902         switch (mac->type) {
1903         case e1000_i350:
1904         case e1000_82580:
1905         case e1000_i354:
1906                 pba = E1000_READ_REG(hw, E1000_RXPBS);
1907                 pba = e1000_rxpbs_adjust_82580(pba);
1908                 break;
1909         case e1000_82576:
1910                 pba = E1000_READ_REG(hw, E1000_RXPBS);
1911                 pba &= E1000_RXPBS_SIZE_MASK_82576;
1912                 break;
1913         case e1000_82575:
1914         case e1000_i210:
1915         case e1000_i211:
1916         default:
1917                 pba = E1000_PBA_34K;
1918                 break;
1919         }
1920
1921         if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1922             (mac->type < e1000_82576)) {
1923                 /* adjust PBA for jumbo frames */
1924                 E1000_WRITE_REG(hw, E1000_PBA, pba);
1925
1926                 /* To maintain wire speed transmits, the Tx FIFO should be
1927                  * large enough to accommodate two full transmit packets,
1928                  * rounded up to the next 1KB and expressed in KB.  Likewise,
1929                  * the Rx FIFO should be large enough to accommodate at least
1930                  * one full receive packet and is similarly rounded up and
1931                  * expressed in KB. */
1932                 pba = E1000_READ_REG(hw, E1000_PBA);
1933                 /* upper 16 bits has Tx packet buffer allocation size in KB */
1934                 tx_space = pba >> 16;
1935                 /* lower 16 bits has Rx packet buffer allocation size in KB */
1936                 pba &= 0xffff;
1937                 /* the tx fifo also stores 16 bytes of information about the tx
1938                  * but don't include ethernet FCS because hardware appends it */
1939                 min_tx_space = (adapter->max_frame_size +
1940                                 sizeof(union e1000_adv_tx_desc) -
1941                                 ETH_FCS_LEN) * 2;
1942                 min_tx_space = ALIGN(min_tx_space, 1024);
1943                 min_tx_space >>= 10;
1944                 /* software strips receive CRC, so leave room for it */
1945                 min_rx_space = adapter->max_frame_size;
1946                 min_rx_space = ALIGN(min_rx_space, 1024);
1947                 min_rx_space >>= 10;
1948
1949                 /* If current Tx allocation is less than the min Tx FIFO size,
1950                  * and the min Tx FIFO size is less than the current Rx FIFO
1951                  * allocation, take space away from current Rx allocation */
1952                 if (tx_space < min_tx_space &&
1953                     ((min_tx_space - tx_space) < pba)) {
1954                         pba = pba - (min_tx_space - tx_space);
1955
1956                         /* if short on rx space, rx wins and must trump tx
1957                          * adjustment */
1958                         if (pba < min_rx_space)
1959                                 pba = min_rx_space;
1960                 }
1961                 E1000_WRITE_REG(hw, E1000_PBA, pba);
1962         }
1963
1964         /* flow control settings */
1965         /* The high water mark must be low enough to fit one full frame
1966          * (or the size used for early receive) above it in the Rx FIFO.
1967          * Set it to the lower of:
1968          * - 90% of the Rx FIFO size, or
1969          * - the full Rx FIFO size minus one full frame */
1970         hwm = min(((pba << 10) * 9 / 10),
1971                         ((pba << 10) - 2 * adapter->max_frame_size));
1972
1973         fc->high_water = hwm & 0xFFFFFFF0;      /* 16-byte granularity */
1974         fc->low_water = fc->high_water - 16;
1975         fc->pause_time = 0xFFFF;
1976         fc->send_xon = 1;
1977         fc->current_mode = fc->requested_mode;
1978
1979         /* disable receive for all VFs and wait one second */
1980         if (adapter->vfs_allocated_count) {
1981                 int i;
1982                 /*
1983                  * Clear all flags except indication that the PF has set
1984                  * the VF MAC addresses administratively
1985                  */
1986                 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1987                         adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
1988
1989                 /* ping all the active vfs to let them know we are going down */
1990                 igb_ping_all_vfs(adapter);
1991
1992                 /* disable transmits and receives */
1993                 E1000_WRITE_REG(hw, E1000_VFRE, 0);
1994                 E1000_WRITE_REG(hw, E1000_VFTE, 0);
1995         }
1996
1997         /* Allow time for pending master requests to run */
1998         e1000_reset_hw(hw);
1999         E1000_WRITE_REG(hw, E1000_WUC, 0);
2000
2001         if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
2002                 e1000_setup_init_funcs(hw, TRUE);
2003                 igb_check_options(adapter);
2004                 e1000_get_bus_info(hw);
2005                 adapter->flags &= ~IGB_FLAG_MEDIA_RESET;
2006         }
2007         if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
2008                 if (igb_enable_mas(adapter))
2009                         dev_err(pci_dev_to_dev(pdev),
2010                                 "Error enabling Media Auto Sense\n");
2011         }
2012         if (e1000_init_hw(hw))
2013                 dev_err(pci_dev_to_dev(pdev), "Hardware Error\n");
2014
2015         /*
2016          * Flow control settings reset on hardware reset, so guarantee flow
2017          * control is off when forcing speed.
2018          */
2019         if (!hw->mac.autoneg)
2020                 e1000_force_mac_fc(hw);
2021
2022         igb_init_dmac(adapter, pba);
2023         /* Re-initialize the thermal sensor on i350 devices. */
2024         if (mac->type == e1000_i350 && hw->bus.func == 0) {
2025                 /*
2026                  * If present, re-initialize the external thermal sensor
2027                  * interface.
2028                  */
2029                 if (adapter->ets)
2030                         e1000_set_i2c_bb(hw);
2031                 e1000_init_thermal_sensor_thresh(hw);
2032         }
2033
2034         /*Re-establish EEE setting */
2035         if (hw->phy.media_type == e1000_media_type_copper) {
2036                 switch (mac->type) {
2037                 case e1000_i350:
2038                 case e1000_i210:
2039                 case e1000_i211:
2040                         e1000_set_eee_i350(hw);
2041                         break;
2042                 case e1000_i354:
2043                         e1000_set_eee_i354(hw);
2044                         break;
2045                 default:
2046                         break;
2047                 }
2048         }
2049
2050         if (!netif_running(adapter->netdev))
2051                 igb_power_down_link(adapter);
2052
2053         igb_update_mng_vlan(adapter);
2054
2055         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
2056         E1000_WRITE_REG(hw, E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
2057
2058
2059 #ifdef HAVE_PTP_1588_CLOCK
2060         /* Re-enable PTP, where applicable. */
2061         igb_ptp_reset(adapter);
2062 #endif /* HAVE_PTP_1588_CLOCK */
2063
2064         e1000_get_phy_info(hw);
2065
2066         adapter->devrc++;
2067 }
2068
2069 #ifdef HAVE_NDO_SET_FEATURES
2070 static kni_netdev_features_t igb_fix_features(struct net_device *netdev,
2071                                               kni_netdev_features_t features)
2072 {
2073         /*
2074          * Since there is no support for separate tx vlan accel
2075          * enabled make sure tx flag is cleared if rx is.
2076          */
2077 #ifdef NETIF_F_HW_VLAN_CTAG_RX
2078         if (!(features & NETIF_F_HW_VLAN_CTAG_RX))
2079                 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
2080 #else
2081         if (!(features & NETIF_F_HW_VLAN_RX))
2082                 features &= ~NETIF_F_HW_VLAN_TX;
2083 #endif
2084
2085         /* If Rx checksum is disabled, then LRO should also be disabled */
2086         if (!(features & NETIF_F_RXCSUM))
2087                 features &= ~NETIF_F_LRO;
2088
2089         return features;
2090 }
2091
2092 static int igb_set_features(struct net_device *netdev,
2093                             kni_netdev_features_t features)
2094 {
2095         u32 changed = netdev->features ^ features;
2096
2097 #ifdef NETIF_F_HW_VLAN_CTAG_RX
2098         if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2099 #else
2100         if (changed & NETIF_F_HW_VLAN_RX)
2101 #endif
2102                 igb_vlan_mode(netdev, features);
2103
2104         return 0;
2105 }
2106
2107 #ifdef NTF_SELF
2108 #ifdef USE_CONST_DEV_UC_CHAR
2109 static int igb_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
2110                            struct net_device *dev,
2111                            const unsigned char *addr,
2112 #ifdef HAVE_NDO_FDB_ADD_VID
2113                            u16 vid,
2114 #endif
2115 #ifdef HAVE_NDO_FDB_ADD_EXTACK
2116                            u16 flags,
2117                            struct netlink_ext_ack *extack)
2118 #else
2119                            u16 flags)
2120 #endif
2121
2122 #else
2123 static int igb_ndo_fdb_add(struct ndmsg *ndm,
2124                            struct net_device *dev,
2125                            unsigned char *addr,
2126                            u16 flags)
2127 #endif
2128 {
2129         struct igb_adapter *adapter = netdev_priv(dev);
2130         struct e1000_hw *hw = &adapter->hw;
2131         int err;
2132
2133         if (!(adapter->vfs_allocated_count))
2134                 return -EOPNOTSUPP;
2135
2136         /* Hardware does not support aging addresses so if a
2137          * ndm_state is given only allow permanent addresses
2138          */
2139         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
2140                 pr_info("%s: FDB only supports static addresses\n",
2141                         igb_driver_name);
2142                 return -EINVAL;
2143         }
2144
2145         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) {
2146                 u32 rar_uc_entries = hw->mac.rar_entry_count -
2147                                         (adapter->vfs_allocated_count + 1);
2148
2149                 if (netdev_uc_count(dev) < rar_uc_entries)
2150                         err = dev_uc_add_excl(dev, addr);
2151                 else
2152                         err = -ENOMEM;
2153         } else if (is_multicast_ether_addr(addr)) {
2154                 err = dev_mc_add_excl(dev, addr);
2155         } else {
2156                 err = -EINVAL;
2157         }
2158
2159         /* Only return duplicate errors if NLM_F_EXCL is set */
2160         if (err == -EEXIST && !(flags & NLM_F_EXCL))
2161                 err = 0;
2162
2163         return err;
2164 }
2165
2166 #ifndef USE_DEFAULT_FDB_DEL_DUMP
2167 #ifdef USE_CONST_DEV_UC_CHAR
2168 static int igb_ndo_fdb_del(struct ndmsg *ndm,
2169                            struct net_device *dev,
2170                            const unsigned char *addr)
2171 #else
2172 static int igb_ndo_fdb_del(struct ndmsg *ndm,
2173                            struct net_device *dev,
2174                            unsigned char *addr)
2175 #endif
2176 {
2177         struct igb_adapter *adapter = netdev_priv(dev);
2178         int err = -EOPNOTSUPP;
2179
2180         if (ndm->ndm_state & NUD_PERMANENT) {
2181                 pr_info("%s: FDB only supports static addresses\n",
2182                         igb_driver_name);
2183                 return -EINVAL;
2184         }
2185
2186         if (adapter->vfs_allocated_count) {
2187                 if (is_unicast_ether_addr(addr))
2188                         err = dev_uc_del(dev, addr);
2189                 else if (is_multicast_ether_addr(addr))
2190                         err = dev_mc_del(dev, addr);
2191                 else
2192                         err = -EINVAL;
2193         }
2194
2195         return err;
2196 }
2197
2198 static int igb_ndo_fdb_dump(struct sk_buff *skb,
2199                             struct netlink_callback *cb,
2200                             struct net_device *dev,
2201                             int idx)
2202 {
2203         struct igb_adapter *adapter = netdev_priv(dev);
2204
2205         if (adapter->vfs_allocated_count)
2206                 idx = ndo_dflt_fdb_dump(skb, cb, dev, idx);
2207
2208         return idx;
2209 }
2210 #endif /* USE_DEFAULT_FDB_DEL_DUMP */
2211
2212 #ifdef HAVE_BRIDGE_ATTRIBS
2213 #ifdef HAVE_NDO_BRIDGE_SET_DEL_LINK_FLAGS
2214 static int igb_ndo_bridge_setlink(struct net_device *dev,
2215                                   struct nlmsghdr *nlh,
2216 #ifdef HAVE_NDO_BRIDGE_SETLINK_EXTACK
2217                                   u16 flags, struct netlink_ext_ack *extack)
2218 #else
2219                                   u16 flags)
2220 #endif
2221
2222 #else
2223 static int igb_ndo_bridge_setlink(struct net_device *dev,
2224                                   struct nlmsghdr *nlh)
2225 #endif /* HAVE_NDO_BRIDGE_SET_DEL_LINK_FLAGS */
2226 {
2227         struct igb_adapter *adapter = netdev_priv(dev);
2228         struct e1000_hw *hw = &adapter->hw;
2229         struct nlattr *attr, *br_spec;
2230         int rem;
2231
2232         if (!(adapter->vfs_allocated_count))
2233                 return -EOPNOTSUPP;
2234
2235         switch (adapter->hw.mac.type) {
2236         case e1000_82576:
2237         case e1000_i350:
2238         case e1000_i354:
2239                 break;
2240         default:
2241                 return -EOPNOTSUPP;
2242         }
2243
2244         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
2245
2246         nla_for_each_nested(attr, br_spec, rem) {
2247                 __u16 mode;
2248
2249                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
2250                         continue;
2251
2252                 mode = nla_get_u16(attr);
2253                 if (mode == BRIDGE_MODE_VEPA) {
2254                         e1000_vmdq_set_loopback_pf(hw, 0);
2255                         adapter->flags &= ~IGB_FLAG_LOOPBACK_ENABLE;
2256                 } else if (mode == BRIDGE_MODE_VEB) {
2257                         e1000_vmdq_set_loopback_pf(hw, 1);
2258                         adapter->flags |= IGB_FLAG_LOOPBACK_ENABLE;
2259                 } else
2260                         return -EINVAL;
2261
2262                 netdev_info(adapter->netdev, "enabling bridge mode: %s\n",
2263                             mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
2264         }
2265
2266         return 0;
2267 }
2268
2269 #ifdef HAVE_BRIDGE_FILTER
2270 #ifdef HAVE_NDO_BRIDGE_GETLINK_NLFLAGS
2271 static int igb_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
2272                                   struct net_device *dev, u32 filter_mask,
2273                                   int nlflags)
2274 #else
2275 static int igb_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
2276                                   struct net_device *dev, u32 filter_mask)
2277 #endif /* HAVE_NDO_BRIDGE_GETLINK_NLFLAGS */
2278 #else
2279 static int igb_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
2280                                   struct net_device *dev)
2281 #endif
2282 {
2283         struct igb_adapter *adapter = netdev_priv(dev);
2284         u16 mode;
2285
2286         if (!(adapter->vfs_allocated_count))
2287                 return -EOPNOTSUPP;
2288
2289         if (adapter->flags & IGB_FLAG_LOOPBACK_ENABLE)
2290                 mode = BRIDGE_MODE_VEB;
2291         else
2292                 mode = BRIDGE_MODE_VEPA;
2293
2294 #ifdef HAVE_NDO_DFLT_BRIDGE_ADD_MASK
2295 #ifdef HAVE_NDO_BRIDGE_GETLINK_NLFLAGS
2296 #ifdef HAVE_NDO_BRIDGE_GETLINK_FILTER_MASK_VLAN_FILL
2297         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, mode, 0, 0,
2298                                 nlflags, filter_mask, NULL);
2299 #else
2300         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, mode, 0, 0, nlflags);
2301 #endif /* HAVE_NDO_BRIDGE_GETLINK_FILTER_MASK_VLAN_FILL */
2302 #else
2303         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, mode, 0, 0);
2304 #endif /* HAVE_NDO_BRIDGE_GETLINK_NLFLAGS */
2305 #else
2306         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, mode);
2307 #endif /* HAVE_NDO_DFLT_BRIDGE_ADD_MASK */
2308 }
2309 #endif /* HAVE_BRIDGE_ATTRIBS */
2310 #endif /* NTF_SELF */
2311
2312 #endif /* HAVE_NDO_SET_FEATURES */
2313 #ifdef HAVE_NET_DEVICE_OPS
2314 static const struct net_device_ops igb_netdev_ops = {
2315         .ndo_open               = igb_open,
2316         .ndo_stop               = igb_close,
2317         .ndo_start_xmit         = igb_xmit_frame,
2318         .ndo_get_stats          = igb_get_stats,
2319         .ndo_set_rx_mode        = igb_set_rx_mode,
2320         .ndo_set_mac_address    = igb_set_mac,
2321         .ndo_change_mtu         = igb_change_mtu,
2322         .ndo_do_ioctl           = igb_ioctl,
2323         .ndo_tx_timeout         = igb_tx_timeout,
2324         .ndo_validate_addr      = eth_validate_addr,
2325         .ndo_vlan_rx_add_vid    = igb_vlan_rx_add_vid,
2326         .ndo_vlan_rx_kill_vid   = igb_vlan_rx_kill_vid,
2327 #ifdef IFLA_VF_MAX
2328         .ndo_set_vf_mac         = igb_ndo_set_vf_mac,
2329         .ndo_set_vf_vlan        = igb_ndo_set_vf_vlan,
2330 #ifdef HAVE_VF_MIN_MAX_TXRATE
2331         .ndo_set_vf_rate        = igb_ndo_set_vf_bw,
2332 #else /* HAVE_VF_MIN_MAX_TXRATE */
2333         .ndo_set_vf_tx_rate     = igb_ndo_set_vf_bw,
2334 #endif /* HAVE_VF_MIN_MAX_TXRATE */
2335         .ndo_get_vf_config      = igb_ndo_get_vf_config,
2336 #ifdef HAVE_VF_SPOOFCHK_CONFIGURE
2337         .ndo_set_vf_spoofchk    = igb_ndo_set_vf_spoofchk,
2338 #endif /* HAVE_VF_SPOOFCHK_CONFIGURE */
2339 #endif /* IFLA_VF_MAX */
2340 #ifdef CONFIG_NET_POLL_CONTROLLER
2341         .ndo_poll_controller    = igb_netpoll,
2342 #endif
2343 #ifdef HAVE_NDO_SET_FEATURES
2344         .ndo_fix_features       = igb_fix_features,
2345         .ndo_set_features       = igb_set_features,
2346 #endif
2347 #ifdef HAVE_VLAN_RX_REGISTER
2348         .ndo_vlan_rx_register   = igb_vlan_mode,
2349 #endif
2350 #ifndef HAVE_RHEL6_NETDEV_OPS_EXT_FDB
2351 #ifdef NTF_SELF
2352         .ndo_fdb_add            = igb_ndo_fdb_add,
2353 #ifndef USE_DEFAULT_FDB_DEL_DUMP
2354         .ndo_fdb_del            = igb_ndo_fdb_del,
2355         .ndo_fdb_dump           = igb_ndo_fdb_dump,
2356 #endif
2357 #endif /* ! HAVE_RHEL6_NETDEV_OPS_EXT_FDB */
2358 #ifdef HAVE_BRIDGE_ATTRIBS
2359         .ndo_bridge_setlink     = igb_ndo_bridge_setlink,
2360         .ndo_bridge_getlink     = igb_ndo_bridge_getlink,
2361 #endif /* HAVE_BRIDGE_ATTRIBS */
2362 #endif
2363 };
2364
2365 #ifdef CONFIG_IGB_VMDQ_NETDEV
2366 static const struct net_device_ops igb_vmdq_ops = {
2367         .ndo_open               = &igb_vmdq_open,
2368         .ndo_stop               = &igb_vmdq_close,
2369         .ndo_start_xmit         = &igb_vmdq_xmit_frame,
2370         .ndo_get_stats          = &igb_vmdq_get_stats,
2371         .ndo_set_rx_mode        = &igb_vmdq_set_rx_mode,
2372         .ndo_validate_addr      = eth_validate_addr,
2373         .ndo_set_mac_address    = &igb_vmdq_set_mac,
2374         .ndo_change_mtu         = &igb_vmdq_change_mtu,
2375         .ndo_tx_timeout         = &igb_vmdq_tx_timeout,
2376         .ndo_vlan_rx_register   = &igb_vmdq_vlan_rx_register,
2377         .ndo_vlan_rx_add_vid    = &igb_vmdq_vlan_rx_add_vid,
2378         .ndo_vlan_rx_kill_vid   = &igb_vmdq_vlan_rx_kill_vid,
2379 };
2380
2381 #endif /* CONFIG_IGB_VMDQ_NETDEV */
2382 #endif /* HAVE_NET_DEVICE_OPS */
2383 #ifdef CONFIG_IGB_VMDQ_NETDEV
2384 void igb_assign_vmdq_netdev_ops(struct net_device *vnetdev)
2385 {
2386 #ifdef HAVE_NET_DEVICE_OPS
2387         vnetdev->netdev_ops = &igb_vmdq_ops;
2388 #else
2389         dev->open = &igb_vmdq_open;
2390         dev->stop = &igb_vmdq_close;
2391         dev->hard_start_xmit = &igb_vmdq_xmit_frame;
2392         dev->get_stats = &igb_vmdq_get_stats;
2393 #ifdef HAVE_SET_RX_MODE
2394         dev->set_rx_mode = &igb_vmdq_set_rx_mode;
2395 #endif
2396         dev->set_multicast_list = &igb_vmdq_set_rx_mode;
2397         dev->set_mac_address = &igb_vmdq_set_mac;
2398         dev->change_mtu = &igb_vmdq_change_mtu;
2399 #ifdef HAVE_TX_TIMEOUT
2400         dev->tx_timeout = &igb_vmdq_tx_timeout;
2401 #endif
2402 #if defined(NETIF_F_HW_VLAN_TX) || defined(NETIF_F_HW_VLAN_CTAG_TX)
2403         dev->vlan_rx_register = &igb_vmdq_vlan_rx_register;
2404         dev->vlan_rx_add_vid = &igb_vmdq_vlan_rx_add_vid;
2405         dev->vlan_rx_kill_vid = &igb_vmdq_vlan_rx_kill_vid;
2406 #endif
2407 #endif
2408         igb_vmdq_set_ethtool_ops(vnetdev);
2409         vnetdev->watchdog_timeo = 5 * HZ;
2410
2411 }
2412
2413 int igb_init_vmdq_netdevs(struct igb_adapter *adapter)
2414 {
2415         int pool, err = 0, base_queue;
2416         struct net_device *vnetdev;
2417         struct igb_vmdq_adapter *vmdq_adapter;
2418
2419         for (pool = 1; pool < adapter->vmdq_pools; pool++) {
2420                 int qpp = (!adapter->rss_queues ? 1 : adapter->rss_queues);
2421                 base_queue = pool * qpp;
2422                 vnetdev = alloc_etherdev(sizeof(struct igb_vmdq_adapter));
2423                 if (!vnetdev) {
2424                         err = -ENOMEM;
2425                         break;
2426                 }
2427                 vmdq_adapter = netdev_priv(vnetdev);
2428                 vmdq_adapter->vnetdev = vnetdev;
2429                 vmdq_adapter->real_adapter = adapter;
2430                 vmdq_adapter->rx_ring = adapter->rx_ring[base_queue];
2431                 vmdq_adapter->tx_ring = adapter->tx_ring[base_queue];
2432                 igb_assign_vmdq_netdev_ops(vnetdev);
2433                 snprintf(vnetdev->name, IFNAMSIZ, "%sv%d",
2434                          adapter->netdev->name, pool);
2435                 vnetdev->features = adapter->netdev->features;
2436 #ifdef HAVE_NETDEV_VLAN_FEATURES
2437                 vnetdev->vlan_features = adapter->netdev->vlan_features;
2438 #endif
2439                 adapter->vmdq_netdev[pool-1] = vnetdev;
2440                 err = register_netdev(vnetdev);
2441                 if (err)
2442                         break;
2443         }
2444         return err;
2445 }
2446
2447 int igb_remove_vmdq_netdevs(struct igb_adapter *adapter)
2448 {
2449         int pool, err = 0;
2450
2451         for (pool = 1; pool < adapter->vmdq_pools; pool++) {
2452                 unregister_netdev(adapter->vmdq_netdev[pool-1]);
2453                 free_netdev(adapter->vmdq_netdev[pool-1]);
2454                 adapter->vmdq_netdev[pool-1] = NULL;
2455         }
2456         return err;
2457 }
2458 #endif /* CONFIG_IGB_VMDQ_NETDEV */
2459
2460 /**
2461  * igb_set_fw_version - Configure version string for ethtool
2462  * @adapter: adapter struct
2463  *
2464  **/
2465 static void igb_set_fw_version(struct igb_adapter *adapter)
2466 {
2467         struct e1000_hw *hw = &adapter->hw;
2468         struct e1000_fw_version fw;
2469
2470         e1000_get_fw_version(hw, &fw);
2471
2472         switch (hw->mac.type) {
2473         case e1000_i210:
2474         case e1000_i211:
2475                 if (!(e1000_get_flash_presence_i210(hw))) {
2476                         snprintf(adapter->fw_version,
2477                             sizeof(adapter->fw_version),
2478                             "%2d.%2d-%d",
2479                             fw.invm_major, fw.invm_minor, fw.invm_img_type);
2480                         break;
2481                 }
2482                 /* fall through */
2483         default:
2484                 /* if option rom is valid, display its version too*/
2485                 if (fw.or_valid) {
2486                         snprintf(adapter->fw_version,
2487                             sizeof(adapter->fw_version),
2488                             "%d.%d, 0x%08x, %d.%d.%d",
2489                             fw.eep_major, fw.eep_minor, fw.etrack_id,
2490                             fw.or_major, fw.or_build, fw.or_patch);
2491                 /* no option rom */
2492                 } else {
2493                         if (fw.etrack_id != 0X0000) {
2494                         snprintf(adapter->fw_version,
2495                             sizeof(adapter->fw_version),
2496                             "%d.%d, 0x%08x",
2497                             fw.eep_major, fw.eep_minor, fw.etrack_id);
2498                         } else {
2499                         snprintf(adapter->fw_version,
2500                             sizeof(adapter->fw_version),
2501                             "%d.%d.%d",
2502                             fw.eep_major, fw.eep_minor, fw.eep_build);
2503                         }
2504                 }
2505                 break;
2506         }
2507
2508         return;
2509 }
2510
2511 /**
2512  * igb_init_mas - init Media Autosense feature if enabled in the NVM
2513  *
2514  * @adapter: adapter struct
2515  **/
2516 static void igb_init_mas(struct igb_adapter *adapter)
2517 {
2518         struct e1000_hw *hw = &adapter->hw;
2519         u16 eeprom_data;
2520
2521         e1000_read_nvm(hw, NVM_COMPAT, 1, &eeprom_data);
2522         switch (hw->bus.func) {
2523         case E1000_FUNC_0:
2524                 if (eeprom_data & IGB_MAS_ENABLE_0)
2525                         adapter->flags |= IGB_FLAG_MAS_ENABLE;
2526                 break;
2527         case E1000_FUNC_1:
2528                 if (eeprom_data & IGB_MAS_ENABLE_1)
2529                         adapter->flags |= IGB_FLAG_MAS_ENABLE;
2530                 break;
2531         case E1000_FUNC_2:
2532                 if (eeprom_data & IGB_MAS_ENABLE_2)
2533                         adapter->flags |= IGB_FLAG_MAS_ENABLE;
2534                 break;
2535         case E1000_FUNC_3:
2536                 if (eeprom_data & IGB_MAS_ENABLE_3)
2537                         adapter->flags |= IGB_FLAG_MAS_ENABLE;
2538                 break;
2539         default:
2540                 /* Shouldn't get here */
2541                 dev_err(pci_dev_to_dev(adapter->pdev),
2542                         "%s:AMS: Invalid port configuration, returning\n",
2543                         adapter->netdev->name);
2544                 break;
2545         }
2546 }
2547
2548 /**
2549  * igb_probe - Device Initialization Routine
2550  * @pdev: PCI device information struct
2551  * @ent: entry in igb_pci_tbl
2552  *
2553  * Returns 0 on success, negative on failure
2554  *
2555  * igb_probe initializes an adapter identified by a pci_dev structure.
2556  * The OS initialization, configuring of the adapter private structure,
2557  * and a hardware reset occur.
2558  **/
2559 static int __devinit igb_probe(struct pci_dev *pdev,
2560                                const struct pci_device_id *ent)
2561 {
2562         struct net_device *netdev;
2563         struct igb_adapter *adapter;
2564         struct e1000_hw *hw;
2565         u16 eeprom_data = 0;
2566         u8 pba_str[E1000_PBANUM_LENGTH];
2567         s32 ret_val;
2568         static int global_quad_port_a; /* global quad port a indication */
2569         int i, err, pci_using_dac;
2570         static int cards_found;
2571
2572         err = pci_enable_device_mem(pdev);
2573         if (err)
2574                 return err;
2575
2576         pci_using_dac = 0;
2577         err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
2578         if (!err) {
2579                 err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
2580                 if (!err)
2581                         pci_using_dac = 1;
2582         } else {
2583                 err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
2584                 if (err) {
2585                         err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
2586                         if (err) {
2587                                 IGB_ERR("No usable DMA configuration, "
2588                                         "aborting\n");
2589                                 goto err_dma;
2590                         }
2591                 }
2592         }
2593
2594 #ifndef HAVE_ASPM_QUIRKS
2595         /* 82575 requires that the pci-e link partner disable the L0s state */
2596         switch (pdev->device) {
2597         case E1000_DEV_ID_82575EB_COPPER:
2598         case E1000_DEV_ID_82575EB_FIBER_SERDES:
2599         case E1000_DEV_ID_82575GB_QUAD_COPPER:
2600                 pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S);
2601         default:
2602                 break;
2603         }
2604
2605 #endif /* HAVE_ASPM_QUIRKS */
2606         err = pci_request_selected_regions(pdev,
2607                                            pci_select_bars(pdev,
2608                                                            IORESOURCE_MEM),
2609                                            igb_driver_name);
2610         if (err)
2611                 goto err_pci_reg;
2612
2613         pci_enable_pcie_error_reporting(pdev);
2614
2615         pci_set_master(pdev);
2616
2617         err = -ENOMEM;
2618 #ifdef HAVE_TX_MQ
2619         netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
2620                                    IGB_MAX_TX_QUEUES);
2621 #else
2622         netdev = alloc_etherdev(sizeof(struct igb_adapter));
2623 #endif /* HAVE_TX_MQ */
2624         if (!netdev)
2625                 goto err_alloc_etherdev;
2626
2627         SET_MODULE_OWNER(netdev);
2628         SET_NETDEV_DEV(netdev, &pdev->dev);
2629
2630         pci_set_drvdata(pdev, netdev);
2631         adapter = netdev_priv(netdev);
2632         adapter->netdev = netdev;
2633         adapter->pdev = pdev;
2634         hw = &adapter->hw;
2635         hw->back = adapter;
2636         adapter->port_num = hw->bus.func;
2637         adapter->msg_enable = (1 << debug) - 1;
2638
2639 #ifdef HAVE_PCI_ERS
2640         err = pci_save_state(pdev);
2641         if (err)
2642                 goto err_ioremap;
2643 #endif
2644         err = -EIO;
2645         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
2646                               pci_resource_len(pdev, 0));
2647         if (!hw->hw_addr)
2648                 goto err_ioremap;
2649
2650 #ifdef HAVE_NET_DEVICE_OPS
2651         netdev->netdev_ops = &igb_netdev_ops;
2652 #else /* HAVE_NET_DEVICE_OPS */
2653         netdev->open = &igb_open;
2654         netdev->stop = &igb_close;
2655         netdev->get_stats = &igb_get_stats;
2656 #ifdef HAVE_SET_RX_MODE
2657         netdev->set_rx_mode = &igb_set_rx_mode;
2658 #endif
2659         netdev->set_multicast_list = &igb_set_rx_mode;
2660         netdev->set_mac_address = &igb_set_mac;
2661         netdev->change_mtu = &igb_change_mtu;
2662         netdev->do_ioctl = &igb_ioctl;
2663 #ifdef HAVE_TX_TIMEOUT
2664         netdev->tx_timeout = &igb_tx_timeout;
2665 #endif
2666         netdev->vlan_rx_register = igb_vlan_mode;
2667         netdev->vlan_rx_add_vid = igb_vlan_rx_add_vid;
2668         netdev->vlan_rx_kill_vid = igb_vlan_rx_kill_vid;
2669 #ifdef CONFIG_NET_POLL_CONTROLLER
2670         netdev->poll_controller = igb_netpoll;
2671 #endif
2672         netdev->hard_start_xmit = &igb_xmit_frame;
2673 #endif /* HAVE_NET_DEVICE_OPS */
2674         igb_set_ethtool_ops(netdev);
2675 #ifdef HAVE_TX_TIMEOUT
2676         netdev->watchdog_timeo = 5 * HZ;
2677 #endif
2678
2679         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
2680
2681         adapter->bd_number = cards_found;
2682
2683         /* setup the private structure */
2684         err = igb_sw_init(adapter);
2685         if (err)
2686                 goto err_sw_init;
2687
2688         e1000_get_bus_info(hw);
2689
2690         hw->phy.autoneg_wait_to_complete = FALSE;
2691         hw->mac.adaptive_ifs = FALSE;
2692
2693         /* Copper options */
2694         if (hw->phy.media_type == e1000_media_type_copper) {
2695                 hw->phy.mdix = AUTO_ALL_MODES;
2696                 hw->phy.disable_polarity_correction = FALSE;
2697                 hw->phy.ms_type = e1000_ms_hw_default;
2698         }
2699
2700         if (e1000_check_reset_block(hw))
2701                 dev_info(pci_dev_to_dev(pdev),
2702                         "PHY reset is blocked due to SOL/IDER session.\n");
2703
2704         /*
2705          * features is initialized to 0 in allocation, it might have bits
2706          * set by igb_sw_init so we should use an or instead of an
2707          * assignment.
2708          */
2709         netdev->features |= NETIF_F_SG |
2710                             NETIF_F_IP_CSUM |
2711 #ifdef NETIF_F_IPV6_CSUM
2712                             NETIF_F_IPV6_CSUM |
2713 #endif
2714 #ifdef NETIF_F_TSO
2715                             NETIF_F_TSO |
2716 #ifdef NETIF_F_TSO6
2717                             NETIF_F_TSO6 |
2718 #endif
2719 #endif /* NETIF_F_TSO */
2720 #ifdef NETIF_F_RXHASH
2721                             NETIF_F_RXHASH |
2722 #endif
2723                             NETIF_F_RXCSUM |
2724 #ifdef NETIF_F_HW_VLAN_CTAG_RX
2725                             NETIF_F_HW_VLAN_CTAG_RX |
2726                             NETIF_F_HW_VLAN_CTAG_TX;
2727 #else
2728                             NETIF_F_HW_VLAN_RX |
2729                             NETIF_F_HW_VLAN_TX;
2730 #endif
2731
2732         if (hw->mac.type >= e1000_82576)
2733                 netdev->features |= NETIF_F_SCTP_CSUM;
2734
2735 #ifdef HAVE_NDO_SET_FEATURES
2736         /* copy netdev features into list of user selectable features */
2737         netdev->hw_features |= netdev->features;
2738 #ifndef IGB_NO_LRO
2739
2740         /* give us the option of enabling LRO later */
2741         netdev->hw_features |= NETIF_F_LRO;
2742 #endif
2743 #else
2744 #ifdef NETIF_F_GRO
2745
2746         /* this is only needed on kernels prior to 2.6.39 */
2747         netdev->features |= NETIF_F_GRO;
2748 #endif
2749 #endif
2750
2751         /* set this bit last since it cannot be part of hw_features */
2752 #ifdef NETIF_F_HW_VLAN_CTAG_FILTER
2753         netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
2754 #else
2755         netdev->features |= NETIF_F_HW_VLAN_FILTER;
2756 #endif
2757
2758 #ifdef HAVE_NETDEV_VLAN_FEATURES
2759         netdev->vlan_features |= NETIF_F_TSO |
2760                                  NETIF_F_TSO6 |
2761                                  NETIF_F_IP_CSUM |
2762                                  NETIF_F_IPV6_CSUM |
2763                                  NETIF_F_SG;
2764
2765 #endif
2766         if (pci_using_dac)
2767                 netdev->features |= NETIF_F_HIGHDMA;
2768
2769         adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
2770 #ifdef DEBUG
2771         if (adapter->dmac != IGB_DMAC_DISABLE)
2772                 printk("%s: DMA Coalescing is enabled..\n", netdev->name);
2773 #endif
2774
2775         /* before reading the NVM, reset the controller to put the device in a
2776          * known good starting state */
2777         e1000_reset_hw(hw);
2778
2779         /* make sure the NVM is good */
2780         if (e1000_validate_nvm_checksum(hw) < 0) {
2781                 dev_err(pci_dev_to_dev(pdev), "The NVM Checksum Is Not"
2782                         " Valid\n");
2783                 err = -EIO;
2784                 goto err_eeprom;
2785         }
2786
2787         /* copy the MAC address out of the NVM */
2788         if (e1000_read_mac_addr(hw))
2789                 dev_err(pci_dev_to_dev(pdev), "NVM Read Error\n");
2790         memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2791 #ifdef ETHTOOL_GPERMADDR
2792         memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
2793
2794         if (!is_valid_ether_addr(netdev->perm_addr)) {
2795 #else
2796         if (!is_valid_ether_addr(netdev->dev_addr)) {
2797 #endif
2798                 dev_err(pci_dev_to_dev(pdev), "Invalid MAC Address\n");
2799                 err = -EIO;
2800                 goto err_eeprom;
2801         }
2802
2803         memcpy(&adapter->mac_table[0].addr, hw->mac.addr, netdev->addr_len);
2804         adapter->mac_table[0].queue = adapter->vfs_allocated_count;
2805         adapter->mac_table[0].state = (IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE);
2806         igb_rar_set(adapter, 0);
2807
2808         /* get firmware version for ethtool -i */
2809         igb_set_fw_version(adapter);
2810
2811         /* Check if Media Autosense is enabled */
2812         if (hw->mac.type == e1000_82580)
2813                 igb_init_mas(adapter);
2814 #ifdef HAVE_TIMER_SETUP
2815         timer_setup(&adapter->watchdog_timer, &igb_watchdog, 0);
2816         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2817                 timer_setup(&adapter->dma_err_timer, &igb_dma_err_timer, 0);
2818         timer_setup(&adapter->phy_info_timer, &igb_update_phy_info, 0);
2819 #else
2820         setup_timer(&adapter->watchdog_timer, &igb_watchdog,
2821                     (unsigned long) adapter);
2822         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2823                 setup_timer(&adapter->dma_err_timer, &igb_dma_err_timer,
2824                             (unsigned long) adapter);
2825         setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
2826                     (unsigned long) adapter);
2827 #endif
2828
2829         INIT_WORK(&adapter->reset_task, igb_reset_task);
2830         INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2831         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2832                 INIT_WORK(&adapter->dma_err_task, igb_dma_err_task);
2833
2834         /* Initialize link properties that are user-changeable */
2835         adapter->fc_autoneg = true;
2836         hw->mac.autoneg = true;
2837         hw->phy.autoneg_advertised = 0x2f;
2838
2839         hw->fc.requested_mode = e1000_fc_default;
2840         hw->fc.current_mode = e1000_fc_default;
2841
2842         e1000_validate_mdi_setting(hw);
2843
2844         /* By default, support wake on port A */
2845         if (hw->bus.func == 0)
2846                 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
2847
2848         /* Check the NVM for wake support for non-port A ports */
2849         if (hw->mac.type >= e1000_82580)
2850                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2851                                  NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2852                                  &eeprom_data);
2853         else if (hw->bus.func == 1)
2854                 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
2855
2856         if (eeprom_data & IGB_EEPROM_APME)
2857                 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
2858
2859         /* now that we have the eeprom settings, apply the special cases where
2860          * the eeprom may be wrong or the board simply won't support wake on
2861          * lan on a particular port */
2862         switch (pdev->device) {
2863         case E1000_DEV_ID_82575GB_QUAD_COPPER:
2864                 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
2865                 break;
2866         case E1000_DEV_ID_82575EB_FIBER_SERDES:
2867         case E1000_DEV_ID_82576_FIBER:
2868         case E1000_DEV_ID_82576_SERDES:
2869                 /* Wake events only supported on port A for dual fiber
2870                  * regardless of eeprom setting */
2871                 if (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_FUNC_1)
2872                         adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
2873                 break;
2874         case E1000_DEV_ID_82576_QUAD_COPPER:
2875         case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
2876                 /* if quad port adapter, disable WoL on all but port A */
2877                 if (global_quad_port_a != 0)
2878                         adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
2879                 else
2880                         adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2881                 /* Reset for multiple quad port adapters */
2882                 if (++global_quad_port_a == 4)
2883                         global_quad_port_a = 0;
2884                 break;
2885         default:
2886                 /* If the device can't wake, don't set software support */
2887                 if (!device_can_wakeup(&adapter->pdev->dev))
2888                         adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
2889                 break;
2890         }
2891
2892         /* initialize the wol settings based on the eeprom settings */
2893         if (adapter->flags & IGB_FLAG_WOL_SUPPORTED)
2894                 adapter->wol |= E1000_WUFC_MAG;
2895
2896         /* Some vendors want WoL disabled by default, but still supported */
2897         if ((hw->mac.type == e1000_i350) &&
2898             (pdev->subsystem_vendor == PCI_VENDOR_ID_HP)) {
2899                 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
2900                 adapter->wol = 0;
2901         }
2902
2903         device_set_wakeup_enable(pci_dev_to_dev(adapter->pdev),
2904                                  adapter->flags & IGB_FLAG_WOL_SUPPORTED);
2905
2906         /* reset the hardware with the new settings */
2907         igb_reset(adapter);
2908         adapter->devrc = 0;
2909
2910 #ifdef HAVE_I2C_SUPPORT
2911         /* Init the I2C interface */
2912         err = igb_init_i2c(adapter);
2913         if (err) {
2914                 dev_err(&pdev->dev, "failed to init i2c interface\n");
2915                 goto err_eeprom;
2916         }
2917 #endif /* HAVE_I2C_SUPPORT */
2918
2919         /* let the f/w know that the h/w is now under the control of the
2920          * driver. */
2921         igb_get_hw_control(adapter);
2922
2923         strncpy(netdev->name, "eth%d", IFNAMSIZ);
2924         err = register_netdev(netdev);
2925         if (err)
2926                 goto err_register;
2927
2928 #ifdef CONFIG_IGB_VMDQ_NETDEV
2929         err = igb_init_vmdq_netdevs(adapter);
2930         if (err)
2931                 goto err_register;
2932 #endif
2933         /* carrier off reporting is important to ethtool even BEFORE open */
2934         netif_carrier_off(netdev);
2935
2936 #ifdef IGB_DCA
2937         if (dca_add_requester(&pdev->dev) == E1000_SUCCESS) {
2938                 adapter->flags |= IGB_FLAG_DCA_ENABLED;
2939                 dev_info(pci_dev_to_dev(pdev), "DCA enabled\n");
2940                 igb_setup_dca(adapter);
2941         }
2942
2943 #endif
2944 #ifdef HAVE_PTP_1588_CLOCK
2945         /* do hw tstamp init after resetting */
2946         igb_ptp_init(adapter);
2947 #endif /* HAVE_PTP_1588_CLOCK */
2948
2949         dev_info(pci_dev_to_dev(pdev), "Intel(R) Gigabit Ethernet Network Connection\n");
2950         /* print bus type/speed/width info */
2951         dev_info(pci_dev_to_dev(pdev), "%s: (PCIe:%s:%s) ",
2952                  netdev->name,
2953                  ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5GT/s" :
2954                   (hw->bus.speed == e1000_bus_speed_5000) ? "5.0GT/s" :
2955                   (hw->mac.type == e1000_i354) ? "integrated" :
2956                                                             "unknown"),
2957                  ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2958                   (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2959                   (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2960                   (hw->mac.type == e1000_i354) ? "integrated" :
2961                    "unknown"));
2962         dev_info(pci_dev_to_dev(pdev), "%s: MAC: ", netdev->name);
2963         for (i = 0; i < 6; i++)
2964                 printk("%2.2x%c", netdev->dev_addr[i], i == 5 ? '\n' : ':');
2965
2966         ret_val = e1000_read_pba_string(hw, pba_str, E1000_PBANUM_LENGTH);
2967         if (ret_val)
2968                 strncpy(pba_str, "Unknown", sizeof(pba_str) - 1);
2969         dev_info(pci_dev_to_dev(pdev), "%s: PBA No: %s\n", netdev->name,
2970                  pba_str);
2971
2972
2973         /* Initialize the thermal sensor on i350 devices. */
2974         if (hw->mac.type == e1000_i350) {
2975                 if (hw->bus.func == 0) {
2976                         u16 ets_word;
2977
2978                         /*
2979                          * Read the NVM to determine if this i350 device
2980                          * supports an external thermal sensor.
2981                          */
2982                         e1000_read_nvm(hw, NVM_ETS_CFG, 1, &ets_word);
2983                         if (ets_word != 0x0000 && ets_word != 0xFFFF)
2984                                 adapter->ets = true;
2985                         else
2986                                 adapter->ets = false;
2987                 }
2988 #ifdef IGB_HWMON
2989
2990                 igb_sysfs_init(adapter);
2991 #else
2992 #ifdef IGB_PROCFS
2993
2994                 igb_procfs_init(adapter);
2995 #endif /* IGB_PROCFS */
2996 #endif /* IGB_HWMON */
2997         } else {
2998                 adapter->ets = false;
2999         }
3000
3001         if (hw->phy.media_type == e1000_media_type_copper) {
3002                 switch (hw->mac.type) {
3003                 case e1000_i350:
3004                 case e1000_i210:
3005                 case e1000_i211:
3006                         /* Enable EEE for internal copper PHY devices */
3007                         err = e1000_set_eee_i350(hw);
3008                         if (!err &&
3009                             (adapter->flags & IGB_FLAG_EEE))
3010                                 adapter->eee_advert =
3011                                         MDIO_EEE_100TX | MDIO_EEE_1000T;
3012                         break;
3013                 case e1000_i354:
3014                         if ((E1000_READ_REG(hw, E1000_CTRL_EXT)) &
3015                             (E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3016                                 err = e1000_set_eee_i354(hw);
3017                                 if ((!err) &&
3018                                     (adapter->flags & IGB_FLAG_EEE))
3019                                         adapter->eee_advert =
3020                                            MDIO_EEE_100TX | MDIO_EEE_1000T;
3021                         }
3022                         break;
3023                 default:
3024                         break;
3025                 }
3026         }
3027
3028         /* send driver version info to firmware */
3029         if (hw->mac.type >= e1000_i350)
3030                 igb_init_fw(adapter);
3031
3032 #ifndef IGB_NO_LRO
3033         if (netdev->features & NETIF_F_LRO)
3034                 dev_info(pci_dev_to_dev(pdev), "Internal LRO is enabled \n");
3035         else
3036                 dev_info(pci_dev_to_dev(pdev), "LRO is disabled \n");
3037 #endif
3038         dev_info(pci_dev_to_dev(pdev),
3039                  "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
3040                  adapter->msix_entries ? "MSI-X" :
3041                  (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
3042                  adapter->num_rx_queues, adapter->num_tx_queues);
3043
3044         cards_found++;
3045
3046         pm_runtime_put_noidle(&pdev->dev);
3047         return 0;
3048
3049 err_register:
3050         igb_release_hw_control(adapter);
3051 #ifdef HAVE_I2C_SUPPORT
3052         memset(&adapter->i2c_adap, 0, sizeof(adapter->i2c_adap));
3053 #endif /* HAVE_I2C_SUPPORT */
3054 err_eeprom:
3055         if (!e1000_check_reset_block(hw))
3056                 e1000_phy_hw_reset(hw);
3057
3058         if (hw->flash_address)
3059                 iounmap(hw->flash_address);
3060 err_sw_init:
3061         igb_clear_interrupt_scheme(adapter);
3062         igb_reset_sriov_capability(adapter);
3063         iounmap(hw->hw_addr);
3064 err_ioremap:
3065         free_netdev(netdev);
3066 err_alloc_etherdev:
3067         pci_release_selected_regions(pdev,
3068                                      pci_select_bars(pdev, IORESOURCE_MEM));
3069 err_pci_reg:
3070 err_dma:
3071         pci_disable_device(pdev);
3072         return err;
3073 }
3074 #ifdef HAVE_I2C_SUPPORT
3075 /*
3076  *  igb_remove_i2c - Cleanup  I2C interface
3077  *  @adapter: pointer to adapter structure
3078  *
3079  */
3080 static void igb_remove_i2c(struct igb_adapter *adapter)
3081 {
3082
3083         /* free the adapter bus structure */
3084         i2c_del_adapter(&adapter->i2c_adap);
3085 }
3086 #endif /* HAVE_I2C_SUPPORT */
3087
3088 /**
3089  * igb_remove - Device Removal Routine
3090  * @pdev: PCI device information struct
3091  *
3092  * igb_remove is called by the PCI subsystem to alert the driver
3093  * that it should release a PCI device.  The could be caused by a
3094  * Hot-Plug event, or because the driver is going to be removed from
3095  * memory.
3096  **/
3097 static void __devexit igb_remove(struct pci_dev *pdev)
3098 {
3099         struct net_device *netdev = pci_get_drvdata(pdev);
3100         struct igb_adapter *adapter = netdev_priv(netdev);
3101         struct e1000_hw *hw = &adapter->hw;
3102
3103         pm_runtime_get_noresume(&pdev->dev);
3104 #ifdef HAVE_I2C_SUPPORT
3105         igb_remove_i2c(adapter);
3106 #endif /* HAVE_I2C_SUPPORT */
3107 #ifdef HAVE_PTP_1588_CLOCK
3108         igb_ptp_stop(adapter);
3109 #endif /* HAVE_PTP_1588_CLOCK */
3110
3111         /* flush_scheduled work may reschedule our watchdog task, so
3112          * explicitly disable watchdog tasks from being rescheduled  */
3113         set_bit(__IGB_DOWN, &adapter->state);
3114         del_timer_sync(&adapter->watchdog_timer);
3115         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
3116                 del_timer_sync(&adapter->dma_err_timer);
3117         del_timer_sync(&adapter->phy_info_timer);
3118
3119         flush_scheduled_work();
3120
3121 #ifdef IGB_DCA
3122         if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
3123                 dev_info(pci_dev_to_dev(pdev), "DCA disabled\n");
3124                 dca_remove_requester(&pdev->dev);
3125                 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
3126                 E1000_WRITE_REG(hw, E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_DISABLE);
3127         }
3128 #endif
3129
3130         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
3131          * would have already happened in close and is redundant. */
3132         igb_release_hw_control(adapter);
3133
3134         unregister_netdev(netdev);
3135 #ifdef CONFIG_IGB_VMDQ_NETDEV
3136         igb_remove_vmdq_netdevs(adapter);
3137 #endif
3138
3139         igb_clear_interrupt_scheme(adapter);
3140         igb_reset_sriov_capability(adapter);
3141
3142         iounmap(hw->hw_addr);
3143         if (hw->flash_address)
3144                 iounmap(hw->flash_address);
3145         pci_release_selected_regions(pdev,
3146                                      pci_select_bars(pdev, IORESOURCE_MEM));
3147
3148 #ifdef IGB_HWMON
3149         igb_sysfs_exit(adapter);
3150 #else
3151 #ifdef IGB_PROCFS
3152         igb_procfs_exit(adapter);
3153 #endif /* IGB_PROCFS */
3154 #endif /* IGB_HWMON */
3155         kfree(adapter->mac_table);
3156         kfree(adapter->shadow_vfta);
3157         free_netdev(netdev);
3158
3159         pci_disable_pcie_error_reporting(pdev);
3160
3161         pci_disable_device(pdev);
3162 }
3163
3164 /**
3165  * igb_sw_init - Initialize general software structures (struct igb_adapter)
3166  * @adapter: board private structure to initialize
3167  *
3168  * igb_sw_init initializes the Adapter private data structure.
3169  * Fields are initialized based on PCI device information and
3170  * OS network device settings (MTU size).
3171  **/
3172 static int igb_sw_init(struct igb_adapter *adapter)
3173 {
3174         struct e1000_hw *hw = &adapter->hw;
3175         struct net_device *netdev = adapter->netdev;
3176         struct pci_dev *pdev = adapter->pdev;
3177
3178         /* PCI config space info */
3179
3180         hw->vendor_id = pdev->vendor;
3181         hw->device_id = pdev->device;
3182         hw->subsystem_vendor_id = pdev->subsystem_vendor;
3183         hw->subsystem_device_id = pdev->subsystem_device;
3184
3185         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
3186
3187         pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
3188
3189         /* set default ring sizes */
3190         adapter->tx_ring_count = IGB_DEFAULT_TXD;
3191         adapter->rx_ring_count = IGB_DEFAULT_RXD;
3192
3193         /* set default work limits */
3194         adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
3195
3196         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
3197                                               VLAN_HLEN;
3198
3199         /* Initialize the hardware-specific values */
3200         if (e1000_setup_init_funcs(hw, TRUE)) {
3201                 dev_err(pci_dev_to_dev(pdev), "Hardware Initialization Failure\n");
3202                 return -EIO;
3203         }
3204
3205         adapter->mac_table = kzalloc(sizeof(struct igb_mac_addr) *
3206                                      hw->mac.rar_entry_count,
3207                                      GFP_ATOMIC);
3208
3209         /* Setup and initialize a copy of the hw vlan table array */
3210         adapter->shadow_vfta = kzalloc(sizeof(u32) * E1000_VFTA_ENTRIES,
3211                                        GFP_ATOMIC);
3212 #ifdef NO_KNI
3213         /* These calls may decrease the number of queues */
3214         if (hw->mac.type < e1000_i210) {
3215                 igb_set_sriov_capability(adapter);
3216         }
3217
3218         if (igb_init_interrupt_scheme(adapter, true)) {
3219                 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for queues\n");
3220                 return -ENOMEM;
3221         }
3222
3223         /* Explicitly disable IRQ since the NIC can be in any state. */
3224         igb_irq_disable(adapter);
3225
3226         set_bit(__IGB_DOWN, &adapter->state);
3227 #endif
3228         return 0;
3229 }
3230
3231 /**
3232  * igb_open - Called when a network interface is made active
3233  * @netdev: network interface device structure
3234  *
3235  * Returns 0 on success, negative value on failure
3236  *
3237  * The open entry point is called when a network interface is made
3238  * active by the system (IFF_UP).  At this point all resources needed
3239  * for transmit and receive operations are allocated, the interrupt
3240  * handler is registered with the OS, the watchdog timer is started,
3241  * and the stack is notified that the interface is ready.
3242  **/
3243 static int __igb_open(struct net_device *netdev, bool resuming)
3244 {
3245         struct igb_adapter *adapter = netdev_priv(netdev);
3246         struct e1000_hw *hw = &adapter->hw;
3247 #ifdef CONFIG_PM_RUNTIME
3248         struct pci_dev *pdev = adapter->pdev;
3249 #endif /* CONFIG_PM_RUNTIME */
3250         int err;
3251         int i;
3252
3253         /* disallow open during test */
3254         if (test_bit(__IGB_TESTING, &adapter->state)) {
3255                 WARN_ON(resuming);
3256                 return -EBUSY;
3257         }
3258
3259 #ifdef CONFIG_PM_RUNTIME
3260         if (!resuming)
3261                 pm_runtime_get_sync(&pdev->dev);
3262 #endif /* CONFIG_PM_RUNTIME */
3263
3264         netif_carrier_off(netdev);
3265
3266         /* allocate transmit descriptors */
3267         err = igb_setup_all_tx_resources(adapter);
3268         if (err)
3269                 goto err_setup_tx;
3270
3271         /* allocate receive descriptors */
3272         err = igb_setup_all_rx_resources(adapter);
3273         if (err)
3274                 goto err_setup_rx;
3275
3276         igb_power_up_link(adapter);
3277
3278         /* before we allocate an interrupt, we must be ready to handle it.
3279          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
3280          * as soon as we call pci_request_irq, so we have to setup our
3281          * clean_rx handler before we do so.  */
3282         igb_configure(adapter);
3283
3284         err = igb_request_irq(adapter);
3285         if (err)
3286                 goto err_req_irq;
3287
3288         /* Notify the stack of the actual queue counts. */
3289         netif_set_real_num_tx_queues(netdev,
3290                                      adapter->vmdq_pools ? 1 :
3291                                      adapter->num_tx_queues);
3292
3293         err = netif_set_real_num_rx_queues(netdev,
3294                                            adapter->vmdq_pools ? 1 :
3295                                            adapter->num_rx_queues);
3296         if (err)
3297                 goto err_set_queues;
3298
3299         /* From here on the code is the same as igb_up() */
3300         clear_bit(__IGB_DOWN, &adapter->state);
3301
3302         for (i = 0; i < adapter->num_q_vectors; i++)
3303                 napi_enable(&(adapter->q_vector[i]->napi));
3304         igb_configure_lli(adapter);
3305
3306         /* Clear any pending interrupts. */
3307         E1000_READ_REG(hw, E1000_ICR);
3308
3309         igb_irq_enable(adapter);
3310
3311         /* notify VFs that reset has been completed */
3312         if (adapter->vfs_allocated_count) {
3313                 u32 reg_data = E1000_READ_REG(hw, E1000_CTRL_EXT);
3314                 reg_data |= E1000_CTRL_EXT_PFRSTD;
3315                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg_data);
3316         }
3317
3318         netif_tx_start_all_queues(netdev);
3319
3320         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
3321                 schedule_work(&adapter->dma_err_task);
3322
3323         /* start the watchdog. */
3324         hw->mac.get_link_status = 1;
3325         schedule_work(&adapter->watchdog_task);
3326
3327         return E1000_SUCCESS;
3328
3329 err_set_queues:
3330         igb_free_irq(adapter);
3331 err_req_irq:
3332         igb_release_hw_control(adapter);
3333         igb_power_down_link(adapter);
3334         igb_free_all_rx_resources(adapter);
3335 err_setup_rx:
3336         igb_free_all_tx_resources(adapter);
3337 err_setup_tx:
3338         igb_reset(adapter);
3339
3340 #ifdef CONFIG_PM_RUNTIME
3341         if (!resuming)
3342                 pm_runtime_put(&pdev->dev);
3343 #endif /* CONFIG_PM_RUNTIME */
3344
3345         return err;
3346 }
3347
3348 static int igb_open(struct net_device *netdev)
3349 {
3350         return __igb_open(netdev, false);
3351 }
3352
3353 /**
3354  * igb_close - Disables a network interface
3355  * @netdev: network interface device structure
3356  *
3357  * Returns 0, this is not allowed to fail
3358  *
3359  * The close entry point is called when an interface is de-activated
3360  * by the OS.  The hardware is still under the driver's control, but
3361  * needs to be disabled.  A global MAC reset is issued to stop the
3362  * hardware, and all transmit and receive resources are freed.
3363  **/
3364 static int __igb_close(struct net_device *netdev, bool suspending)
3365 {
3366         struct igb_adapter *adapter = netdev_priv(netdev);
3367 #ifdef CONFIG_PM_RUNTIME
3368         struct pci_dev *pdev = adapter->pdev;
3369 #endif /* CONFIG_PM_RUNTIME */
3370
3371         WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
3372
3373 #ifdef CONFIG_PM_RUNTIME
3374         if (!suspending)
3375                 pm_runtime_get_sync(&pdev->dev);
3376 #endif /* CONFIG_PM_RUNTIME */
3377
3378         igb_down(adapter);
3379
3380         igb_release_hw_control(adapter);
3381
3382         igb_free_irq(adapter);
3383
3384         igb_free_all_tx_resources(adapter);
3385         igb_free_all_rx_resources(adapter);
3386
3387 #ifdef CONFIG_PM_RUNTIME
3388         if (!suspending)
3389                 pm_runtime_put_sync(&pdev->dev);
3390 #endif /* CONFIG_PM_RUNTIME */
3391
3392         return 0;
3393 }
3394
3395 static int igb_close(struct net_device *netdev)
3396 {
3397         return __igb_close(netdev, false);
3398 }
3399
3400 /**
3401  * igb_setup_tx_resources - allocate Tx resources (Descriptors)
3402  * @tx_ring: tx descriptor ring (for a specific queue) to setup
3403  *
3404  * Return 0 on success, negative on failure
3405  **/
3406 int igb_setup_tx_resources(struct igb_ring *tx_ring)
3407 {
3408         struct device *dev = tx_ring->dev;
3409         int size;
3410
3411         size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3412         tx_ring->tx_buffer_info = vzalloc(size);
3413         if (!tx_ring->tx_buffer_info)
3414                 goto err;
3415
3416         /* round up to nearest 4K */
3417         tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
3418         tx_ring->size = ALIGN(tx_ring->size, 4096);
3419
3420         tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
3421                                            &tx_ring->dma, GFP_KERNEL);
3422
3423         if (!tx_ring->desc)
3424                 goto err;
3425
3426         tx_ring->next_to_use = 0;
3427         tx_ring->next_to_clean = 0;
3428
3429         return 0;
3430
3431 err:
3432         vfree(tx_ring->tx_buffer_info);
3433         dev_err(dev,
3434                 "Unable to allocate memory for the transmit descriptor ring\n");
3435         return -ENOMEM;
3436 }
3437
3438 /**
3439  * igb_setup_all_tx_resources - wrapper to allocate Tx resources
3440  *                                (Descriptors) for all queues
3441  * @adapter: board private structure
3442  *
3443  * Return 0 on success, negative on failure
3444  **/
3445 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
3446 {
3447         struct pci_dev *pdev = adapter->pdev;
3448         int i, err = 0;
3449
3450         for (i = 0; i < adapter->num_tx_queues; i++) {
3451                 err = igb_setup_tx_resources(adapter->tx_ring[i]);
3452                 if (err) {
3453                         dev_err(pci_dev_to_dev(pdev),
3454                                 "Allocation for Tx Queue %u failed\n", i);
3455                         for (i--; i >= 0; i--)
3456                                 igb_free_tx_resources(adapter->tx_ring[i]);
3457                         break;
3458                 }
3459         }
3460
3461         return err;
3462 }
3463
3464 /**
3465  * igb_setup_tctl - configure the transmit control registers
3466  * @adapter: Board private structure
3467  **/
3468 void igb_setup_tctl(struct igb_adapter *adapter)
3469 {
3470         struct e1000_hw *hw = &adapter->hw;
3471         u32 tctl;
3472
3473         /* disable queue 0 which is enabled by default on 82575 and 82576 */
3474         E1000_WRITE_REG(hw, E1000_TXDCTL(0), 0);
3475
3476         /* Program the Transmit Control Register */
3477         tctl = E1000_READ_REG(hw, E1000_TCTL);
3478         tctl &= ~E1000_TCTL_CT;
3479         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
3480                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
3481
3482         e1000_config_collision_dist(hw);
3483
3484         /* Enable transmits */
3485         tctl |= E1000_TCTL_EN;
3486
3487         E1000_WRITE_REG(hw, E1000_TCTL, tctl);
3488 }
3489
3490 static u32 igb_tx_wthresh(struct igb_adapter *adapter)
3491 {
3492         struct e1000_hw *hw = &adapter->hw;
3493         switch (hw->mac.type) {
3494         case e1000_i354:
3495                 return 4;
3496         case e1000_82576:
3497                 if (adapter->msix_entries)
3498                         return 1;
3499         default:
3500                 break;
3501         }
3502
3503         return 16;
3504 }
3505
3506 /**
3507  * igb_configure_tx_ring - Configure transmit ring after Reset
3508  * @adapter: board private structure
3509  * @ring: tx ring to configure
3510  *
3511  * Configure a transmit ring after a reset.
3512  **/
3513 void igb_configure_tx_ring(struct igb_adapter *adapter,
3514                            struct igb_ring *ring)
3515 {
3516         struct e1000_hw *hw = &adapter->hw;
3517         u32 txdctl = 0;
3518         u64 tdba = ring->dma;
3519         int reg_idx = ring->reg_idx;
3520
3521         /* disable the queue */
3522         E1000_WRITE_REG(hw, E1000_TXDCTL(reg_idx), 0);
3523         E1000_WRITE_FLUSH(hw);
3524         mdelay(10);
3525
3526         E1000_WRITE_REG(hw, E1000_TDLEN(reg_idx),
3527                         ring->count * sizeof(union e1000_adv_tx_desc));
3528         E1000_WRITE_REG(hw, E1000_TDBAL(reg_idx),
3529                         tdba & 0x00000000ffffffffULL);
3530         E1000_WRITE_REG(hw, E1000_TDBAH(reg_idx), tdba >> 32);
3531
3532         ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
3533         E1000_WRITE_REG(hw, E1000_TDH(reg_idx), 0);
3534         writel(0, ring->tail);
3535
3536         txdctl |= IGB_TX_PTHRESH;
3537         txdctl |= IGB_TX_HTHRESH << 8;
3538         txdctl |= igb_tx_wthresh(adapter) << 16;
3539
3540         txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
3541         E1000_WRITE_REG(hw, E1000_TXDCTL(reg_idx), txdctl);
3542 }
3543
3544 /**
3545  * igb_configure_tx - Configure transmit Unit after Reset
3546  * @adapter: board private structure
3547  *
3548  * Configure the Tx unit of the MAC after a reset.
3549  **/
3550 static void igb_configure_tx(struct igb_adapter *adapter)
3551 {
3552         int i;
3553
3554         for (i = 0; i < adapter->num_tx_queues; i++)
3555                 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
3556 }
3557
3558 /**
3559  * igb_setup_rx_resources - allocate Rx resources (Descriptors)
3560  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
3561  *
3562  * Returns 0 on success, negative on failure
3563  **/
3564 int igb_setup_rx_resources(struct igb_ring *rx_ring)
3565 {
3566         struct device *dev = rx_ring->dev;
3567         int size, desc_len;
3568
3569         size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3570         rx_ring->rx_buffer_info = vzalloc(size);
3571         if (!rx_ring->rx_buffer_info)
3572                 goto err;
3573
3574         desc_len = sizeof(union e1000_adv_rx_desc);
3575
3576         /* Round up to nearest 4K */
3577         rx_ring->size = rx_ring->count * desc_len;
3578         rx_ring->size = ALIGN(rx_ring->size, 4096);
3579
3580         rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
3581                                            &rx_ring->dma, GFP_KERNEL);
3582
3583         if (!rx_ring->desc)
3584                 goto err;
3585
3586         rx_ring->next_to_alloc = 0;
3587         rx_ring->next_to_clean = 0;
3588         rx_ring->next_to_use = 0;
3589
3590         return 0;
3591
3592 err:
3593         vfree(rx_ring->rx_buffer_info);
3594         rx_ring->rx_buffer_info = NULL;
3595         dev_err(dev, "Unable to allocate memory for the receive descriptor"
3596                 " ring\n");
3597         return -ENOMEM;
3598 }
3599
3600 /**
3601  * igb_setup_all_rx_resources - wrapper to allocate Rx resources
3602  *                                (Descriptors) for all queues
3603  * @adapter: board private structure
3604  *
3605  * Return 0 on success, negative on failure
3606  **/
3607 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
3608 {
3609         struct pci_dev *pdev = adapter->pdev;
3610         int i, err = 0;
3611
3612         for (i = 0; i < adapter->num_rx_queues; i++) {
3613                 err = igb_setup_rx_resources(adapter->rx_ring[i]);
3614                 if (err) {
3615                         dev_err(pci_dev_to_dev(pdev),
3616                                 "Allocation for Rx Queue %u failed\n", i);
3617                         for (i--; i >= 0; i--)
3618                                 igb_free_rx_resources(adapter->rx_ring[i]);
3619                         break;
3620                 }
3621         }
3622
3623         return err;
3624 }
3625
3626 /**
3627  * igb_setup_mrqc - configure the multiple receive queue control registers
3628  * @adapter: Board private structure
3629  **/
3630 static void igb_setup_mrqc(struct igb_adapter *adapter)
3631 {
3632         struct e1000_hw *hw = &adapter->hw;
3633         u32 mrqc, rxcsum;
3634         u32 j, num_rx_queues, shift = 0, shift2 = 0;
3635         static const u32 rsskey[10] = { 0xDA565A6D, 0xC20E5B25, 0x3D256741,
3636                                         0xB08FA343, 0xCB2BCAD0, 0xB4307BAE,
3637                                         0xA32DCB77, 0x0CF23080, 0x3BB7426A,
3638                                         0xFA01ACBE };
3639
3640         /* Fill out hash function seeds */
3641         for (j = 0; j < 10; j++)
3642                 E1000_WRITE_REG(hw, E1000_RSSRK(j), rsskey[j]);
3643
3644         num_rx_queues = adapter->rss_queues;
3645
3646         /* 82575 and 82576 supports 2 RSS queues for VMDq */
3647         switch (hw->mac.type) {
3648         case e1000_82575:
3649                 if (adapter->vmdq_pools) {
3650                         shift = 2;
3651                         shift2 = 6;
3652                         break;
3653                 }
3654                 shift = 6;
3655                 break;
3656         case e1000_82576:
3657                 /* 82576 supports 2 RSS queues for SR-IOV */
3658                 if (adapter->vfs_allocated_count || adapter->vmdq_pools) {
3659                         shift = 3;
3660                         num_rx_queues = 2;
3661                 }
3662                 break;
3663         default:
3664                 break;
3665         }
3666
3667         /*
3668          * Populate the redirection table 4 entries at a time.  To do this
3669          * we are generating the results for n and n+2 and then interleaving
3670          * those with the results with n+1 and n+3.
3671          */
3672         for (j = 0; j < 32; j++) {
3673                 /* first pass generates n and n+2 */
3674                 u32 base = ((j * 0x00040004) + 0x00020000) * num_rx_queues;
3675                 u32 reta = (base & 0x07800780) >> (7 - shift);
3676
3677                 /* second pass generates n+1 and n+3 */
3678                 base += 0x00010001 * num_rx_queues;
3679                 reta |= (base & 0x07800780) << (1 + shift);
3680
3681                 /* generate 2nd table for 82575 based parts */
3682                 if (shift2)
3683                         reta |= (0x01010101 * num_rx_queues) << shift2;
3684
3685                 E1000_WRITE_REG(hw, E1000_RETA(j), reta);
3686         }
3687
3688         /*
3689          * Disable raw packet checksumming so that RSS hash is placed in
3690          * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
3691          * offloads as they are enabled by default
3692          */
3693         rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
3694         rxcsum |= E1000_RXCSUM_PCSD;
3695
3696         if (adapter->hw.mac.type >= e1000_82576)
3697                 /* Enable Receive Checksum Offload for SCTP */
3698                 rxcsum |= E1000_RXCSUM_CRCOFL;
3699
3700         /* Don't need to set TUOFL or IPOFL, they default to 1 */
3701         E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
3702
3703         /* Generate RSS hash based on packet types, TCP/UDP
3704          * port numbers and/or IPv4/v6 src and dst addresses
3705          */
3706         mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
3707                E1000_MRQC_RSS_FIELD_IPV4_TCP |
3708                E1000_MRQC_RSS_FIELD_IPV6 |
3709                E1000_MRQC_RSS_FIELD_IPV6_TCP |
3710                E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
3711
3712         if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV4_UDP)
3713                 mrqc |= E1000_MRQC_RSS_FIELD_IPV4_UDP;
3714         if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV6_UDP)
3715                 mrqc |= E1000_MRQC_RSS_FIELD_IPV6_UDP;
3716
3717         /* If VMDq is enabled then we set the appropriate mode for that, else
3718          * we default to RSS so that an RSS hash is calculated per packet even
3719          * if we are only using one queue */
3720         if (adapter->vfs_allocated_count || adapter->vmdq_pools) {
3721                 if (hw->mac.type > e1000_82575) {
3722                         /* Set the default pool for the PF's first queue */
3723                         u32 vtctl = E1000_READ_REG(hw, E1000_VT_CTL);
3724                         vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
3725                                    E1000_VT_CTL_DISABLE_DEF_POOL);
3726                         vtctl |= adapter->vfs_allocated_count <<
3727                                 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
3728                         E1000_WRITE_REG(hw, E1000_VT_CTL, vtctl);
3729                 } else if (adapter->rss_queues > 1) {
3730                         /* set default queue for pool 1 to queue 2 */
3731                         E1000_WRITE_REG(hw, E1000_VT_CTL,
3732                                         adapter->rss_queues << 7);
3733                 }
3734                 if (adapter->rss_queues > 1)
3735                         mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
3736                 else
3737                         mrqc |= E1000_MRQC_ENABLE_VMDQ;
3738         } else {
3739                 mrqc |= E1000_MRQC_ENABLE_RSS_4Q;
3740         }
3741         igb_vmm_control(adapter);
3742
3743         E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
3744 }
3745
3746 /**
3747  * igb_setup_rctl - configure the receive control registers
3748  * @adapter: Board private structure
3749  **/
3750 void igb_setup_rctl(struct igb_adapter *adapter)
3751 {
3752         struct e1000_hw *hw = &adapter->hw;
3753         u32 rctl;
3754
3755         rctl = E1000_READ_REG(hw, E1000_RCTL);
3756
3757         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3758         rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
3759
3760         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
3761                 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
3762
3763         /*
3764          * enable stripping of CRC. It's unlikely this will break BMC
3765          * redirection as it did with e1000. Newer features require
3766          * that the HW strips the CRC.
3767          */
3768         rctl |= E1000_RCTL_SECRC;
3769
3770         /* disable store bad packets and clear size bits. */
3771         rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
3772
3773         /* enable LPE to prevent packets larger than max_frame_size */
3774         rctl |= E1000_RCTL_LPE;
3775
3776         /* disable queue 0 to prevent tail write w/o re-config */
3777         E1000_WRITE_REG(hw, E1000_RXDCTL(0), 0);
3778
3779         /* Attention!!!  For SR-IOV PF driver operations you must enable
3780          * queue drop for all VF and PF queues to prevent head of line blocking
3781          * if an un-trusted VF does not provide descriptors to hardware.
3782          */
3783         if (adapter->vfs_allocated_count) {
3784                 /* set all queue drop enable bits */
3785                 E1000_WRITE_REG(hw, E1000_QDE, ALL_QUEUES);
3786         }
3787
3788         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3789 }
3790
3791 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
3792                                    int vfn)
3793 {
3794         struct e1000_hw *hw = &adapter->hw;
3795         u32 vmolr;
3796
3797         /* if it isn't the PF check to see if VFs are enabled and
3798          * increase the size to support vlan tags */
3799         if (vfn < adapter->vfs_allocated_count &&
3800             adapter->vf_data[vfn].vlans_enabled)
3801                 size += VLAN_HLEN;
3802
3803 #ifdef CONFIG_IGB_VMDQ_NETDEV
3804         if (vfn >= adapter->vfs_allocated_count) {
3805                 int queue = vfn - adapter->vfs_allocated_count;
3806                 struct igb_vmdq_adapter *vadapter;
3807
3808                 vadapter = netdev_priv(adapter->vmdq_netdev[queue-1]);
3809                 if (vadapter->vlgrp)
3810                         size += VLAN_HLEN;
3811         }
3812 #endif
3813         vmolr = E1000_READ_REG(hw, E1000_VMOLR(vfn));
3814         vmolr &= ~E1000_VMOLR_RLPML_MASK;
3815         vmolr |= size | E1000_VMOLR_LPE;
3816         E1000_WRITE_REG(hw, E1000_VMOLR(vfn), vmolr);
3817
3818         return 0;
3819 }
3820
3821 /**
3822  * igb_rlpml_set - set maximum receive packet size
3823  * @adapter: board private structure
3824  *
3825  * Configure maximum receivable packet size.
3826  **/
3827 static void igb_rlpml_set(struct igb_adapter *adapter)
3828 {
3829         u32 max_frame_size = adapter->max_frame_size;
3830         struct e1000_hw *hw = &adapter->hw;
3831         u16 pf_id = adapter->vfs_allocated_count;
3832
3833         if (adapter->vmdq_pools && hw->mac.type != e1000_82575) {
3834                 int i;
3835                 for (i = 0; i < adapter->vmdq_pools; i++)
3836                         igb_set_vf_rlpml(adapter, max_frame_size, pf_id + i);
3837                 /*
3838                  * If we're in VMDQ or SR-IOV mode, then set global RLPML
3839                  * to our max jumbo frame size, in case we need to enable
3840                  * jumbo frames on one of the rings later.
3841                  * This will not pass over-length frames into the default
3842                  * queue because it's gated by the VMOLR.RLPML.
3843                  */
3844                 max_frame_size = MAX_JUMBO_FRAME_SIZE;
3845         }
3846         /* Set VF RLPML for the PF device. */
3847         if (adapter->vfs_allocated_count)
3848                 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
3849
3850         E1000_WRITE_REG(hw, E1000_RLPML, max_frame_size);
3851 }
3852
3853 static inline void igb_set_vf_vlan_strip(struct igb_adapter *adapter,
3854                                         int vfn, bool enable)
3855 {
3856         struct e1000_hw *hw = &adapter->hw;
3857         u32 val;
3858         void __iomem *reg;
3859
3860         if (hw->mac.type < e1000_82576)
3861                 return;
3862
3863         if (hw->mac.type == e1000_i350)
3864                 reg = hw->hw_addr + E1000_DVMOLR(vfn);
3865         else
3866                 reg = hw->hw_addr + E1000_VMOLR(vfn);
3867
3868         val = readl(reg);
3869         if (enable)
3870                 val |= E1000_VMOLR_STRVLAN;
3871         else
3872                 val &= ~(E1000_VMOLR_STRVLAN);
3873         writel(val, reg);
3874 }
3875 static inline void igb_set_vmolr(struct igb_adapter *adapter,
3876                                  int vfn, bool aupe)
3877 {
3878         struct e1000_hw *hw = &adapter->hw;
3879         u32 vmolr;
3880
3881         /*
3882          * This register exists only on 82576 and newer so if we are older then
3883          * we should exit and do nothing
3884          */
3885         if (hw->mac.type < e1000_82576)
3886                 return;
3887
3888         vmolr = E1000_READ_REG(hw, E1000_VMOLR(vfn));
3889
3890         if (aupe)
3891                 vmolr |= E1000_VMOLR_AUPE;        /* Accept untagged packets */
3892         else
3893                 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
3894
3895         /* clear all bits that might not be set */
3896         vmolr &= ~E1000_VMOLR_RSSE;
3897
3898         if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
3899                 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3900
3901         vmolr |= E1000_VMOLR_BAM;          /* Accept broadcast */
3902         vmolr |= E1000_VMOLR_LPE;          /* Accept long packets */
3903
3904         E1000_WRITE_REG(hw, E1000_VMOLR(vfn), vmolr);
3905 }
3906
3907 /**
3908  * igb_configure_rx_ring - Configure a receive ring after Reset
3909  * @adapter: board private structure
3910  * @ring: receive ring to be configured
3911  *
3912  * Configure the Rx unit of the MAC after a reset.
3913  **/
3914 void igb_configure_rx_ring(struct igb_adapter *adapter,
3915                            struct igb_ring *ring)
3916 {
3917         struct e1000_hw *hw = &adapter->hw;
3918         u64 rdba = ring->dma;
3919         int reg_idx = ring->reg_idx;
3920         u32 srrctl = 0, rxdctl = 0;
3921
3922 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
3923         /*
3924          * RLPML prevents us from receiving a frame larger than max_frame so
3925          * it is safe to just set the rx_buffer_len to max_frame without the
3926          * risk of an skb over panic.
3927          */
3928         ring->rx_buffer_len = max_t(u32, adapter->max_frame_size,
3929                                     MAXIMUM_ETHERNET_VLAN_SIZE);
3930
3931 #endif
3932         /* disable the queue */
3933         E1000_WRITE_REG(hw, E1000_RXDCTL(reg_idx), 0);
3934
3935         /* Set DMA base address registers */
3936         E1000_WRITE_REG(hw, E1000_RDBAL(reg_idx),
3937                         rdba & 0x00000000ffffffffULL);
3938         E1000_WRITE_REG(hw, E1000_RDBAH(reg_idx), rdba >> 32);
3939         E1000_WRITE_REG(hw, E1000_RDLEN(reg_idx),
3940                        ring->count * sizeof(union e1000_adv_rx_desc));
3941
3942         /* initialize head and tail */
3943         ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
3944         E1000_WRITE_REG(hw, E1000_RDH(reg_idx), 0);
3945         writel(0, ring->tail);
3946
3947         /* reset next-to- use/clean to place SW in sync with hardwdare */
3948         ring->next_to_clean = 0;
3949         ring->next_to_use = 0;
3950 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
3951         ring->next_to_alloc = 0;
3952
3953 #endif
3954         /* set descriptor configuration */
3955 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
3956         srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3957         srrctl |= IGB_RX_BUFSZ >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3958 #else /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
3959         srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
3960                  E1000_SRRCTL_BSIZEPKT_SHIFT;
3961 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
3962         srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
3963 #ifdef HAVE_PTP_1588_CLOCK
3964         if (hw->mac.type >= e1000_82580)
3965                 srrctl |= E1000_SRRCTL_TIMESTAMP;
3966 #endif /* HAVE_PTP_1588_CLOCK */
3967         /*
3968          * We should set the drop enable bit if:
3969          *  SR-IOV is enabled
3970          *   or
3971          *  Flow Control is disabled and number of RX queues > 1
3972          *
3973          *  This allows us to avoid head of line blocking for security
3974          *  and performance reasons.
3975          */
3976         if (adapter->vfs_allocated_count ||
3977             (adapter->num_rx_queues > 1 &&
3978              (hw->fc.requested_mode == e1000_fc_none ||
3979               hw->fc.requested_mode == e1000_fc_rx_pause)))
3980                 srrctl |= E1000_SRRCTL_DROP_EN;
3981
3982         E1000_WRITE_REG(hw, E1000_SRRCTL(reg_idx), srrctl);
3983
3984         /* set filtering for VMDQ pools */
3985         igb_set_vmolr(adapter, reg_idx & 0x7, true);
3986
3987         rxdctl |= IGB_RX_PTHRESH;
3988         rxdctl |= IGB_RX_HTHRESH << 8;
3989         rxdctl |= IGB_RX_WTHRESH << 16;
3990
3991         /* enable receive descriptor fetching */
3992         rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3993         E1000_WRITE_REG(hw, E1000_RXDCTL(reg_idx), rxdctl);
3994 }
3995
3996 /**
3997  * igb_configure_rx - Configure receive Unit after Reset
3998  * @adapter: board private structure
3999  *
4000  * Configure the Rx unit of the MAC after a reset.
4001  **/
4002 static void igb_configure_rx(struct igb_adapter *adapter)
4003 {
4004         int i;
4005
4006         /* set UTA to appropriate mode */
4007         igb_set_uta(adapter);
4008
4009         igb_full_sync_mac_table(adapter);
4010         /* Setup the HW Rx Head and Tail Descriptor Pointers and
4011          * the Base and Length of the Rx Descriptor Ring */
4012         for (i = 0; i < adapter->num_rx_queues; i++)
4013                 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
4014 }
4015
4016 /**
4017  * igb_free_tx_resources - Free Tx Resources per Queue
4018  * @tx_ring: Tx descriptor ring for a specific queue
4019  *
4020  * Free all transmit software resources
4021  **/
4022 void igb_free_tx_resources(struct igb_ring *tx_ring)
4023 {
4024         igb_clean_tx_ring(tx_ring);
4025
4026         vfree(tx_ring->tx_buffer_info);
4027         tx_ring->tx_buffer_info = NULL;
4028
4029         /* if not set, then don't free */
4030         if (!tx_ring->desc)
4031                 return;
4032
4033         dma_free_coherent(tx_ring->dev, tx_ring->size,
4034                           tx_ring->desc, tx_ring->dma);
4035
4036         tx_ring->desc = NULL;
4037 }
4038
4039 /**
4040  * igb_free_all_tx_resources - Free Tx Resources for All Queues
4041  * @adapter: board private structure
4042  *
4043  * Free all transmit software resources
4044  **/
4045 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
4046 {
4047         int i;
4048
4049         for (i = 0; i < adapter->num_tx_queues; i++)
4050                 igb_free_tx_resources(adapter->tx_ring[i]);
4051 }
4052
4053 void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
4054                                     struct igb_tx_buffer *tx_buffer)
4055 {
4056         if (tx_buffer->skb) {
4057                 dev_kfree_skb_any(tx_buffer->skb);
4058                 if (dma_unmap_len(tx_buffer, len))
4059                         dma_unmap_single(ring->dev,
4060                                          dma_unmap_addr(tx_buffer, dma),
4061                                          dma_unmap_len(tx_buffer, len),
4062                                          DMA_TO_DEVICE);
4063         } else if (dma_unmap_len(tx_buffer, len)) {
4064                 dma_unmap_page(ring->dev,
4065                                dma_unmap_addr(tx_buffer, dma),
4066                                dma_unmap_len(tx_buffer, len),
4067                                DMA_TO_DEVICE);
4068         }
4069         tx_buffer->next_to_watch = NULL;
4070         tx_buffer->skb = NULL;
4071         dma_unmap_len_set(tx_buffer, len, 0);
4072         /* buffer_info must be completely set up in the transmit path */
4073 }
4074
4075 /**
4076  * igb_clean_tx_ring - Free Tx Buffers
4077  * @tx_ring: ring to be cleaned
4078  **/
4079 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
4080 {
4081         struct igb_tx_buffer *buffer_info;
4082         unsigned long size;
4083         u16 i;
4084
4085         if (!tx_ring->tx_buffer_info)
4086                 return;
4087         /* Free all the Tx ring sk_buffs */
4088
4089         for (i = 0; i < tx_ring->count; i++) {
4090                 buffer_info = &tx_ring->tx_buffer_info[i];
4091                 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
4092         }
4093
4094         netdev_tx_reset_queue(txring_txq(tx_ring));
4095
4096         size = sizeof(struct igb_tx_buffer) * tx_ring->count;
4097         memset(tx_ring->tx_buffer_info, 0, size);
4098
4099         /* Zero out the descriptor ring */
4100         memset(tx_ring->desc, 0, tx_ring->size);
4101
4102         tx_ring->next_to_use = 0;
4103         tx_ring->next_to_clean = 0;
4104 }
4105
4106 /**
4107  * igb_clean_all_tx_rings - Free Tx Buffers for all queues
4108  * @adapter: board private structure
4109  **/
4110 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
4111 {
4112         int i;
4113
4114         for (i = 0; i < adapter->num_tx_queues; i++)
4115                 igb_clean_tx_ring(adapter->tx_ring[i]);
4116 }
4117
4118 /**
4119  * igb_free_rx_resources - Free Rx Resources
4120  * @rx_ring: ring to clean the resources from
4121  *
4122  * Free all receive software resources
4123  **/
4124 void igb_free_rx_resources(struct igb_ring *rx_ring)
4125 {
4126         igb_clean_rx_ring(rx_ring);
4127
4128         vfree(rx_ring->rx_buffer_info);
4129         rx_ring->rx_buffer_info = NULL;
4130
4131         /* if not set, then don't free */
4132         if (!rx_ring->desc)
4133                 return;
4134
4135         dma_free_coherent(rx_ring->dev, rx_ring->size,
4136                           rx_ring->desc, rx_ring->dma);
4137
4138         rx_ring->desc = NULL;
4139 }
4140
4141 /**
4142  * igb_free_all_rx_resources - Free Rx Resources for All Queues
4143  * @adapter: board private structure
4144  *
4145  * Free all receive software resources
4146  **/
4147 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
4148 {
4149         int i;
4150
4151         for (i = 0; i < adapter->num_rx_queues; i++)
4152                 igb_free_rx_resources(adapter->rx_ring[i]);
4153 }
4154
4155 /**
4156  * igb_clean_rx_ring - Free Rx Buffers per Queue
4157  * @rx_ring: ring to free buffers from
4158  **/
4159 void igb_clean_rx_ring(struct igb_ring *rx_ring)
4160 {
4161         unsigned long size;
4162         u16 i;
4163
4164         if (!rx_ring->rx_buffer_info)
4165                 return;
4166
4167 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
4168         if (rx_ring->skb)
4169                 dev_kfree_skb(rx_ring->skb);
4170         rx_ring->skb = NULL;
4171
4172 #endif
4173         /* Free all the Rx ring sk_buffs */
4174         for (i = 0; i < rx_ring->count; i++) {
4175                 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
4176 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
4177                 if (buffer_info->dma) {
4178                         dma_unmap_single(rx_ring->dev,
4179                                          buffer_info->dma,
4180                                          rx_ring->rx_buffer_len,
4181                                          DMA_FROM_DEVICE);
4182                         buffer_info->dma = 0;
4183                 }
4184
4185                 if (buffer_info->skb) {
4186                         dev_kfree_skb(buffer_info->skb);
4187                         buffer_info->skb = NULL;
4188                 }
4189 #else
4190                 if (!buffer_info->page)
4191                         continue;
4192
4193                 dma_unmap_page(rx_ring->dev,
4194                                buffer_info->dma,
4195                                PAGE_SIZE,
4196                                DMA_FROM_DEVICE);
4197                 __free_page(buffer_info->page);
4198
4199                 buffer_info->page = NULL;
4200 #endif
4201         }
4202
4203         size = sizeof(struct igb_rx_buffer) * rx_ring->count;
4204         memset(rx_ring->rx_buffer_info, 0, size);
4205
4206         /* Zero out the descriptor ring */
4207         memset(rx_ring->desc, 0, rx_ring->size);
4208
4209         rx_ring->next_to_alloc = 0;
4210         rx_ring->next_to_clean = 0;
4211         rx_ring->next_to_use = 0;
4212 }
4213
4214 /**
4215  * igb_clean_all_rx_rings - Free Rx Buffers for all queues
4216  * @adapter: board private structure
4217  **/
4218 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
4219 {
4220         int i;
4221
4222         for (i = 0; i < adapter->num_rx_queues; i++)
4223                 igb_clean_rx_ring(adapter->rx_ring[i]);
4224 }
4225
4226 /**
4227  * igb_set_mac - Change the Ethernet Address of the NIC
4228  * @netdev: network interface device structure
4229  * @p: pointer to an address structure
4230  *
4231  * Returns 0 on success, negative on failure
4232  **/
4233 static int igb_set_mac(struct net_device *netdev, void *p)
4234 {
4235         struct igb_adapter *adapter = netdev_priv(netdev);
4236         struct e1000_hw *hw = &adapter->hw;
4237         struct sockaddr *addr = p;
4238
4239         if (!is_valid_ether_addr(addr->sa_data))
4240                 return -EADDRNOTAVAIL;
4241
4242         igb_del_mac_filter(adapter, hw->mac.addr,
4243                            adapter->vfs_allocated_count);
4244         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
4245         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
4246
4247         /* set the correct pool for the new PF MAC address in entry 0 */
4248         return igb_add_mac_filter(adapter, hw->mac.addr,
4249                            adapter->vfs_allocated_count);
4250 }
4251
4252 /**
4253  * igb_write_mc_addr_list - write multicast addresses to MTA
4254  * @netdev: network interface device structure
4255  *
4256  * Writes multicast address list to the MTA hash table.
4257  * Returns: -ENOMEM on failure
4258  *                0 on no addresses written
4259  *                X on writing X addresses to MTA
4260  **/
4261 int igb_write_mc_addr_list(struct net_device *netdev)
4262 {
4263         struct igb_adapter *adapter = netdev_priv(netdev);
4264         struct e1000_hw *hw = &adapter->hw;
4265 #ifdef NETDEV_HW_ADDR_T_MULTICAST
4266         struct netdev_hw_addr *ha;
4267 #else
4268         struct dev_mc_list *ha;
4269 #endif
4270         u8  *mta_list;
4271         int i, count;
4272 #ifdef CONFIG_IGB_VMDQ_NETDEV
4273         int vm;
4274 #endif
4275         count = netdev_mc_count(netdev);
4276 #ifdef CONFIG_IGB_VMDQ_NETDEV
4277         for (vm = 1; vm < adapter->vmdq_pools; vm++) {
4278                 if (!adapter->vmdq_netdev[vm])
4279                         break;
4280                 if (!netif_running(adapter->vmdq_netdev[vm]))
4281                         continue;
4282                 count += netdev_mc_count(adapter->vmdq_netdev[vm]);
4283         }
4284 #endif
4285
4286         if (!count) {
4287                 e1000_update_mc_addr_list(hw, NULL, 0);
4288                 return 0;
4289         }
4290         mta_list = kzalloc(count * 6, GFP_ATOMIC);
4291         if (!mta_list)
4292                 return -ENOMEM;
4293
4294         /* The shared function expects a packed array of only addresses. */
4295         i = 0;
4296         netdev_for_each_mc_addr(ha, netdev)
4297 #ifdef NETDEV_HW_ADDR_T_MULTICAST
4298                 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
4299 #else
4300                 memcpy(mta_list + (i++ * ETH_ALEN), ha->dmi_addr, ETH_ALEN);
4301 #endif
4302 #ifdef CONFIG_IGB_VMDQ_NETDEV
4303         for (vm = 1; vm < adapter->vmdq_pools; vm++) {
4304                 if (!adapter->vmdq_netdev[vm])
4305                         break;
4306                 if (!netif_running(adapter->vmdq_netdev[vm]) ||
4307                     !netdev_mc_count(adapter->vmdq_netdev[vm]))
4308                         continue;
4309                 netdev_for_each_mc_addr(ha, adapter->vmdq_netdev[vm])
4310 #ifdef NETDEV_HW_ADDR_T_MULTICAST
4311                         memcpy(mta_list + (i++ * ETH_ALEN),
4312                                ha->addr, ETH_ALEN);
4313 #else
4314                         memcpy(mta_list + (i++ * ETH_ALEN),
4315                                ha->dmi_addr, ETH_ALEN);
4316 #endif
4317         }
4318 #endif
4319         e1000_update_mc_addr_list(hw, mta_list, i);
4320         kfree(mta_list);
4321
4322         return count;
4323 }
4324
4325 void igb_rar_set(struct igb_adapter *adapter, u32 index)
4326 {
4327         u32 rar_low, rar_high;
4328         struct e1000_hw *hw = &adapter->hw;
4329         u8 *addr = adapter->mac_table[index].addr;
4330         /* HW expects these in little endian so we reverse the byte order
4331          * from network order (big endian) to little endian
4332          */
4333         rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
4334                   ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
4335         rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
4336
4337         /* Indicate to hardware the Address is Valid. */
4338         if (adapter->mac_table[index].state & IGB_MAC_STATE_IN_USE)
4339                 rar_high |= E1000_RAH_AV;
4340
4341         if (hw->mac.type == e1000_82575)
4342                 rar_high |= E1000_RAH_POOL_1 * adapter->mac_table[index].queue;
4343         else
4344                 rar_high |= E1000_RAH_POOL_1 << adapter->mac_table[index].queue;
4345
4346         E1000_WRITE_REG(hw, E1000_RAL(index), rar_low);
4347         E1000_WRITE_FLUSH(hw);
4348         E1000_WRITE_REG(hw, E1000_RAH(index), rar_high);
4349         E1000_WRITE_FLUSH(hw);
4350 }
4351
4352 void igb_full_sync_mac_table(struct igb_adapter *adapter)
4353 {
4354         struct e1000_hw *hw = &adapter->hw;
4355         int i;
4356         for (i = 0; i < hw->mac.rar_entry_count; i++) {
4357                         igb_rar_set(adapter, i);
4358         }
4359 }
4360
4361 void igb_sync_mac_table(struct igb_adapter *adapter)
4362 {
4363         struct e1000_hw *hw = &adapter->hw;
4364         int i;
4365         for (i = 0; i < hw->mac.rar_entry_count; i++) {
4366                 if (adapter->mac_table[i].state & IGB_MAC_STATE_MODIFIED)
4367                         igb_rar_set(adapter, i);
4368                 adapter->mac_table[i].state &= ~(IGB_MAC_STATE_MODIFIED);
4369         }
4370 }
4371
4372 int igb_available_rars(struct igb_adapter *adapter)
4373 {
4374         struct e1000_hw *hw = &adapter->hw;
4375         int i, count = 0;
4376
4377         for (i = 0; i < hw->mac.rar_entry_count; i++) {
4378                 if (adapter->mac_table[i].state == 0)
4379                         count++;
4380         }
4381         return count;
4382 }
4383
4384 #ifdef HAVE_SET_RX_MODE
4385 /**
4386  * igb_write_uc_addr_list - write unicast addresses to RAR table
4387  * @netdev: network interface device structure
4388  *
4389  * Writes unicast address list to the RAR table.
4390  * Returns: -ENOMEM on failure/insufficient address space
4391  *                0 on no addresses written
4392  *                X on writing X addresses to the RAR table
4393  **/
4394 static int igb_write_uc_addr_list(struct net_device *netdev)
4395 {
4396         struct igb_adapter *adapter = netdev_priv(netdev);
4397         unsigned int vfn = adapter->vfs_allocated_count;
4398         int count = 0;
4399
4400         /* return ENOMEM indicating insufficient memory for addresses */
4401         if (netdev_uc_count(netdev) > igb_available_rars(adapter))
4402                 return -ENOMEM;
4403         if (!netdev_uc_empty(netdev)) {
4404 #ifdef NETDEV_HW_ADDR_T_UNICAST
4405                 struct netdev_hw_addr *ha;
4406 #else
4407                 struct dev_mc_list *ha;
4408 #endif
4409                 netdev_for_each_uc_addr(ha, netdev) {
4410 #ifdef NETDEV_HW_ADDR_T_UNICAST
4411                         igb_del_mac_filter(adapter, ha->addr, vfn);
4412                         igb_add_mac_filter(adapter, ha->addr, vfn);
4413 #else
4414                         igb_del_mac_filter(adapter, ha->da_addr, vfn);
4415                         igb_add_mac_filter(adapter, ha->da_addr, vfn);
4416 #endif
4417                         count++;
4418                 }
4419         }
4420         return count;
4421 }
4422
4423 #endif /* HAVE_SET_RX_MODE */
4424 /**
4425  * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
4426  * @netdev: network interface device structure
4427  *
4428  * The set_rx_mode entry point is called whenever the unicast or multicast
4429  * address lists or the network interface flags are updated.  This routine is
4430  * responsible for configuring the hardware for proper unicast, multicast,
4431  * promiscuous mode, and all-multi behavior.
4432  **/
4433 static void igb_set_rx_mode(struct net_device *netdev)
4434 {
4435         struct igb_adapter *adapter = netdev_priv(netdev);
4436         struct e1000_hw *hw = &adapter->hw;
4437         unsigned int vfn = adapter->vfs_allocated_count;
4438         u32 rctl, vmolr = 0;
4439         int count;
4440
4441         /* Check for Promiscuous and All Multicast modes */
4442         rctl = E1000_READ_REG(hw, E1000_RCTL);
4443
4444         /* clear the effected bits */
4445         rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
4446
4447         if (netdev->flags & IFF_PROMISC) {
4448                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
4449                 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
4450                 /* retain VLAN HW filtering if in VT mode */
4451                 if (adapter->vfs_allocated_count || adapter->vmdq_pools)
4452                         rctl |= E1000_RCTL_VFE;
4453         } else {
4454                 if (netdev->flags & IFF_ALLMULTI) {
4455                         rctl |= E1000_RCTL_MPE;
4456                         vmolr |= E1000_VMOLR_MPME;
4457                 } else {
4458                         /*
4459                          * Write addresses to the MTA, if the attempt fails
4460                          * then we should just turn on promiscuous mode so
4461                          * that we can at least receive multicast traffic
4462                          */
4463                         count = igb_write_mc_addr_list(netdev);
4464                         if (count < 0) {
4465                                 rctl |= E1000_RCTL_MPE;
4466                                 vmolr |= E1000_VMOLR_MPME;
4467                         } else if (count) {
4468                                 vmolr |= E1000_VMOLR_ROMPE;
4469                         }
4470                 }
4471 #ifdef HAVE_SET_RX_MODE
4472                 /*
4473                  * Write addresses to available RAR registers, if there is not
4474                  * sufficient space to store all the addresses then enable
4475                  * unicast promiscuous mode
4476                  */
4477                 count = igb_write_uc_addr_list(netdev);
4478                 if (count < 0) {
4479                         rctl |= E1000_RCTL_UPE;
4480                         vmolr |= E1000_VMOLR_ROPE;
4481                 }
4482 #endif /* HAVE_SET_RX_MODE */
4483                 rctl |= E1000_RCTL_VFE;
4484         }
4485         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4486
4487         /*
4488          * In order to support SR-IOV and eventually VMDq it is necessary to set
4489          * the VMOLR to enable the appropriate modes.  Without this workaround
4490          * we will have issues with VLAN tag stripping not being done for frames
4491          * that are only arriving because we are the default pool
4492          */
4493         if (hw->mac.type < e1000_82576)
4494                 return;
4495
4496         vmolr |= E1000_READ_REG(hw, E1000_VMOLR(vfn)) &
4497                  ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
4498         E1000_WRITE_REG(hw, E1000_VMOLR(vfn), vmolr);
4499         igb_restore_vf_multicasts(adapter);
4500 }
4501
4502 static void igb_check_wvbr(struct igb_adapter *adapter)
4503 {
4504         struct e1000_hw *hw = &adapter->hw;
4505         u32 wvbr = 0;
4506
4507         switch (hw->mac.type) {
4508         case e1000_82576:
4509         case e1000_i350:
4510                 if (!(wvbr = E1000_READ_REG(hw, E1000_WVBR)))
4511                         return;
4512                 break;
4513         default:
4514                 break;
4515         }
4516
4517         adapter->wvbr |= wvbr;
4518 }
4519
4520 #define IGB_STAGGERED_QUEUE_OFFSET 8
4521
4522 static void igb_spoof_check(struct igb_adapter *adapter)
4523 {
4524         int j;
4525
4526         if (!adapter->wvbr)
4527                 return;
4528
4529         switch (adapter->hw.mac.type) {
4530         case e1000_82576:
4531                 for (j = 0; j < adapter->vfs_allocated_count; j++) {
4532                         if (adapter->wvbr & (1 << j) ||
4533                             adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
4534                                 DPRINTK(DRV, WARNING,
4535                                         "Spoof event(s) detected on VF %d\n", j);
4536                                 adapter->wvbr &=
4537                                         ~((1 << j) |
4538                                           (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
4539                         }
4540                 }
4541                 break;
4542         case e1000_i350:
4543                 for (j = 0; j < adapter->vfs_allocated_count; j++) {
4544                         if (adapter->wvbr & (1 << j)) {
4545                                 DPRINTK(DRV, WARNING,
4546                                         "Spoof event(s) detected on VF %d\n", j);
4547                                 adapter->wvbr &= ~(1 << j);
4548                         }
4549                 }
4550                 break;
4551         default:
4552                 break;
4553         }
4554 }
4555
4556 /* Need to wait a few seconds after link up to get diagnostic information from
4557  * the phy */
4558 #ifdef HAVE_TIMER_SETUP
4559 static void igb_update_phy_info(struct timer_list *t)
4560 {
4561         struct igb_adapter *adapter = from_timer(adapter, t, phy_info_timer);
4562 #else
4563 static void igb_update_phy_info(unsigned long data)
4564 {
4565         struct igb_adapter *adapter = (struct igb_adapter *) data;
4566 #endif
4567         e1000_get_phy_info(&adapter->hw);
4568 }
4569
4570 /**
4571  * igb_has_link - check shared code for link and determine up/down
4572  * @adapter: pointer to driver private info
4573  **/
4574 bool igb_has_link(struct igb_adapter *adapter)
4575 {
4576         struct e1000_hw *hw = &adapter->hw;
4577         bool link_active = FALSE;
4578
4579         /* get_link_status is set on LSC (link status) interrupt or
4580          * rx sequence error interrupt.  get_link_status will stay
4581          * false until the e1000_check_for_link establishes link
4582          * for copper adapters ONLY
4583          */
4584         switch (hw->phy.media_type) {
4585         case e1000_media_type_copper:
4586                 if (!hw->mac.get_link_status)
4587                         return true;
4588         case e1000_media_type_internal_serdes:
4589                 e1000_check_for_link(hw);
4590                 link_active = !hw->mac.get_link_status;
4591                 break;
4592         case e1000_media_type_unknown:
4593         default:
4594                 break;
4595         }
4596
4597         if (((hw->mac.type == e1000_i210) ||
4598              (hw->mac.type == e1000_i211)) &&
4599              (hw->phy.id == I210_I_PHY_ID)) {
4600                 if (!netif_carrier_ok(adapter->netdev)) {
4601                         adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
4602                 } else if (!(adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)) {
4603                         adapter->flags |= IGB_FLAG_NEED_LINK_UPDATE;
4604                         adapter->link_check_timeout = jiffies;
4605                 }
4606         }
4607
4608         return link_active;
4609 }
4610
4611 /**
4612  * igb_watchdog - Timer Call-back
4613  * @data: pointer to adapter cast into an unsigned long
4614  **/
4615 #ifdef HAVE_TIMER_SETUP
4616 static void igb_watchdog(struct timer_list *t)
4617 {
4618         struct igb_adapter *adapter = from_timer(adapter, t, watchdog_timer);
4619 #else
4620 static void igb_watchdog(unsigned long data)
4621 {
4622         struct igb_adapter *adapter = (struct igb_adapter *)data;
4623 #endif
4624         /* Do the rest outside of interrupt context */
4625         schedule_work(&adapter->watchdog_task);
4626 }
4627
4628 static void igb_watchdog_task(struct work_struct *work)
4629 {
4630         struct igb_adapter *adapter = container_of(work,
4631                                                    struct igb_adapter,
4632                                                    watchdog_task);
4633         struct e1000_hw *hw = &adapter->hw;
4634         struct net_device *netdev = adapter->netdev;
4635         u32 link;
4636         int i;
4637         u32 thstat, ctrl_ext;
4638         u32 connsw;
4639
4640         link = igb_has_link(adapter);
4641         /* Force link down if we have fiber to swap to */
4642         if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
4643                 if (hw->phy.media_type == e1000_media_type_copper) {
4644                         connsw = E1000_READ_REG(hw, E1000_CONNSW);
4645                         if (!(connsw & E1000_CONNSW_AUTOSENSE_EN))
4646                                 link = 0;
4647                 }
4648         }
4649
4650         if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE) {
4651                 if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
4652                         adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
4653                 else
4654                         link = FALSE;
4655         }
4656
4657         if (link) {
4658                 /* Perform a reset if the media type changed. */
4659                 if (hw->dev_spec._82575.media_changed) {
4660                         hw->dev_spec._82575.media_changed = false;
4661                         adapter->flags |= IGB_FLAG_MEDIA_RESET;
4662                         igb_reset(adapter);
4663                 }
4664
4665                 /* Cancel scheduled suspend requests. */
4666                 pm_runtime_resume(netdev->dev.parent);
4667
4668                 if (!netif_carrier_ok(netdev)) {
4669                         u32 ctrl;
4670                         e1000_get_speed_and_duplex(hw,
4671                                                    &adapter->link_speed,
4672                                                    &adapter->link_duplex);
4673
4674                         ctrl = E1000_READ_REG(hw, E1000_CTRL);
4675                         /* Links status message must follow this format */
4676                         printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
4677                                  "Flow Control: %s\n",
4678                                netdev->name,
4679                                adapter->link_speed,
4680                                adapter->link_duplex == FULL_DUPLEX ?
4681                                  "Full Duplex" : "Half Duplex",
4682                                ((ctrl & E1000_CTRL_TFCE) &&
4683                                 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX":
4684                                ((ctrl & E1000_CTRL_RFCE) ?  "RX" :
4685                                ((ctrl & E1000_CTRL_TFCE) ?  "TX" : "None")));
4686                         /* adjust timeout factor according to speed/duplex */
4687                         adapter->tx_timeout_factor = 1;
4688                         switch (adapter->link_speed) {
4689                         case SPEED_10:
4690                                 adapter->tx_timeout_factor = 14;
4691                                 break;
4692                         case SPEED_100:
4693                                 /* maybe add some timeout factor ? */
4694                                 break;
4695                         default:
4696                                 break;
4697                         }
4698
4699                         netif_carrier_on(netdev);
4700                         netif_tx_wake_all_queues(netdev);
4701
4702                         igb_ping_all_vfs(adapter);
4703 #ifdef IFLA_VF_MAX
4704                         igb_check_vf_rate_limit(adapter);
4705 #endif /* IFLA_VF_MAX */
4706
4707                         /* link state has changed, schedule phy info update */
4708                         if (!test_bit(__IGB_DOWN, &adapter->state))
4709                                 mod_timer(&adapter->phy_info_timer,
4710                                           round_jiffies(jiffies + 2 * HZ));
4711                 }
4712         } else {
4713                 if (netif_carrier_ok(netdev)) {
4714                         adapter->link_speed = 0;
4715                         adapter->link_duplex = 0;
4716                         /* check for thermal sensor event on i350 */
4717                         if (hw->mac.type == e1000_i350) {
4718                                 thstat = E1000_READ_REG(hw, E1000_THSTAT);
4719                                 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
4720                                 if ((hw->phy.media_type ==
4721                                         e1000_media_type_copper) &&
4722                                         !(ctrl_ext &
4723                                         E1000_CTRL_EXT_LINK_MODE_SGMII)) {
4724                                         if (thstat & E1000_THSTAT_PWR_DOWN) {
4725                                                 printk(KERN_ERR "igb: %s The "
4726                                                 "network adapter was stopped "
4727                                                 "because it overheated.\n",
4728                                                 netdev->name);
4729                                         }
4730                                         if (thstat & E1000_THSTAT_LINK_THROTTLE) {
4731                                                 printk(KERN_INFO
4732                                                         "igb: %s The network "
4733                                                         "adapter supported "
4734                                                         "link speed "
4735                                                         "was downshifted "
4736                                                         "because it "
4737                                                         "overheated.\n",
4738                                                         netdev->name);
4739                                         }
4740                                 }
4741                         }
4742
4743                         /* Links status message must follow this format */
4744                         printk(KERN_INFO "igb: %s NIC Link is Down\n",
4745                                netdev->name);
4746                         netif_carrier_off(netdev);
4747                         netif_tx_stop_all_queues(netdev);
4748
4749                         igb_ping_all_vfs(adapter);
4750
4751                         /* link state has changed, schedule phy info update */
4752                         if (!test_bit(__IGB_DOWN, &adapter->state))
4753                                 mod_timer(&adapter->phy_info_timer,
4754                                           round_jiffies(jiffies + 2 * HZ));
4755                         /* link is down, time to check for alternate media */
4756                         if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
4757                                 igb_check_swap_media(adapter);
4758                                 if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
4759                                         schedule_work(&adapter->reset_task);
4760                                         /* return immediately */
4761                                         return;
4762                                 }
4763                         }
4764                         pm_schedule_suspend(netdev->dev.parent,
4765                                             MSEC_PER_SEC * 5);
4766
4767                 /* also check for alternate media here */
4768                 } else if (!netif_carrier_ok(netdev) &&
4769                            (adapter->flags & IGB_FLAG_MAS_ENABLE)) {
4770                         hw->mac.ops.power_up_serdes(hw);
4771                         igb_check_swap_media(adapter);
4772                         if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
4773                                 schedule_work(&adapter->reset_task);
4774                                 /* return immediately */
4775                                 return;
4776                         }
4777                 }
4778         }
4779
4780         igb_update_stats(adapter);
4781
4782         for (i = 0; i < adapter->num_tx_queues; i++) {
4783                 struct igb_ring *tx_ring = adapter->tx_ring[i];
4784                 if (!netif_carrier_ok(netdev)) {
4785                         /* We've lost link, so the controller stops DMA,
4786                          * but we've got queued Tx work that's never going
4787                          * to get done, so reset controller to flush Tx.
4788                          * (Do the reset outside of interrupt context). */
4789                         if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
4790                                 adapter->tx_timeout_count++;
4791                                 schedule_work(&adapter->reset_task);
4792                                 /* return immediately since reset is imminent */
4793                                 return;
4794                         }
4795                 }
4796
4797                 /* Force detection of hung controller every watchdog period */
4798                 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
4799         }
4800
4801         /* Cause software interrupt to ensure rx ring is cleaned */
4802         if (adapter->msix_entries) {
4803                 u32 eics = 0;
4804                 for (i = 0; i < adapter->num_q_vectors; i++)
4805                         eics |= adapter->q_vector[i]->eims_value;
4806                 E1000_WRITE_REG(hw, E1000_EICS, eics);
4807         } else {
4808                 E1000_WRITE_REG(hw, E1000_ICS, E1000_ICS_RXDMT0);
4809         }
4810
4811         igb_spoof_check(adapter);
4812
4813         /* Reset the timer */
4814         if (!test_bit(__IGB_DOWN, &adapter->state)) {
4815                 if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)
4816                         mod_timer(&adapter->watchdog_timer,
4817                                   round_jiffies(jiffies +  HZ));
4818                 else
4819                         mod_timer(&adapter->watchdog_timer,
4820                                   round_jiffies(jiffies + 2 * HZ));
4821         }
4822 }
4823
4824 static void igb_dma_err_task(struct work_struct *work)
4825 {
4826         struct igb_adapter *adapter = container_of(work,
4827                                                    struct igb_adapter,
4828                                                    dma_err_task);
4829         int vf;
4830         struct e1000_hw *hw = &adapter->hw;
4831         struct net_device *netdev = adapter->netdev;
4832         u32 hgptc;
4833         u32 ciaa, ciad;
4834
4835         hgptc = E1000_READ_REG(hw, E1000_HGPTC);
4836         if (hgptc) /* If incrementing then no need for the check below */
4837                 goto dma_timer_reset;
4838         /*
4839          * Check to see if a bad DMA write target from an errant or
4840          * malicious VF has caused a PCIe error.  If so then we can
4841          * issue a VFLR to the offending VF(s) and then resume without
4842          * requesting a full slot reset.
4843          */
4844
4845         for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4846                 ciaa = (vf << 16) | 0x80000000;
4847                 /* 32 bit read so align, we really want status at offset 6 */
4848                 ciaa |= PCI_COMMAND;
4849                 E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4850                 ciad = E1000_READ_REG(hw, E1000_CIAD);
4851                 ciaa &= 0x7FFFFFFF;
4852                 /* disable debug mode asap after reading data */
4853                 E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4854                 /* Get the upper 16 bits which will be the PCI status reg */
4855                 ciad >>= 16;
4856                 if (ciad & (PCI_STATUS_REC_MASTER_ABORT |
4857                             PCI_STATUS_REC_TARGET_ABORT |
4858                             PCI_STATUS_SIG_SYSTEM_ERROR)) {
4859                         netdev_err(netdev, "VF %d suffered error\n", vf);
4860                         /* Issue VFLR */
4861                         ciaa = (vf << 16) | 0x80000000;
4862                         ciaa |= 0xA8;
4863                         E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4864                         ciad = 0x00008000;  /* VFLR */
4865                         E1000_WRITE_REG(hw, E1000_CIAD, ciad);
4866                         ciaa &= 0x7FFFFFFF;
4867                         E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4868                 }
4869         }
4870 dma_timer_reset:
4871         /* Reset the timer */
4872         if (!test_bit(__IGB_DOWN, &adapter->state))
4873                 mod_timer(&adapter->dma_err_timer,
4874                           round_jiffies(jiffies + HZ / 10));
4875 }
4876
4877 /**
4878  * igb_dma_err_timer - Timer Call-back
4879  * @data: pointer to adapter cast into an unsigned long
4880  **/
4881 #ifdef HAVE_TIMER_SETUP
4882 static void igb_dma_err_timer(struct timer_list *t)
4883 {
4884         struct igb_adapter *adapter = from_timer(adapter, t, dma_err_timer);
4885 #else
4886 static void igb_dma_err_timer(unsigned long data)
4887 {
4888         struct igb_adapter *adapter = (struct igb_adapter *)data;
4889 #endif
4890         /* Do the rest outside of interrupt context */
4891         schedule_work(&adapter->dma_err_task);
4892 }
4893
4894 enum latency_range {
4895         lowest_latency = 0,
4896         low_latency = 1,
4897         bulk_latency = 2,
4898         latency_invalid = 255
4899 };
4900
4901 /**
4902  * igb_update_ring_itr - update the dynamic ITR value based on packet size
4903  *
4904  *      Stores a new ITR value based on strictly on packet size.  This
4905  *      algorithm is less sophisticated than that used in igb_update_itr,
4906  *      due to the difficulty of synchronizing statistics across multiple
4907  *      receive rings.  The divisors and thresholds used by this function
4908  *      were determined based on theoretical maximum wire speed and testing
4909  *      data, in order to minimize response time while increasing bulk
4910  *      throughput.
4911  *      This functionality is controlled by the InterruptThrottleRate module
4912  *      parameter (see igb_param.c)
4913  *      NOTE:  This function is called only when operating in a multiqueue
4914  *             receive environment.
4915  * @q_vector: pointer to q_vector
4916  **/
4917 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
4918 {
4919         int new_val = q_vector->itr_val;
4920         int avg_wire_size = 0;
4921         struct igb_adapter *adapter = q_vector->adapter;
4922         unsigned int packets;
4923
4924         /* For non-gigabit speeds, just fix the interrupt rate at 4000
4925          * ints/sec - ITR timer value of 120 ticks.
4926          */
4927         switch (adapter->link_speed) {
4928         case SPEED_10:
4929         case SPEED_100:
4930                 new_val = IGB_4K_ITR;
4931                 goto set_itr_val;
4932         default:
4933                 break;
4934         }
4935
4936         packets = q_vector->rx.total_packets;
4937         if (packets)
4938                 avg_wire_size = q_vector->rx.total_bytes / packets;
4939
4940         packets = q_vector->tx.total_packets;
4941         if (packets)
4942                 avg_wire_size = max_t(u32, avg_wire_size,
4943                                       q_vector->tx.total_bytes / packets);
4944
4945         /* if avg_wire_size isn't set no work was done */
4946         if (!avg_wire_size)
4947                 goto clear_counts;
4948
4949         /* Add 24 bytes to size to account for CRC, preamble, and gap */
4950         avg_wire_size += 24;
4951
4952         /* Don't starve jumbo frames */
4953         avg_wire_size = min(avg_wire_size, 3000);
4954
4955         /* Give a little boost to mid-size frames */
4956         if ((avg_wire_size > 300) && (avg_wire_size < 1200))
4957                 new_val = avg_wire_size / 3;
4958         else
4959                 new_val = avg_wire_size / 2;
4960
4961         /* conservative mode (itr 3) eliminates the lowest_latency setting */
4962         if (new_val < IGB_20K_ITR &&
4963             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4964              (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4965                 new_val = IGB_20K_ITR;
4966
4967 set_itr_val:
4968         if (new_val != q_vector->itr_val) {
4969                 q_vector->itr_val = new_val;
4970                 q_vector->set_itr = 1;
4971         }
4972 clear_counts:
4973         q_vector->rx.total_bytes = 0;
4974         q_vector->rx.total_packets = 0;
4975         q_vector->tx.total_bytes = 0;
4976         q_vector->tx.total_packets = 0;
4977 }
4978
4979 /**
4980  * igb_update_itr - update the dynamic ITR value based on statistics
4981  *      Stores a new ITR value based on packets and byte
4982  *      counts during the last interrupt.  The advantage of per interrupt
4983  *      computation is faster updates and more accurate ITR for the current
4984  *      traffic pattern.  Constants in this function were computed
4985  *      based on theoretical maximum wire speed and thresholds were set based
4986  *      on testing data as well as attempting to minimize response time
4987  *      while increasing bulk throughput.
4988  *      this functionality is controlled by the InterruptThrottleRate module
4989  *      parameter (see igb_param.c)
4990  *      NOTE:  These calculations are only valid when operating in a single-
4991  *             queue environment.
4992  * @q_vector: pointer to q_vector
4993  * @ring_container: ring info to update the itr for
4994  **/
4995 static void igb_update_itr(struct igb_q_vector *q_vector,
4996                            struct igb_ring_container *ring_container)
4997 {
4998         unsigned int packets = ring_container->total_packets;
4999         unsigned int bytes = ring_container->total_bytes;
5000         u8 itrval = ring_container->itr;
5001
5002         /* no packets, exit with status unchanged */
5003         if (packets == 0)
5004                 return;
5005
5006         switch (itrval) {
5007         case lowest_latency:
5008                 /* handle TSO and jumbo frames */
5009                 if (bytes/packets > 8000)
5010                         itrval = bulk_latency;
5011                 else if ((packets < 5) && (bytes > 512))
5012                         itrval = low_latency;
5013                 break;
5014         case low_latency:  /* 50 usec aka 20000 ints/s */
5015                 if (bytes > 10000) {
5016                         /* this if handles the TSO accounting */
5017                         if (bytes/packets > 8000) {
5018                                 itrval = bulk_latency;
5019                         } else if ((packets < 10) || ((bytes/packets) > 1200)) {
5020                                 itrval = bulk_latency;
5021                         } else if (packets > 35) {
5022                                 itrval = lowest_latency;
5023                         }
5024                 } else if (bytes/packets > 2000) {
5025                         itrval = bulk_latency;
5026                 } else if (packets <= 2 && bytes < 512) {
5027                         itrval = lowest_latency;
5028                 }
5029                 break;
5030         case bulk_latency: /* 250 usec aka 4000 ints/s */
5031                 if (bytes > 25000) {
5032                         if (packets > 35)
5033                                 itrval = low_latency;
5034                 } else if (bytes < 1500) {
5035                         itrval = low_latency;
5036                 }
5037                 break;
5038         }
5039
5040         /* clear work counters since we have the values we need */
5041         ring_container->total_bytes = 0;
5042         ring_container->total_packets = 0;
5043
5044         /* write updated itr to ring container */
5045         ring_container->itr = itrval;
5046 }
5047
5048 static void igb_set_itr(struct igb_q_vector *q_vector)
5049 {
5050         struct igb_adapter *adapter = q_vector->adapter;
5051         u32 new_itr = q_vector->itr_val;
5052         u8 current_itr = 0;
5053
5054         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
5055         switch (adapter->link_speed) {
5056         case SPEED_10:
5057         case SPEED_100:
5058                 current_itr = 0;
5059                 new_itr = IGB_4K_ITR;
5060                 goto set_itr_now;
5061         default:
5062                 break;
5063         }
5064
5065         igb_update_itr(q_vector, &q_vector->tx);
5066         igb_update_itr(q_vector, &q_vector->rx);
5067
5068         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
5069
5070         /* conservative mode (itr 3) eliminates the lowest_latency setting */
5071         if (current_itr == lowest_latency &&
5072             ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
5073              (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
5074                 current_itr = low_latency;
5075
5076         switch (current_itr) {
5077         /* counts and packets in update_itr are dependent on these numbers */
5078         case lowest_latency:
5079                 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
5080                 break;
5081         case low_latency:
5082                 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
5083                 break;
5084         case bulk_latency:
5085                 new_itr = IGB_4K_ITR;  /* 4,000 ints/sec */
5086                 break;
5087         default:
5088                 break;
5089         }
5090
5091 set_itr_now:
5092         if (new_itr != q_vector->itr_val) {
5093                 /* this attempts to bias the interrupt rate towards Bulk
5094                  * by adding intermediate steps when interrupt rate is
5095                  * increasing */
5096                 new_itr = new_itr > q_vector->itr_val ?
5097                              max((new_itr * q_vector->itr_val) /
5098                                  (new_itr + (q_vector->itr_val >> 2)),
5099                                  new_itr) :
5100                              new_itr;
5101                 /* Don't write the value here; it resets the adapter's
5102                  * internal timer, and causes us to delay far longer than
5103                  * we should between interrupts.  Instead, we write the ITR
5104                  * value at the beginning of the next interrupt so the timing
5105                  * ends up being correct.
5106                  */
5107                 q_vector->itr_val = new_itr;
5108                 q_vector->set_itr = 1;
5109         }
5110 }
5111
5112 void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
5113                      u32 type_tucmd, u32 mss_l4len_idx)
5114 {
5115         struct e1000_adv_tx_context_desc *context_desc;
5116         u16 i = tx_ring->next_to_use;
5117
5118         context_desc = IGB_TX_CTXTDESC(tx_ring, i);
5119
5120         i++;
5121         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
5122
5123         /* set bits to identify this as an advanced context descriptor */
5124         type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
5125
5126         /* For 82575, context index must be unique per ring. */
5127         if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
5128                 mss_l4len_idx |= tx_ring->reg_idx << 4;
5129
5130         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
5131         context_desc->seqnum_seed       = 0;
5132         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
5133         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
5134 }
5135
5136 static int igb_tso(struct igb_ring *tx_ring,
5137                    struct igb_tx_buffer *first,
5138                    u8 *hdr_len)
5139 {
5140 #ifdef NETIF_F_TSO
5141         struct sk_buff *skb = first->skb;
5142         u32 vlan_macip_lens, type_tucmd;
5143         u32 mss_l4len_idx, l4len;
5144
5145         if (skb->ip_summed != CHECKSUM_PARTIAL)
5146                 return 0;
5147
5148         if (!skb_is_gso(skb))
5149 #endif /* NETIF_F_TSO */
5150                 return 0;
5151 #ifdef NETIF_F_TSO
5152
5153         if (skb_header_cloned(skb)) {
5154                 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
5155                 if (err)
5156                         return err;
5157         }
5158
5159         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
5160         type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
5161
5162         if (first->protocol == __constant_htons(ETH_P_IP)) {
5163                 struct iphdr *iph = ip_hdr(skb);
5164                 iph->tot_len = 0;
5165                 iph->check = 0;
5166                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
5167                                                          iph->daddr, 0,
5168                                                          IPPROTO_TCP,
5169                                                          0);
5170                 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
5171                 first->tx_flags |= IGB_TX_FLAGS_TSO |
5172                                    IGB_TX_FLAGS_CSUM |
5173                                    IGB_TX_FLAGS_IPV4;
5174 #ifdef NETIF_F_TSO6
5175         } else if (skb_is_gso_v6(skb)) {
5176                 ipv6_hdr(skb)->payload_len = 0;
5177                 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
5178                                                        &ipv6_hdr(skb)->daddr,
5179                                                        0, IPPROTO_TCP, 0);
5180                 first->tx_flags |= IGB_TX_FLAGS_TSO |
5181                                    IGB_TX_FLAGS_CSUM;
5182 #endif
5183         }
5184
5185         /* compute header lengths */
5186         l4len = tcp_hdrlen(skb);
5187         *hdr_len = skb_transport_offset(skb) + l4len;
5188
5189         /* update gso size and bytecount with header size */
5190         first->gso_segs = skb_shinfo(skb)->gso_segs;
5191         first->bytecount += (first->gso_segs - 1) * *hdr_len;
5192
5193         /* MSS L4LEN IDX */
5194         mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
5195         mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
5196
5197         /* VLAN MACLEN IPLEN */
5198         vlan_macip_lens = skb_network_header_len(skb);
5199         vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
5200         vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
5201
5202         igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
5203
5204         return 1;
5205 #endif  /* NETIF_F_TSO */
5206 }
5207
5208 static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
5209 {
5210         struct sk_buff *skb = first->skb;
5211         u32 vlan_macip_lens = 0;
5212         u32 mss_l4len_idx = 0;
5213         u32 type_tucmd = 0;
5214
5215         if (skb->ip_summed != CHECKSUM_PARTIAL) {
5216                 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
5217                         return;
5218         } else {
5219                 u8 nexthdr = 0;
5220                 switch (first->protocol) {
5221                 case __constant_htons(ETH_P_IP):
5222                         vlan_macip_lens |= skb_network_header_len(skb);
5223                         type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
5224                         nexthdr = ip_hdr(skb)->protocol;
5225                         break;
5226 #ifdef NETIF_F_IPV6_CSUM
5227                 case __constant_htons(ETH_P_IPV6):
5228                         vlan_macip_lens |= skb_network_header_len(skb);
5229                         nexthdr = ipv6_hdr(skb)->nexthdr;
5230                         break;
5231 #endif
5232                 default:
5233                         if (unlikely(net_ratelimit())) {
5234                                 dev_warn(tx_ring->dev,
5235                                  "partial checksum but proto=%x!\n",
5236                                  first->protocol);
5237                         }
5238                         break;
5239                 }
5240
5241                 switch (nexthdr) {
5242                 case IPPROTO_TCP:
5243                         type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
5244                         mss_l4len_idx = tcp_hdrlen(skb) <<
5245                                         E1000_ADVTXD_L4LEN_SHIFT;
5246                         break;
5247 #ifdef HAVE_SCTP
5248                 case IPPROTO_SCTP:
5249                         type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
5250                         mss_l4len_idx = sizeof(struct sctphdr) <<
5251                                         E1000_ADVTXD_L4LEN_SHIFT;
5252                         break;
5253 #endif
5254                 case IPPROTO_UDP:
5255                         mss_l4len_idx = sizeof(struct udphdr) <<
5256                                         E1000_ADVTXD_L4LEN_SHIFT;
5257                         break;
5258                 default:
5259                         if (unlikely(net_ratelimit())) {
5260                                 dev_warn(tx_ring->dev,
5261                                  "partial checksum but l4 proto=%x!\n",
5262                                  nexthdr);
5263                         }
5264                         break;
5265                 }
5266
5267                 /* update TX checksum flag */
5268                 first->tx_flags |= IGB_TX_FLAGS_CSUM;
5269         }
5270
5271         vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
5272         vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
5273
5274         igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
5275 }
5276
5277 #define IGB_SET_FLAG(_input, _flag, _result) \
5278         ((_flag <= _result) ? \
5279          ((u32)(_input & _flag) * (_result / _flag)) : \
5280          ((u32)(_input & _flag) / (_flag / _result)))
5281
5282 static u32 igb_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
5283 {
5284         /* set type for advanced descriptor with frame checksum insertion */
5285         u32 cmd_type = E1000_ADVTXD_DTYP_DATA |
5286                        E1000_ADVTXD_DCMD_DEXT |
5287                        E1000_ADVTXD_DCMD_IFCS;
5288
5289         /* set HW vlan bit if vlan is present */
5290         cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_VLAN,
5291                                  (E1000_ADVTXD_DCMD_VLE));
5292
5293         /* set segmentation bits for TSO */
5294         cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSO,
5295                                  (E1000_ADVTXD_DCMD_TSE));
5296
5297         /* set timestamp bit if present */
5298         cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSTAMP,
5299                                  (E1000_ADVTXD_MAC_TSTAMP));
5300
5301         return cmd_type;
5302 }
5303
5304 static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
5305                                  union e1000_adv_tx_desc *tx_desc,
5306                                  u32 tx_flags, unsigned int paylen)
5307 {
5308         u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
5309
5310         /* 82575 requires a unique index per ring */
5311         if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
5312                 olinfo_status |= tx_ring->reg_idx << 4;
5313
5314         /* insert L4 checksum */
5315         olinfo_status |= IGB_SET_FLAG(tx_flags,
5316                                       IGB_TX_FLAGS_CSUM,
5317                                       (E1000_TXD_POPTS_TXSM << 8));
5318
5319         /* insert IPv4 checksum */
5320         olinfo_status |= IGB_SET_FLAG(tx_flags,
5321                                       IGB_TX_FLAGS_IPV4,
5322                                       (E1000_TXD_POPTS_IXSM << 8));
5323
5324         tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
5325 }
5326
5327 static void igb_tx_map(struct igb_ring *tx_ring,
5328                        struct igb_tx_buffer *first,
5329                        const u8 hdr_len)
5330 {
5331         struct sk_buff *skb = first->skb;
5332         struct igb_tx_buffer *tx_buffer;
5333         union e1000_adv_tx_desc *tx_desc;
5334         struct skb_frag_struct *frag;
5335         dma_addr_t dma;
5336         unsigned int data_len, size;
5337         u32 tx_flags = first->tx_flags;
5338         u32 cmd_type = igb_tx_cmd_type(skb, tx_flags);
5339         u16 i = tx_ring->next_to_use;
5340
5341         tx_desc = IGB_TX_DESC(tx_ring, i);
5342
5343         igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
5344
5345         size = skb_headlen(skb);
5346         data_len = skb->data_len;
5347
5348         dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
5349
5350         tx_buffer = first;
5351
5352         for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
5353                 if (dma_mapping_error(tx_ring->dev, dma))
5354                         goto dma_error;
5355
5356                 /* record length, and DMA address */
5357                 dma_unmap_len_set(tx_buffer, len, size);
5358                 dma_unmap_addr_set(tx_buffer, dma, dma);
5359
5360                 tx_desc->read.buffer_addr = cpu_to_le64(dma);
5361
5362                 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
5363                         tx_desc->read.cmd_type_len =
5364                                 cpu_to_le32(cmd_type ^ IGB_MAX_DATA_PER_TXD);
5365
5366                         i++;
5367                         tx_desc++;
5368                         if (i == tx_ring->count) {
5369                                 tx_desc = IGB_TX_DESC(tx_ring, 0);
5370                                 i = 0;
5371                         }
5372                         tx_desc->read.olinfo_status = 0;
5373
5374                         dma += IGB_MAX_DATA_PER_TXD;
5375                         size -= IGB_MAX_DATA_PER_TXD;
5376
5377                         tx_desc->read.buffer_addr = cpu_to_le64(dma);
5378                 }
5379
5380                 if (likely(!data_len))
5381                         break;
5382
5383                 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
5384
5385                 i++;
5386                 tx_desc++;
5387                 if (i == tx_ring->count) {
5388                         tx_desc = IGB_TX_DESC(tx_ring, 0);
5389                         i = 0;
5390                 }
5391                 tx_desc->read.olinfo_status = 0;
5392
5393                 size = skb_frag_size(frag);
5394                 data_len -= size;
5395
5396                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
5397                                        size, DMA_TO_DEVICE);
5398
5399                 tx_buffer = &tx_ring->tx_buffer_info[i];
5400         }
5401
5402         /* write last descriptor with RS and EOP bits */
5403         cmd_type |= size | IGB_TXD_DCMD;
5404         tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
5405
5406         netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
5407         /* set the timestamp */
5408         first->time_stamp = jiffies;
5409
5410         /*
5411          * Force memory writes to complete before letting h/w know there
5412          * are new descriptors to fetch.  (Only applicable for weak-ordered
5413          * memory model archs, such as IA-64).
5414          *
5415          * We also need this memory barrier to make certain all of the
5416          * status bits have been updated before next_to_watch is written.
5417          */
5418         wmb();
5419
5420         /* set next_to_watch value indicating a packet is present */
5421         first->next_to_watch = tx_desc;
5422
5423         i++;
5424         if (i == tx_ring->count)
5425                 i = 0;
5426
5427         tx_ring->next_to_use = i;
5428
5429         writel(i, tx_ring->tail);
5430
5431         /* we need this if more than one processor can write to our tail
5432          * at a time, it syncronizes IO on IA64/Altix systems */
5433         mmiowb();
5434
5435         return;
5436
5437 dma_error:
5438         dev_err(tx_ring->dev, "TX DMA map failed\n");
5439
5440         /* clear dma mappings for failed tx_buffer_info map */
5441         for (;;) {
5442                 tx_buffer = &tx_ring->tx_buffer_info[i];
5443                 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer);
5444                 if (tx_buffer == first)
5445                         break;
5446                 if (i == 0)
5447                         i = tx_ring->count;
5448                 i--;
5449         }
5450
5451         tx_ring->next_to_use = i;
5452 }
5453
5454 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
5455 {
5456         struct net_device *netdev = netdev_ring(tx_ring);
5457
5458         if (netif_is_multiqueue(netdev))
5459                 netif_stop_subqueue(netdev, ring_queue_index(tx_ring));
5460         else
5461                 netif_stop_queue(netdev);
5462
5463         /* Herbert's original patch had:
5464          *  smp_mb__after_netif_stop_queue();
5465          * but since that doesn't exist yet, just open code it. */
5466         smp_mb();
5467
5468         /* We need to check again in a case another CPU has just
5469          * made room available. */
5470         if (igb_desc_unused(tx_ring) < size)
5471                 return -EBUSY;
5472
5473         /* A reprieve! */
5474         if (netif_is_multiqueue(netdev))
5475                 netif_wake_subqueue(netdev, ring_queue_index(tx_ring));
5476         else
5477                 netif_wake_queue(netdev);
5478
5479         tx_ring->tx_stats.restart_queue++;
5480
5481         return 0;
5482 }
5483
5484 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
5485 {
5486         if (igb_desc_unused(tx_ring) >= size)
5487                 return 0;
5488         return __igb_maybe_stop_tx(tx_ring, size);
5489 }
5490
5491 netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
5492                                 struct igb_ring *tx_ring)
5493 {
5494         struct igb_tx_buffer *first;
5495         int tso;
5496         u32 tx_flags = 0;
5497 #if PAGE_SIZE > IGB_MAX_DATA_PER_TXD
5498         unsigned short f;
5499 #endif
5500         u16 count = TXD_USE_COUNT(skb_headlen(skb));
5501         __be16 protocol = vlan_get_protocol(skb);
5502         u8 hdr_len = 0;
5503
5504         /*
5505          * need: 1 descriptor per page * PAGE_SIZE/IGB_MAX_DATA_PER_TXD,
5506          *       + 1 desc for skb_headlen/IGB_MAX_DATA_PER_TXD,
5507          *       + 2 desc gap to keep tail from touching head,
5508          *       + 1 desc for context descriptor,
5509          * otherwise try next time
5510          */
5511 #if PAGE_SIZE > IGB_MAX_DATA_PER_TXD
5512         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
5513                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
5514 #else
5515         count += skb_shinfo(skb)->nr_frags;
5516 #endif
5517         if (igb_maybe_stop_tx(tx_ring, count + 3)) {
5518                 /* this is a hard error */
5519                 return NETDEV_TX_BUSY;
5520         }
5521
5522         /* record the location of the first descriptor for this packet */
5523         first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
5524         first->skb = skb;
5525         first->bytecount = skb->len;
5526         first->gso_segs = 1;
5527
5528         skb_tx_timestamp(skb);
5529
5530 #ifdef HAVE_PTP_1588_CLOCK
5531         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
5532                 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
5533                 if (!adapter->ptp_tx_skb) {
5534                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
5535                         tx_flags |= IGB_TX_FLAGS_TSTAMP;
5536
5537                         adapter->ptp_tx_skb = skb_get(skb);
5538                         adapter->ptp_tx_start = jiffies;
5539                         if (adapter->hw.mac.type == e1000_82576)
5540                                 schedule_work(&adapter->ptp_tx_work);
5541                 }
5542         }
5543 #endif /* HAVE_PTP_1588_CLOCK */
5544
5545         if (vlan_tx_tag_present(skb)) {
5546                 tx_flags |= IGB_TX_FLAGS_VLAN;
5547                 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
5548         }
5549
5550         /* record initial flags and protocol */
5551         first->tx_flags = tx_flags;
5552         first->protocol = protocol;
5553
5554         tso = igb_tso(tx_ring, first, &hdr_len);
5555         if (tso < 0)
5556                 goto out_drop;
5557         else if (!tso)
5558                 igb_tx_csum(tx_ring, first);
5559
5560         igb_tx_map(tx_ring, first, hdr_len);
5561
5562 #ifndef HAVE_TRANS_START_IN_QUEUE
5563         netdev_ring(tx_ring)->trans_start = jiffies;
5564
5565 #endif
5566         /* Make sure there is space in the ring for the next send. */
5567         igb_maybe_stop_tx(tx_ring, DESC_NEEDED);
5568
5569         return NETDEV_TX_OK;
5570
5571 out_drop:
5572         igb_unmap_and_free_tx_resource(tx_ring, first);
5573
5574         return NETDEV_TX_OK;
5575 }
5576
5577 #ifdef HAVE_TX_MQ
5578 static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
5579                                                     struct sk_buff *skb)
5580 {
5581         unsigned int r_idx = skb->queue_mapping;
5582
5583         if (r_idx >= adapter->num_tx_queues)
5584                 r_idx = r_idx % adapter->num_tx_queues;
5585
5586         return adapter->tx_ring[r_idx];
5587 }
5588 #else
5589 #define igb_tx_queue_mapping(_adapter, _skb) (_adapter)->tx_ring[0]
5590 #endif
5591
5592 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
5593                                   struct net_device *netdev)
5594 {
5595         struct igb_adapter *adapter = netdev_priv(netdev);
5596
5597         if (test_bit(__IGB_DOWN, &adapter->state)) {
5598                 dev_kfree_skb_any(skb);
5599                 return NETDEV_TX_OK;
5600         }
5601
5602         if (skb->len <= 0) {
5603                 dev_kfree_skb_any(skb);
5604                 return NETDEV_TX_OK;
5605         }
5606
5607         /*
5608          * The minimum packet size with TCTL.PSP set is 17 so pad the skb
5609          * in order to meet this minimum size requirement.
5610          */
5611         if (skb->len < 17) {
5612                 if (skb_padto(skb, 17))
5613                         return NETDEV_TX_OK;
5614                 skb->len = 17;
5615         }
5616
5617         return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
5618 }
5619
5620 /**
5621  * igb_tx_timeout - Respond to a Tx Hang
5622  * @netdev: network interface device structure
5623  **/
5624 static void igb_tx_timeout(struct net_device *netdev)
5625 {
5626         struct igb_adapter *adapter = netdev_priv(netdev);
5627         struct e1000_hw *hw = &adapter->hw;
5628
5629         /* Do the reset outside of interrupt context */
5630         adapter->tx_timeout_count++;
5631
5632         if (hw->mac.type >= e1000_82580)
5633                 hw->dev_spec._82575.global_device_reset = true;
5634
5635         schedule_work(&adapter->reset_task);
5636         E1000_WRITE_REG(hw, E1000_EICS,
5637                         (adapter->eims_enable_mask & ~adapter->eims_other));
5638 }
5639
5640 static void igb_reset_task(struct work_struct *work)
5641 {
5642         struct igb_adapter *adapter;
5643         adapter = container_of(work, struct igb_adapter, reset_task);
5644
5645         igb_reinit_locked(adapter);
5646 }
5647
5648 /**
5649  * igb_get_stats - Get System Network Statistics
5650  * @netdev: network interface device structure
5651  *
5652  * Returns the address of the device statistics structure.
5653  * The statistics are updated here and also from the timer callback.
5654  **/
5655 static struct net_device_stats *igb_get_stats(struct net_device *netdev)
5656 {
5657         struct igb_adapter *adapter = netdev_priv(netdev);
5658
5659         if (!test_bit(__IGB_RESETTING, &adapter->state))
5660                 igb_update_stats(adapter);
5661
5662 #ifdef HAVE_NETDEV_STATS_IN_NETDEV
5663         /* only return the current stats */
5664         return &netdev->stats;
5665 #else
5666         /* only return the current stats */
5667         return &adapter->net_stats;
5668 #endif /* HAVE_NETDEV_STATS_IN_NETDEV */
5669 }
5670
5671 /**
5672  * igb_change_mtu - Change the Maximum Transfer Unit
5673  * @netdev: network interface device structure
5674  * @new_mtu: new value for maximum frame size
5675  *
5676  * Returns 0 on success, negative on failure
5677  **/
5678 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
5679 {
5680         struct igb_adapter *adapter = netdev_priv(netdev);
5681         struct e1000_hw *hw = &adapter->hw;
5682         struct pci_dev *pdev = adapter->pdev;
5683         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
5684
5685         if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
5686                 dev_err(pci_dev_to_dev(pdev), "Invalid MTU setting\n");
5687                 return -EINVAL;
5688         }
5689
5690 #define MAX_STD_JUMBO_FRAME_SIZE 9238
5691         if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
5692                 dev_err(pci_dev_to_dev(pdev), "MTU > 9216 not supported.\n");
5693                 return -EINVAL;
5694         }
5695
5696         /* adjust max frame to be at least the size of a standard frame */
5697         if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
5698                 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
5699
5700         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
5701                 usleep_range(1000, 2000);
5702
5703         /* igb_down has a dependency on max_frame_size */
5704         adapter->max_frame_size = max_frame;
5705
5706         if (netif_running(netdev))
5707                 igb_down(adapter);
5708
5709         dev_info(pci_dev_to_dev(pdev), "changing MTU from %d to %d\n",
5710                 netdev->mtu, new_mtu);
5711         netdev->mtu = new_mtu;
5712         hw->dev_spec._82575.mtu = new_mtu;
5713
5714         if (netif_running(netdev))
5715                 igb_up(adapter);
5716         else
5717                 igb_reset(adapter);
5718
5719         clear_bit(__IGB_RESETTING, &adapter->state);
5720
5721         return 0;
5722 }
5723
5724 /**
5725  * igb_update_stats - Update the board statistics counters
5726  * @adapter: board private structure
5727  **/
5728
5729 void igb_update_stats(struct igb_adapter *adapter)
5730 {
5731 #ifdef HAVE_NETDEV_STATS_IN_NETDEV
5732         struct net_device_stats *net_stats = &adapter->netdev->stats;
5733 #else
5734         struct net_device_stats *net_stats = &adapter->net_stats;
5735 #endif /* HAVE_NETDEV_STATS_IN_NETDEV */
5736         struct e1000_hw *hw = &adapter->hw;
5737 #ifdef HAVE_PCI_ERS
5738         struct pci_dev *pdev = adapter->pdev;
5739 #endif
5740         u32 reg, mpc;
5741         u16 phy_tmp;
5742         int i;
5743         u64 bytes, packets;
5744 #ifndef IGB_NO_LRO
5745         u32 flushed = 0, coal = 0;
5746         struct igb_q_vector *q_vector;
5747 #endif
5748
5749 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
5750
5751         /*
5752          * Prevent stats update while adapter is being reset, or if the pci
5753          * connection is down.
5754          */
5755         if (adapter->link_speed == 0)
5756                 return;
5757 #ifdef HAVE_PCI_ERS
5758         if (pci_channel_offline(pdev))
5759                 return;
5760
5761 #endif
5762 #ifndef IGB_NO_LRO
5763         for (i = 0; i < adapter->num_q_vectors; i++) {
5764                 q_vector = adapter->q_vector[i];
5765                 if (!q_vector)
5766                         continue;
5767                 flushed += q_vector->lrolist.stats.flushed;
5768                 coal += q_vector->lrolist.stats.coal;
5769         }
5770         adapter->lro_stats.flushed = flushed;
5771         adapter->lro_stats.coal = coal;
5772
5773 #endif
5774         bytes = 0;
5775         packets = 0;
5776         for (i = 0; i < adapter->num_rx_queues; i++) {
5777                 u32 rqdpc_tmp = E1000_READ_REG(hw, E1000_RQDPC(i)) & 0x0FFF;
5778                 struct igb_ring *ring = adapter->rx_ring[i];
5779                 ring->rx_stats.drops += rqdpc_tmp;
5780                 net_stats->rx_fifo_errors += rqdpc_tmp;
5781 #ifdef CONFIG_IGB_VMDQ_NETDEV
5782                 if (!ring->vmdq_netdev) {
5783                         bytes += ring->rx_stats.bytes;
5784                         packets += ring->rx_stats.packets;
5785                 }
5786 #else
5787                 bytes += ring->rx_stats.bytes;
5788                 packets += ring->rx_stats.packets;
5789 #endif
5790         }
5791
5792         net_stats->rx_bytes = bytes;
5793         net_stats->rx_packets = packets;
5794
5795         bytes = 0;
5796         packets = 0;
5797         for (i = 0; i < adapter->num_tx_queues; i++) {
5798                 struct igb_ring *ring = adapter->tx_ring[i];
5799 #ifdef CONFIG_IGB_VMDQ_NETDEV
5800                 if (!ring->vmdq_netdev) {
5801                         bytes += ring->tx_stats.bytes;
5802                         packets += ring->tx_stats.packets;
5803                 }
5804 #else
5805                 bytes += ring->tx_stats.bytes;
5806                 packets += ring->tx_stats.packets;
5807 #endif
5808         }
5809         net_stats->tx_bytes = bytes;
5810         net_stats->tx_packets = packets;
5811
5812         /* read stats registers */
5813         adapter->stats.crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
5814         adapter->stats.gprc += E1000_READ_REG(hw, E1000_GPRC);
5815         adapter->stats.gorc += E1000_READ_REG(hw, E1000_GORCL);
5816         E1000_READ_REG(hw, E1000_GORCH); /* clear GORCL */
5817         adapter->stats.bprc += E1000_READ_REG(hw, E1000_BPRC);
5818         adapter->stats.mprc += E1000_READ_REG(hw, E1000_MPRC);
5819         adapter->stats.roc += E1000_READ_REG(hw, E1000_ROC);
5820
5821         adapter->stats.prc64 += E1000_READ_REG(hw, E1000_PRC64);
5822         adapter->stats.prc127 += E1000_READ_REG(hw, E1000_PRC127);
5823         adapter->stats.prc255 += E1000_READ_REG(hw, E1000_PRC255);
5824         adapter->stats.prc511 += E1000_READ_REG(hw, E1000_PRC511);
5825         adapter->stats.prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
5826         adapter->stats.prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
5827         adapter->stats.symerrs += E1000_READ_REG(hw, E1000_SYMERRS);
5828         adapter->stats.sec += E1000_READ_REG(hw, E1000_SEC);
5829
5830         mpc = E1000_READ_REG(hw, E1000_MPC);
5831         adapter->stats.mpc += mpc;
5832         net_stats->rx_fifo_errors += mpc;
5833         adapter->stats.scc += E1000_READ_REG(hw, E1000_SCC);
5834         adapter->stats.ecol += E1000_READ_REG(hw, E1000_ECOL);
5835         adapter->stats.mcc += E1000_READ_REG(hw, E1000_MCC);
5836         adapter->stats.latecol += E1000_READ_REG(hw, E1000_LATECOL);
5837         adapter->stats.dc += E1000_READ_REG(hw, E1000_DC);
5838         adapter->stats.rlec += E1000_READ_REG(hw, E1000_RLEC);
5839         adapter->stats.xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
5840         adapter->stats.xontxc += E1000_READ_REG(hw, E1000_XONTXC);
5841         adapter->stats.xoffrxc += E1000_READ_REG(hw, E1000_XOFFRXC);
5842         adapter->stats.xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
5843         adapter->stats.fcruc += E1000_READ_REG(hw, E1000_FCRUC);
5844         adapter->stats.gptc += E1000_READ_REG(hw, E1000_GPTC);
5845         adapter->stats.gotc += E1000_READ_REG(hw, E1000_GOTCL);
5846         E1000_READ_REG(hw, E1000_GOTCH); /* clear GOTCL */
5847         adapter->stats.rnbc += E1000_READ_REG(hw, E1000_RNBC);
5848         adapter->stats.ruc += E1000_READ_REG(hw, E1000_RUC);
5849         adapter->stats.rfc += E1000_READ_REG(hw, E1000_RFC);
5850         adapter->stats.rjc += E1000_READ_REG(hw, E1000_RJC);
5851         adapter->stats.tor += E1000_READ_REG(hw, E1000_TORH);
5852         adapter->stats.tot += E1000_READ_REG(hw, E1000_TOTH);
5853         adapter->stats.tpr += E1000_READ_REG(hw, E1000_TPR);
5854
5855         adapter->stats.ptc64 += E1000_READ_REG(hw, E1000_PTC64);
5856         adapter->stats.ptc127 += E1000_READ_REG(hw, E1000_PTC127);
5857         adapter->stats.ptc255 += E1000_READ_REG(hw, E1000_PTC255);
5858         adapter->stats.ptc511 += E1000_READ_REG(hw, E1000_PTC511);
5859         adapter->stats.ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
5860         adapter->stats.ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
5861
5862         adapter->stats.mptc += E1000_READ_REG(hw, E1000_MPTC);
5863         adapter->stats.bptc += E1000_READ_REG(hw, E1000_BPTC);
5864
5865         adapter->stats.tpt += E1000_READ_REG(hw, E1000_TPT);
5866         adapter->stats.colc += E1000_READ_REG(hw, E1000_COLC);
5867
5868         adapter->stats.algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
5869         /* read internal phy sepecific stats */
5870         reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
5871         if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
5872                 adapter->stats.rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
5873
5874                 /* this stat has invalid values on i210/i211 */
5875                 if ((hw->mac.type != e1000_i210) &&
5876                     (hw->mac.type != e1000_i211))
5877                         adapter->stats.tncrs += E1000_READ_REG(hw, E1000_TNCRS);
5878         }
5879         adapter->stats.tsctc += E1000_READ_REG(hw, E1000_TSCTC);
5880         adapter->stats.tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
5881
5882         adapter->stats.iac += E1000_READ_REG(hw, E1000_IAC);
5883         adapter->stats.icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
5884         adapter->stats.icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
5885         adapter->stats.icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
5886         adapter->stats.ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
5887         adapter->stats.ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
5888         adapter->stats.ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
5889         adapter->stats.ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
5890         adapter->stats.icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
5891
5892         /* Fill out the OS statistics structure */
5893         net_stats->multicast = adapter->stats.mprc;
5894         net_stats->collisions = adapter->stats.colc;
5895
5896         /* Rx Errors */
5897
5898         /* RLEC on some newer hardware can be incorrect so build
5899          * our own version based on RUC and ROC */
5900         net_stats->rx_errors = adapter->stats.rxerrc +
5901                 adapter->stats.crcerrs + adapter->stats.algnerrc +
5902                 adapter->stats.ruc + adapter->stats.roc +
5903                 adapter->stats.cexterr;
5904         net_stats->rx_length_errors = adapter->stats.ruc +
5905                                       adapter->stats.roc;
5906         net_stats->rx_crc_errors = adapter->stats.crcerrs;
5907         net_stats->rx_frame_errors = adapter->stats.algnerrc;
5908         net_stats->rx_missed_errors = adapter->stats.mpc;
5909
5910         /* Tx Errors */
5911         net_stats->tx_errors = adapter->stats.ecol +
5912                                adapter->stats.latecol;
5913         net_stats->tx_aborted_errors = adapter->stats.ecol;
5914         net_stats->tx_window_errors = adapter->stats.latecol;
5915         net_stats->tx_carrier_errors = adapter->stats.tncrs;
5916
5917         /* Tx Dropped needs to be maintained elsewhere */
5918
5919         /* Phy Stats */
5920         if (hw->phy.media_type == e1000_media_type_copper) {
5921                 if ((adapter->link_speed == SPEED_1000) &&
5922                    (!e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
5923                         phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
5924                         adapter->phy_stats.idle_errors += phy_tmp;
5925                 }
5926         }
5927
5928         /* Management Stats */
5929         adapter->stats.mgptc += E1000_READ_REG(hw, E1000_MGTPTC);
5930         adapter->stats.mgprc += E1000_READ_REG(hw, E1000_MGTPRC);
5931         if (hw->mac.type > e1000_82580) {
5932                 adapter->stats.o2bgptc += E1000_READ_REG(hw, E1000_O2BGPTC);
5933                 adapter->stats.o2bspc += E1000_READ_REG(hw, E1000_O2BSPC);
5934                 adapter->stats.b2ospc += E1000_READ_REG(hw, E1000_B2OSPC);
5935                 adapter->stats.b2ogprc += E1000_READ_REG(hw, E1000_B2OGPRC);
5936         }
5937 }
5938
5939 static irqreturn_t igb_msix_other(int irq, void *data)
5940 {
5941         struct igb_adapter *adapter = data;
5942         struct e1000_hw *hw = &adapter->hw;
5943         u32 icr = E1000_READ_REG(hw, E1000_ICR);
5944         /* reading ICR causes bit 31 of EICR to be cleared */
5945
5946         if (icr & E1000_ICR_DRSTA)
5947                 schedule_work(&adapter->reset_task);
5948
5949         if (icr & E1000_ICR_DOUTSYNC) {
5950                 /* HW is reporting DMA is out of sync */
5951                 adapter->stats.doosync++;
5952                 /* The DMA Out of Sync is also indication of a spoof event
5953                  * in IOV mode. Check the Wrong VM Behavior register to
5954                  * see if it is really a spoof event. */
5955                 igb_check_wvbr(adapter);
5956         }
5957
5958         /* Check for a mailbox event */
5959         if (icr & E1000_ICR_VMMB)
5960                 igb_msg_task(adapter);
5961
5962         if (icr & E1000_ICR_LSC) {
5963                 hw->mac.get_link_status = 1;
5964                 /* guard against interrupt when we're going down */
5965                 if (!test_bit(__IGB_DOWN, &adapter->state))
5966                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
5967         }
5968
5969 #ifdef HAVE_PTP_1588_CLOCK
5970         if (icr & E1000_ICR_TS) {
5971                 u32 tsicr = E1000_READ_REG(hw, E1000_TSICR);
5972
5973                 if (tsicr & E1000_TSICR_TXTS) {
5974                         /* acknowledge the interrupt */
5975                         E1000_WRITE_REG(hw, E1000_TSICR, E1000_TSICR_TXTS);
5976                         /* retrieve hardware timestamp */
5977                         schedule_work(&adapter->ptp_tx_work);
5978                 }
5979         }
5980 #endif /* HAVE_PTP_1588_CLOCK */
5981
5982         /* Check for MDD event */
5983         if (icr & E1000_ICR_MDDET)
5984                 igb_process_mdd_event(adapter);
5985
5986         E1000_WRITE_REG(hw, E1000_EIMS, adapter->eims_other);
5987
5988         return IRQ_HANDLED;
5989 }
5990
5991 static void igb_write_itr(struct igb_q_vector *q_vector)
5992 {
5993         struct igb_adapter *adapter = q_vector->adapter;
5994         u32 itr_val = q_vector->itr_val & 0x7FFC;
5995
5996         if (!q_vector->set_itr)
5997                 return;
5998
5999         if (!itr_val)
6000                 itr_val = 0x4;
6001
6002         if (adapter->hw.mac.type == e1000_82575)
6003                 itr_val |= itr_val << 16;
6004         else
6005                 itr_val |= E1000_EITR_CNT_IGNR;
6006
6007         writel(itr_val, q_vector->itr_register);
6008         q_vector->set_itr = 0;
6009 }
6010
6011 static irqreturn_t igb_msix_ring(int irq, void *data)
6012 {
6013         struct igb_q_vector *q_vector = data;
6014
6015         /* Write the ITR value calculated from the previous interrupt. */
6016         igb_write_itr(q_vector);
6017
6018         napi_schedule(&q_vector->napi);
6019
6020         return IRQ_HANDLED;
6021 }
6022
6023 #ifdef IGB_DCA
6024 static void igb_update_tx_dca(struct igb_adapter *adapter,
6025                               struct igb_ring *tx_ring,
6026                               int cpu)
6027 {
6028         struct e1000_hw *hw = &adapter->hw;
6029         u32 txctrl = dca3_get_tag(tx_ring->dev, cpu);
6030
6031         if (hw->mac.type != e1000_82575)
6032                 txctrl <<= E1000_DCA_TXCTRL_CPUID_SHIFT_82576;
6033
6034         /*
6035          * We can enable relaxed ordering for reads, but not writes when
6036          * DCA is enabled.  This is due to a known issue in some chipsets
6037          * which will cause the DCA tag to be cleared.
6038          */
6039         txctrl |= E1000_DCA_TXCTRL_DESC_RRO_EN |
6040                   E1000_DCA_TXCTRL_DATA_RRO_EN |
6041                   E1000_DCA_TXCTRL_DESC_DCA_EN;
6042
6043         E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(tx_ring->reg_idx), txctrl);
6044 }
6045
6046 static void igb_update_rx_dca(struct igb_adapter *adapter,
6047                               struct igb_ring *rx_ring,
6048                               int cpu)
6049 {
6050         struct e1000_hw *hw = &adapter->hw;
6051         u32 rxctrl = dca3_get_tag(&adapter->pdev->dev, cpu);
6052
6053         if (hw->mac.type != e1000_82575)
6054                 rxctrl <<= E1000_DCA_RXCTRL_CPUID_SHIFT_82576;
6055
6056         /*
6057          * We can enable relaxed ordering for reads, but not writes when
6058          * DCA is enabled.  This is due to a known issue in some chipsets
6059          * which will cause the DCA tag to be cleared.
6060          */
6061         rxctrl |= E1000_DCA_RXCTRL_DESC_RRO_EN |
6062                   E1000_DCA_RXCTRL_DESC_DCA_EN;
6063
6064         E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(rx_ring->reg_idx), rxctrl);
6065 }
6066
6067 static void igb_update_dca(struct igb_q_vector *q_vector)
6068 {
6069         struct igb_adapter *adapter = q_vector->adapter;
6070         int cpu = get_cpu();
6071
6072         if (q_vector->cpu == cpu)
6073                 goto out_no_update;
6074
6075         if (q_vector->tx.ring)
6076                 igb_update_tx_dca(adapter, q_vector->tx.ring, cpu);
6077
6078         if (q_vector->rx.ring)
6079                 igb_update_rx_dca(adapter, q_vector->rx.ring, cpu);
6080
6081         q_vector->cpu = cpu;
6082 out_no_update:
6083         put_cpu();
6084 }
6085
6086 static void igb_setup_dca(struct igb_adapter *adapter)
6087 {
6088         struct e1000_hw *hw = &adapter->hw;
6089         int i;
6090
6091         if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
6092                 return;
6093
6094         /* Always use CB2 mode, difference is masked in the CB driver. */
6095         E1000_WRITE_REG(hw, E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
6096
6097         for (i = 0; i < adapter->num_q_vectors; i++) {
6098                 adapter->q_vector[i]->cpu = -1;
6099                 igb_update_dca(adapter->q_vector[i]);
6100         }
6101 }
6102
6103 static int __igb_notify_dca(struct device *dev, void *data)
6104 {
6105         struct net_device *netdev = dev_get_drvdata(dev);
6106         struct igb_adapter *adapter = netdev_priv(netdev);
6107         struct pci_dev *pdev = adapter->pdev;
6108         struct e1000_hw *hw = &adapter->hw;
6109         unsigned long event = *(unsigned long *)data;
6110
6111         switch (event) {
6112         case DCA_PROVIDER_ADD:
6113                 /* if already enabled, don't do it again */
6114                 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
6115                         break;
6116                 if (dca_add_requester(dev) == E1000_SUCCESS) {
6117                         adapter->flags |= IGB_FLAG_DCA_ENABLED;
6118                         dev_info(pci_dev_to_dev(pdev), "DCA enabled\n");
6119                         igb_setup_dca(adapter);
6120                         break;
6121                 }
6122                 /* Fall Through since DCA is disabled. */
6123         case DCA_PROVIDER_REMOVE:
6124                 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
6125                         /* without this a class_device is left
6126                          * hanging around in the sysfs model */
6127                         dca_remove_requester(dev);
6128                         dev_info(pci_dev_to_dev(pdev), "DCA disabled\n");
6129                         adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
6130                         E1000_WRITE_REG(hw, E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_DISABLE);
6131                 }
6132                 break;
6133         }
6134
6135         return E1000_SUCCESS;
6136 }
6137
6138 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
6139                           void *p)
6140 {
6141         int ret_val;
6142
6143         ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
6144                                          __igb_notify_dca);
6145
6146         return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
6147 }
6148 #endif /* IGB_DCA */
6149
6150 static int igb_vf_configure(struct igb_adapter *adapter, int vf)
6151 {
6152         unsigned char mac_addr[ETH_ALEN];
6153
6154         random_ether_addr(mac_addr);
6155         igb_set_vf_mac(adapter, vf, mac_addr);
6156
6157 #ifdef IFLA_VF_MAX
6158 #ifdef HAVE_VF_SPOOFCHK_CONFIGURE
6159         /* By default spoof check is enabled for all VFs */
6160         adapter->vf_data[vf].spoofchk_enabled = true;
6161 #endif
6162 #endif
6163
6164         return true;
6165 }
6166
6167 static void igb_ping_all_vfs(struct igb_adapter *adapter)
6168 {
6169         struct e1000_hw *hw = &adapter->hw;
6170         u32 ping;
6171         int i;
6172
6173         for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
6174                 ping = E1000_PF_CONTROL_MSG;
6175                 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
6176                         ping |= E1000_VT_MSGTYPE_CTS;
6177                 e1000_write_mbx(hw, &ping, 1, i);
6178         }
6179 }
6180
6181 /**
6182  *  igb_mta_set_ - Set multicast filter table address
6183  *  @adapter: pointer to the adapter structure
6184  *  @hash_value: determines the MTA register and bit to set
6185  *
6186  *  The multicast table address is a register array of 32-bit registers.
6187  *  The hash_value is used to determine what register the bit is in, the
6188  *  current value is read, the new bit is OR'd in and the new value is
6189  *  written back into the register.
6190  **/
6191 void igb_mta_set(struct igb_adapter *adapter, u32 hash_value)
6192 {
6193         struct e1000_hw *hw = &adapter->hw;
6194         u32 hash_bit, hash_reg, mta;
6195
6196         /*
6197          * The MTA is a register array of 32-bit registers. It is
6198          * treated like an array of (32*mta_reg_count) bits.  We want to
6199          * set bit BitArray[hash_value]. So we figure out what register
6200          * the bit is in, read it, OR in the new bit, then write
6201          * back the new value.  The (hw->mac.mta_reg_count - 1) serves as a
6202          * mask to bits 31:5 of the hash value which gives us the
6203          * register we're modifying.  The hash bit within that register
6204          * is determined by the lower 5 bits of the hash value.
6205          */
6206         hash_reg = (hash_value >> 5) & (hw->mac.mta_reg_count - 1);
6207         hash_bit = hash_value & 0x1F;
6208
6209         mta = E1000_READ_REG_ARRAY(hw, E1000_MTA, hash_reg);
6210
6211         mta |= (1 << hash_bit);
6212
6213         E1000_WRITE_REG_ARRAY(hw, E1000_MTA, hash_reg, mta);
6214         E1000_WRITE_FLUSH(hw);
6215 }
6216
6217 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
6218 {
6219
6220         struct e1000_hw *hw = &adapter->hw;
6221         u32 vmolr = E1000_READ_REG(hw, E1000_VMOLR(vf));
6222         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6223
6224         vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
6225                             IGB_VF_FLAG_MULTI_PROMISC);
6226         vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
6227
6228 #ifdef IGB_ENABLE_VF_PROMISC
6229         if (*msgbuf & E1000_VF_SET_PROMISC_UNICAST) {
6230                 vmolr |= E1000_VMOLR_ROPE;
6231                 vf_data->flags |= IGB_VF_FLAG_UNI_PROMISC;
6232                 *msgbuf &= ~E1000_VF_SET_PROMISC_UNICAST;
6233         }
6234 #endif
6235         if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
6236                 vmolr |= E1000_VMOLR_MPME;
6237                 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
6238                 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
6239         } else {
6240                 /*
6241                  * if we have hashes and we are clearing a multicast promisc
6242                  * flag we need to write the hashes to the MTA as this step
6243                  * was previously skipped
6244                  */
6245                 if (vf_data->num_vf_mc_hashes > 30) {
6246                         vmolr |= E1000_VMOLR_MPME;
6247                 } else if (vf_data->num_vf_mc_hashes) {
6248                         int j;
6249                         vmolr |= E1000_VMOLR_ROMPE;
6250                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
6251                                 igb_mta_set(adapter, vf_data->vf_mc_hashes[j]);
6252                 }
6253         }
6254
6255         E1000_WRITE_REG(hw, E1000_VMOLR(vf), vmolr);
6256
6257         /* there are flags left unprocessed, likely not supported */
6258         if (*msgbuf & E1000_VT_MSGINFO_MASK)
6259                 return -EINVAL;
6260
6261         return 0;
6262
6263 }
6264
6265 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
6266                                   u32 *msgbuf, u32 vf)
6267 {
6268         int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
6269         u16 *hash_list = (u16 *)&msgbuf[1];
6270         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6271         int i;
6272
6273         /* salt away the number of multicast addresses assigned
6274          * to this VF for later use to restore when the PF multi cast
6275          * list changes
6276          */
6277         vf_data->num_vf_mc_hashes = n;
6278
6279         /* only up to 30 hash values supported */
6280         if (n > 30)
6281                 n = 30;
6282
6283         /* store the hashes for later use */
6284         for (i = 0; i < n; i++)
6285                 vf_data->vf_mc_hashes[i] = hash_list[i];
6286
6287         /* Flush and reset the mta with the new values */
6288         igb_set_rx_mode(adapter->netdev);
6289
6290         return 0;
6291 }
6292
6293 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
6294 {
6295         struct e1000_hw *hw = &adapter->hw;
6296         struct vf_data_storage *vf_data;
6297         int i, j;
6298
6299         for (i = 0; i < adapter->vfs_allocated_count; i++) {
6300                 u32 vmolr = E1000_READ_REG(hw, E1000_VMOLR(i));
6301                 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
6302
6303                 vf_data = &adapter->vf_data[i];
6304
6305                 if ((vf_data->num_vf_mc_hashes > 30) ||
6306                     (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
6307                         vmolr |= E1000_VMOLR_MPME;
6308                 } else if (vf_data->num_vf_mc_hashes) {
6309                         vmolr |= E1000_VMOLR_ROMPE;
6310                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
6311                                 igb_mta_set(adapter, vf_data->vf_mc_hashes[j]);
6312                 }
6313                 E1000_WRITE_REG(hw, E1000_VMOLR(i), vmolr);
6314         }
6315 }
6316
6317 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
6318 {
6319         struct e1000_hw *hw = &adapter->hw;
6320         u32 pool_mask, reg, vid;
6321         u16 vlan_default;
6322         int i;
6323
6324         pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
6325
6326         /* Find the vlan filter for this id */
6327         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
6328                 reg = E1000_READ_REG(hw, E1000_VLVF(i));
6329
6330                 /* remove the vf from the pool */
6331                 reg &= ~pool_mask;
6332
6333                 /* if pool is empty then remove entry from vfta */
6334                 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
6335                     (reg & E1000_VLVF_VLANID_ENABLE)) {
6336                         reg = 0;
6337                         vid = reg & E1000_VLVF_VLANID_MASK;
6338                         igb_vfta_set(adapter, vid, FALSE);
6339                 }
6340
6341                 E1000_WRITE_REG(hw, E1000_VLVF(i), reg);
6342         }
6343
6344         adapter->vf_data[vf].vlans_enabled = 0;
6345
6346         vlan_default = adapter->vf_data[vf].default_vf_vlan_id;
6347         if (vlan_default)
6348                 igb_vlvf_set(adapter, vlan_default, true, vf);
6349 }
6350
6351 s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
6352 {
6353         struct e1000_hw *hw = &adapter->hw;
6354         u32 reg, i;
6355
6356         /* The vlvf table only exists on 82576 hardware and newer */
6357         if (hw->mac.type < e1000_82576)
6358                 return -1;
6359
6360         /* we only need to do this if VMDq is enabled */
6361         if (!adapter->vmdq_pools)
6362                 return -1;
6363
6364         /* Find the vlan filter for this id */
6365         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
6366                 reg = E1000_READ_REG(hw, E1000_VLVF(i));
6367                 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
6368                     vid == (reg & E1000_VLVF_VLANID_MASK))
6369                         break;
6370         }
6371
6372         if (add) {
6373                 if (i == E1000_VLVF_ARRAY_SIZE) {
6374                         /* Did not find a matching VLAN ID entry that was
6375                          * enabled.  Search for a free filter entry, i.e.
6376                          * one without the enable bit set
6377                          */
6378                         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
6379                                 reg = E1000_READ_REG(hw, E1000_VLVF(i));
6380                                 if (!(reg & E1000_VLVF_VLANID_ENABLE))
6381                                         break;
6382                         }
6383                 }
6384                 if (i < E1000_VLVF_ARRAY_SIZE) {
6385                         /* Found an enabled/available entry */
6386                         reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
6387
6388                         /* if !enabled we need to set this up in vfta */
6389                         if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
6390                                 /* add VID to filter table */
6391                                 igb_vfta_set(adapter, vid, TRUE);
6392                                 reg |= E1000_VLVF_VLANID_ENABLE;
6393                         }
6394                         reg &= ~E1000_VLVF_VLANID_MASK;
6395                         reg |= vid;
6396                         E1000_WRITE_REG(hw, E1000_VLVF(i), reg);
6397
6398                         /* do not modify RLPML for PF devices */
6399                         if (vf >= adapter->vfs_allocated_count)
6400                                 return E1000_SUCCESS;
6401
6402                         if (!adapter->vf_data[vf].vlans_enabled) {
6403                                 u32 size;
6404                                 reg = E1000_READ_REG(hw, E1000_VMOLR(vf));
6405                                 size = reg & E1000_VMOLR_RLPML_MASK;
6406                                 size += 4;
6407                                 reg &= ~E1000_VMOLR_RLPML_MASK;
6408                                 reg |= size;
6409                                 E1000_WRITE_REG(hw, E1000_VMOLR(vf), reg);
6410                         }
6411
6412                         adapter->vf_data[vf].vlans_enabled++;
6413                 }
6414         } else {
6415                 if (i < E1000_VLVF_ARRAY_SIZE) {
6416                         /* remove vf from the pool */
6417                         reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
6418                         /* if pool is empty then remove entry from vfta */
6419                         if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
6420                                 reg = 0;
6421                                 igb_vfta_set(adapter, vid, FALSE);
6422                         }
6423                         E1000_WRITE_REG(hw, E1000_VLVF(i), reg);
6424
6425                         /* do not modify RLPML for PF devices */
6426                         if (vf >= adapter->vfs_allocated_count)
6427                                 return E1000_SUCCESS;
6428
6429                         adapter->vf_data[vf].vlans_enabled--;
6430                         if (!adapter->vf_data[vf].vlans_enabled) {
6431                                 u32 size;
6432                                 reg = E1000_READ_REG(hw, E1000_VMOLR(vf));
6433                                 size = reg & E1000_VMOLR_RLPML_MASK;
6434                                 size -= 4;
6435                                 reg &= ~E1000_VMOLR_RLPML_MASK;
6436                                 reg |= size;
6437                                 E1000_WRITE_REG(hw, E1000_VMOLR(vf), reg);
6438                         }
6439                 }
6440         }
6441         return E1000_SUCCESS;
6442 }
6443
6444 #ifdef IFLA_VF_MAX
6445 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
6446 {
6447         struct e1000_hw *hw = &adapter->hw;
6448
6449         if (vid)
6450                 E1000_WRITE_REG(hw, E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
6451         else
6452                 E1000_WRITE_REG(hw, E1000_VMVIR(vf), 0);
6453 }
6454
6455 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
6456 #ifdef HAVE_VF_VLAN_PROTO
6457                                int vf, u16 vlan, u8 qos, __be16 vlan_proto)
6458 #else
6459                                int vf, u16 vlan, u8 qos)
6460 #endif
6461 {
6462         int err = 0;
6463         struct igb_adapter *adapter = netdev_priv(netdev);
6464
6465         /* VLAN IDs accepted range 0-4094 */
6466         if ((vf >= adapter->vfs_allocated_count) || (vlan > VLAN_VID_MASK-1) || (qos > 7))
6467                 return -EINVAL;
6468
6469 #ifdef HAVE_VF_VLAN_PROTO
6470         if (vlan_proto != htons(ETH_P_8021Q))
6471                 return -EPROTONOSUPPORT;
6472 #endif
6473
6474         if (vlan || qos) {
6475                 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
6476                 if (err)
6477                         goto out;
6478                 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
6479                 igb_set_vmolr(adapter, vf, !vlan);
6480                 adapter->vf_data[vf].pf_vlan = vlan;
6481                 adapter->vf_data[vf].pf_qos = qos;
6482                 igb_set_vf_vlan_strip(adapter, vf, true);
6483                 dev_info(&adapter->pdev->dev,
6484                          "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
6485                 if (test_bit(__IGB_DOWN, &adapter->state)) {
6486                         dev_warn(&adapter->pdev->dev,
6487                                  "The VF VLAN has been set,"
6488                                  " but the PF device is not up.\n");
6489                         dev_warn(&adapter->pdev->dev,
6490                                  "Bring the PF device up before"
6491                                  " attempting to use the VF device.\n");
6492                 }
6493         } else {
6494                 if (adapter->vf_data[vf].pf_vlan)
6495                         dev_info(&adapter->pdev->dev,
6496                                  "Clearing VLAN on VF %d\n", vf);
6497                 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
6498                                    false, vf);
6499                 igb_set_vmvir(adapter, vlan, vf);
6500                 igb_set_vmolr(adapter, vf, true);
6501                 igb_set_vf_vlan_strip(adapter, vf, false);
6502                 adapter->vf_data[vf].pf_vlan = 0;
6503                 adapter->vf_data[vf].pf_qos = 0;
6504        }
6505 out:
6506        return err;
6507 }
6508
6509 #ifdef HAVE_VF_SPOOFCHK_CONFIGURE
6510 static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
6511                                 bool setting)
6512 {
6513         struct igb_adapter *adapter = netdev_priv(netdev);
6514         struct e1000_hw *hw = &adapter->hw;
6515         u32 dtxswc, reg_offset;
6516
6517         if (!adapter->vfs_allocated_count)
6518                 return -EOPNOTSUPP;
6519
6520         if (vf >= adapter->vfs_allocated_count)
6521                 return -EINVAL;
6522
6523         reg_offset = (hw->mac.type == e1000_82576) ? E1000_DTXSWC : E1000_TXSWC;
6524         dtxswc = E1000_READ_REG(hw, reg_offset);
6525         if (setting)
6526                 dtxswc |= ((1 << vf) |
6527                            (1 << (vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT)));
6528         else
6529                 dtxswc &= ~((1 << vf) |
6530                             (1 << (vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT)));
6531         E1000_WRITE_REG(hw, reg_offset, dtxswc);
6532
6533         adapter->vf_data[vf].spoofchk_enabled = setting;
6534         return E1000_SUCCESS;
6535 }
6536 #endif /* HAVE_VF_SPOOFCHK_CONFIGURE */
6537 #endif /* IFLA_VF_MAX */
6538
6539 static int igb_find_vlvf_entry(struct igb_adapter *adapter, int vid)
6540 {
6541         struct e1000_hw *hw = &adapter->hw;
6542         int i;
6543         u32 reg;
6544
6545         /* Find the vlan filter for this id */
6546         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
6547                 reg = E1000_READ_REG(hw, E1000_VLVF(i));
6548                 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
6549                     vid == (reg & E1000_VLVF_VLANID_MASK))
6550                         break;
6551         }
6552
6553         if (i >= E1000_VLVF_ARRAY_SIZE)
6554                 i = -1;
6555
6556         return i;
6557 }
6558
6559 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
6560 {
6561         struct e1000_hw *hw = &adapter->hw;
6562         int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
6563         int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
6564         int err = 0;
6565
6566         if (vid)
6567                 igb_set_vf_vlan_strip(adapter, vf, true);
6568         else
6569                 igb_set_vf_vlan_strip(adapter, vf, false);
6570
6571         /* If in promiscuous mode we need to make sure the PF also has
6572          * the VLAN filter set.
6573          */
6574         if (add && (adapter->netdev->flags & IFF_PROMISC))
6575                 err = igb_vlvf_set(adapter, vid, add,
6576                                    adapter->vfs_allocated_count);
6577         if (err)
6578                 goto out;
6579
6580         err = igb_vlvf_set(adapter, vid, add, vf);
6581
6582         if (err)
6583                 goto out;
6584
6585         /* Go through all the checks to see if the VLAN filter should
6586          * be wiped completely.
6587          */
6588         if (!add && (adapter->netdev->flags & IFF_PROMISC)) {
6589                 u32 vlvf, bits;
6590
6591                 int regndx = igb_find_vlvf_entry(adapter, vid);
6592                 if (regndx < 0)
6593                         goto out;
6594                 /* See if any other pools are set for this VLAN filter
6595                  * entry other than the PF.
6596                  */
6597                 vlvf = bits = E1000_READ_REG(hw, E1000_VLVF(regndx));
6598                 bits &= 1 << (E1000_VLVF_POOLSEL_SHIFT +
6599                               adapter->vfs_allocated_count);
6600                 /* If the filter was removed then ensure PF pool bit
6601                  * is cleared if the PF only added itself to the pool
6602                  * because the PF is in promiscuous mode.
6603                  */
6604                 if ((vlvf & VLAN_VID_MASK) == vid &&
6605 #ifndef HAVE_VLAN_RX_REGISTER
6606                     !test_bit(vid, adapter->active_vlans) &&
6607 #endif
6608                     !bits)
6609                         igb_vlvf_set(adapter, vid, add,
6610                                      adapter->vfs_allocated_count);
6611         }
6612
6613 out:
6614         return err;
6615 }
6616
6617 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
6618 {
6619         struct e1000_hw *hw = &adapter->hw;
6620
6621         /* clear flags except flag that the PF has set the MAC */
6622         adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
6623         adapter->vf_data[vf].last_nack = jiffies;
6624
6625         /* reset offloads to defaults */
6626         igb_set_vmolr(adapter, vf, true);
6627
6628         /* reset vlans for device */
6629         igb_clear_vf_vfta(adapter, vf);
6630 #ifdef IFLA_VF_MAX
6631         if (adapter->vf_data[vf].pf_vlan)
6632                 igb_ndo_set_vf_vlan(adapter->netdev, vf,
6633                                     adapter->vf_data[vf].pf_vlan,
6634 #ifdef HAVE_VF_VLAN_PROTO
6635                                     adapter->vf_data[vf].pf_qos,
6636                                     htons(ETH_P_8021Q));
6637 #else
6638                                     adapter->vf_data[vf].pf_qos);
6639 #endif
6640         else
6641                 igb_clear_vf_vfta(adapter, vf);
6642 #endif
6643
6644         /* reset multicast table array for vf */
6645         adapter->vf_data[vf].num_vf_mc_hashes = 0;
6646
6647         /* Flush and reset the mta with the new values */
6648         igb_set_rx_mode(adapter->netdev);
6649
6650         /*
6651          * Reset the VFs TDWBAL and TDWBAH registers which are not
6652          * cleared by a VFLR
6653          */
6654         E1000_WRITE_REG(hw, E1000_TDWBAH(vf), 0);
6655         E1000_WRITE_REG(hw, E1000_TDWBAL(vf), 0);
6656         if (hw->mac.type == e1000_82576) {
6657                 E1000_WRITE_REG(hw, E1000_TDWBAH(IGB_MAX_VF_FUNCTIONS + vf), 0);
6658                 E1000_WRITE_REG(hw, E1000_TDWBAL(IGB_MAX_VF_FUNCTIONS + vf), 0);
6659         }
6660 }
6661
6662 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
6663 {
6664         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
6665
6666         /* generate a new mac address as we were hotplug removed/added */
6667         if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
6668                 random_ether_addr(vf_mac);
6669
6670         /* process remaining reset events */
6671         igb_vf_reset(adapter, vf);
6672 }
6673
6674 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
6675 {
6676         struct e1000_hw *hw = &adapter->hw;
6677         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
6678         u32 reg, msgbuf[3];
6679         u8 *addr = (u8 *)(&msgbuf[1]);
6680
6681         /* process all the same items cleared in a function level reset */
6682         igb_vf_reset(adapter, vf);
6683
6684         /* set vf mac address */
6685         igb_del_mac_filter(adapter, vf_mac, vf);
6686         igb_add_mac_filter(adapter, vf_mac, vf);
6687
6688         /* enable transmit and receive for vf */
6689         reg = E1000_READ_REG(hw, E1000_VFTE);
6690         E1000_WRITE_REG(hw, E1000_VFTE, reg | (1 << vf));
6691         reg = E1000_READ_REG(hw, E1000_VFRE);
6692         E1000_WRITE_REG(hw, E1000_VFRE, reg | (1 << vf));
6693
6694         adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
6695
6696         /* reply to reset with ack and vf mac address */
6697         msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
6698         memcpy(addr, vf_mac, 6);
6699         e1000_write_mbx(hw, msgbuf, 3, vf);
6700 }
6701
6702 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
6703 {
6704         /*
6705          * The VF MAC Address is stored in a packed array of bytes
6706          * starting at the second 32 bit word of the msg array
6707          */
6708         unsigned char *addr = (unsigned char *)&msg[1];
6709         int err = -1;
6710
6711         if (is_valid_ether_addr(addr))
6712                 err = igb_set_vf_mac(adapter, vf, addr);
6713
6714         return err;
6715 }
6716
6717 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
6718 {
6719         struct e1000_hw *hw = &adapter->hw;
6720         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6721         u32 msg = E1000_VT_MSGTYPE_NACK;
6722
6723         /* if device isn't clear to send it shouldn't be reading either */
6724         if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
6725             time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
6726                 e1000_write_mbx(hw, &msg, 1, vf);
6727                 vf_data->last_nack = jiffies;
6728         }
6729 }
6730
6731 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
6732 {
6733         struct pci_dev *pdev = adapter->pdev;
6734         u32 msgbuf[E1000_VFMAILBOX_SIZE];
6735         struct e1000_hw *hw = &adapter->hw;
6736         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6737         s32 retval;
6738
6739         retval = e1000_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
6740
6741         if (retval) {
6742                 dev_err(pci_dev_to_dev(pdev), "Error receiving message from VF\n");
6743                 return;
6744         }
6745
6746         /* this is a message we already processed, do nothing */
6747         if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
6748                 return;
6749
6750         /*
6751          * until the vf completes a reset it should not be
6752          * allowed to start any configuration.
6753          */
6754
6755         if (msgbuf[0] == E1000_VF_RESET) {
6756                 igb_vf_reset_msg(adapter, vf);
6757                 return;
6758         }
6759
6760         if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
6761                 msgbuf[0] = E1000_VT_MSGTYPE_NACK;
6762                 if (time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
6763                         e1000_write_mbx(hw, msgbuf, 1, vf);
6764                         vf_data->last_nack = jiffies;
6765                 }
6766                 return;
6767         }
6768
6769         switch ((msgbuf[0] & 0xFFFF)) {
6770         case E1000_VF_SET_MAC_ADDR:
6771                 retval = -EINVAL;
6772 #ifndef IGB_DISABLE_VF_MAC_SET
6773                 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
6774                         retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
6775                 else
6776                         DPRINTK(DRV, INFO,
6777                                 "VF %d attempted to override administratively "
6778                                 "set MAC address\nReload the VF driver to "
6779                                 "resume operations\n", vf);
6780 #endif
6781                 break;
6782         case E1000_VF_SET_PROMISC:
6783                 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
6784                 break;
6785         case E1000_VF_SET_MULTICAST:
6786                 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
6787                 break;
6788         case E1000_VF_SET_LPE:
6789                 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
6790                 break;
6791         case E1000_VF_SET_VLAN:
6792                 retval = -1;
6793 #ifdef IFLA_VF_MAX
6794                 if (vf_data->pf_vlan)
6795                         DPRINTK(DRV, INFO,
6796                                 "VF %d attempted to override administratively "
6797                                 "set VLAN tag\nReload the VF driver to "
6798                                 "resume operations\n", vf);
6799                 else
6800 #endif
6801                         retval = igb_set_vf_vlan(adapter, msgbuf, vf);
6802                 break;
6803         default:
6804                 dev_err(pci_dev_to_dev(pdev), "Unhandled Msg %08x\n", msgbuf[0]);
6805                 retval = -E1000_ERR_MBX;
6806                 break;
6807         }
6808
6809         /* notify the VF of the results of what it sent us */
6810         if (retval)
6811                 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
6812         else
6813                 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
6814
6815         msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
6816
6817         e1000_write_mbx(hw, msgbuf, 1, vf);
6818 }
6819
6820 static void igb_msg_task(struct igb_adapter *adapter)
6821 {
6822         struct e1000_hw *hw = &adapter->hw;
6823         u32 vf;
6824
6825         for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
6826                 /* process any reset requests */
6827                 if (!e1000_check_for_rst(hw, vf))
6828                         igb_vf_reset_event(adapter, vf);
6829
6830                 /* process any messages pending */
6831                 if (!e1000_check_for_msg(hw, vf))
6832                         igb_rcv_msg_from_vf(adapter, vf);
6833
6834                 /* process any acks */
6835                 if (!e1000_check_for_ack(hw, vf))
6836                         igb_rcv_ack_from_vf(adapter, vf);
6837         }
6838 }
6839
6840 /**
6841  *  igb_set_uta - Set unicast filter table address
6842  *  @adapter: board private structure
6843  *
6844  *  The unicast table address is a register array of 32-bit registers.
6845  *  The table is meant to be used in a way similar to how the MTA is used
6846  *  however due to certain limitations in the hardware it is necessary to
6847  *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
6848  *  enable bit to allow vlan tag stripping when promiscuous mode is enabled
6849  **/
6850 static void igb_set_uta(struct igb_adapter *adapter)
6851 {
6852         struct e1000_hw *hw = &adapter->hw;
6853         int i;
6854
6855         /* The UTA table only exists on 82576 hardware and newer */
6856         if (hw->mac.type < e1000_82576)
6857                 return;
6858
6859         /* we only need to do this if VMDq is enabled */
6860         if (!adapter->vmdq_pools)
6861                 return;
6862
6863         for (i = 0; i < hw->mac.uta_reg_count; i++)
6864                 E1000_WRITE_REG_ARRAY(hw, E1000_UTA, i, ~0);
6865 }
6866
6867 /**
6868  * igb_intr_msi - Interrupt Handler
6869  * @irq: interrupt number
6870  * @data: pointer to a network interface device structure
6871  **/
6872 static irqreturn_t igb_intr_msi(int irq, void *data)
6873 {
6874         struct igb_adapter *adapter = data;
6875         struct igb_q_vector *q_vector = adapter->q_vector[0];
6876         struct e1000_hw *hw = &adapter->hw;
6877         /* read ICR disables interrupts using IAM */
6878         u32 icr = E1000_READ_REG(hw, E1000_ICR);
6879
6880         igb_write_itr(q_vector);
6881
6882         if (icr & E1000_ICR_DRSTA)
6883                 schedule_work(&adapter->reset_task);
6884
6885         if (icr & E1000_ICR_DOUTSYNC) {
6886                 /* HW is reporting DMA is out of sync */
6887                 adapter->stats.doosync++;
6888         }
6889
6890         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
6891                 hw->mac.get_link_status = 1;
6892                 if (!test_bit(__IGB_DOWN, &adapter->state))
6893                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
6894         }
6895
6896 #ifdef HAVE_PTP_1588_CLOCK
6897         if (icr & E1000_ICR_TS) {
6898                 u32 tsicr = E1000_READ_REG(hw, E1000_TSICR);
6899
6900                 if (tsicr & E1000_TSICR_TXTS) {
6901                         /* acknowledge the interrupt */
6902                         E1000_WRITE_REG(hw, E1000_TSICR, E1000_TSICR_TXTS);
6903                         /* retrieve hardware timestamp */
6904                         schedule_work(&adapter->ptp_tx_work);
6905                 }
6906         }
6907 #endif /* HAVE_PTP_1588_CLOCK */
6908
6909         napi_schedule(&q_vector->napi);
6910
6911         return IRQ_HANDLED;
6912 }
6913
6914 /**
6915  * igb_intr - Legacy Interrupt Handler
6916  * @irq: interrupt number
6917  * @data: pointer to a network interface device structure
6918  **/
6919 static irqreturn_t igb_intr(int irq, void *data)
6920 {
6921         struct igb_adapter *adapter = data;
6922         struct igb_q_vector *q_vector = adapter->q_vector[0];
6923         struct e1000_hw *hw = &adapter->hw;
6924         /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
6925          * need for the IMC write */
6926         u32 icr = E1000_READ_REG(hw, E1000_ICR);
6927
6928         /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
6929          * not set, then the adapter didn't send an interrupt */
6930         if (!(icr & E1000_ICR_INT_ASSERTED))
6931                 return IRQ_NONE;
6932
6933         igb_write_itr(q_vector);
6934
6935         if (icr & E1000_ICR_DRSTA)
6936                 schedule_work(&adapter->reset_task);
6937
6938         if (icr & E1000_ICR_DOUTSYNC) {
6939                 /* HW is reporting DMA is out of sync */
6940                 adapter->stats.doosync++;
6941         }
6942
6943         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
6944                 hw->mac.get_link_status = 1;
6945                 /* guard against interrupt when we're going down */
6946                 if (!test_bit(__IGB_DOWN, &adapter->state))
6947                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
6948         }
6949
6950 #ifdef HAVE_PTP_1588_CLOCK
6951         if (icr & E1000_ICR_TS) {
6952                 u32 tsicr = E1000_READ_REG(hw, E1000_TSICR);
6953
6954                 if (tsicr & E1000_TSICR_TXTS) {
6955                         /* acknowledge the interrupt */
6956                         E1000_WRITE_REG(hw, E1000_TSICR, E1000_TSICR_TXTS);
6957                         /* retrieve hardware timestamp */
6958                         schedule_work(&adapter->ptp_tx_work);
6959                 }
6960         }
6961 #endif /* HAVE_PTP_1588_CLOCK */
6962
6963         napi_schedule(&q_vector->napi);
6964
6965         return IRQ_HANDLED;
6966 }
6967
6968 void igb_ring_irq_enable(struct igb_q_vector *q_vector)
6969 {
6970         struct igb_adapter *adapter = q_vector->adapter;
6971         struct e1000_hw *hw = &adapter->hw;
6972
6973         if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
6974             (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
6975                 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
6976                         igb_set_itr(q_vector);
6977                 else
6978                         igb_update_ring_itr(q_vector);
6979         }
6980
6981         if (!test_bit(__IGB_DOWN, &adapter->state)) {
6982                 if (adapter->msix_entries)
6983                         E1000_WRITE_REG(hw, E1000_EIMS, q_vector->eims_value);
6984                 else
6985                         igb_irq_enable(adapter);
6986         }
6987 }
6988
6989 /**
6990  * igb_poll - NAPI Rx polling callback
6991  * @napi: napi polling structure
6992  * @budget: count of how many packets we should handle
6993  **/
6994 static int igb_poll(struct napi_struct *napi, int budget)
6995 {
6996         struct igb_q_vector *q_vector = container_of(napi, struct igb_q_vector, napi);
6997         bool clean_complete = true;
6998
6999 #ifdef IGB_DCA
7000         if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
7001                 igb_update_dca(q_vector);
7002 #endif
7003         if (q_vector->tx.ring)
7004                 clean_complete = igb_clean_tx_irq(q_vector);
7005
7006         if (q_vector->rx.ring)
7007                 clean_complete &= igb_clean_rx_irq(q_vector, budget);
7008
7009 #ifndef HAVE_NETDEV_NAPI_LIST
7010         /* if netdev is disabled we need to stop polling */
7011         if (!netif_running(q_vector->adapter->netdev))
7012                 clean_complete = true;
7013
7014 #endif
7015         /* If all work not completed, return budget and keep polling */
7016         if (!clean_complete)
7017                 return budget;
7018
7019         /* If not enough Rx work done, exit the polling mode */
7020         napi_complete(napi);
7021         igb_ring_irq_enable(q_vector);
7022
7023         return 0;
7024 }
7025
7026 /**
7027  * igb_clean_tx_irq - Reclaim resources after transmit completes
7028  * @q_vector: pointer to q_vector containing needed info
7029  * returns TRUE if ring is completely cleaned
7030  **/
7031 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
7032 {
7033         struct igb_adapter *adapter = q_vector->adapter;
7034         struct igb_ring *tx_ring = q_vector->tx.ring;
7035         struct igb_tx_buffer *tx_buffer;
7036         union e1000_adv_tx_desc *tx_desc;
7037         unsigned int total_bytes = 0, total_packets = 0;
7038         unsigned int budget = q_vector->tx.work_limit;
7039         unsigned int i = tx_ring->next_to_clean;
7040
7041         if (test_bit(__IGB_DOWN, &adapter->state))
7042                 return true;
7043
7044         tx_buffer = &tx_ring->tx_buffer_info[i];
7045         tx_desc = IGB_TX_DESC(tx_ring, i);
7046         i -= tx_ring->count;
7047
7048         do {
7049                 union e1000_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
7050
7051                 /* if next_to_watch is not set then there is no work pending */
7052                 if (!eop_desc)
7053                         break;
7054
7055                 /* prevent any other reads prior to eop_desc */
7056                 read_barrier_depends();
7057
7058                 /* if DD is not set pending work has not been completed */
7059                 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
7060                         break;
7061
7062                 /* clear next_to_watch to prevent false hangs */
7063                 tx_buffer->next_to_watch = NULL;
7064
7065                 /* update the statistics for this packet */
7066                 total_bytes += tx_buffer->bytecount;
7067                 total_packets += tx_buffer->gso_segs;
7068
7069                 /* free the skb */
7070                 dev_kfree_skb_any(tx_buffer->skb);
7071
7072                 /* unmap skb header data */
7073                 dma_unmap_single(tx_ring->dev,
7074                                  dma_unmap_addr(tx_buffer, dma),
7075                                  dma_unmap_len(tx_buffer, len),
7076                                  DMA_TO_DEVICE);
7077
7078                 /* clear tx_buffer data */
7079                 tx_buffer->skb = NULL;
7080                 dma_unmap_len_set(tx_buffer, len, 0);
7081
7082                 /* clear last DMA location and unmap remaining buffers */
7083                 while (tx_desc != eop_desc) {
7084                         tx_buffer++;
7085                         tx_desc++;
7086                         i++;
7087                         if (unlikely(!i)) {
7088                                 i -= tx_ring->count;
7089                                 tx_buffer = tx_ring->tx_buffer_info;
7090                                 tx_desc = IGB_TX_DESC(tx_ring, 0);
7091                         }
7092
7093                         /* unmap any remaining paged data */
7094                         if (dma_unmap_len(tx_buffer, len)) {
7095                                 dma_unmap_page(tx_ring->dev,
7096                                                dma_unmap_addr(tx_buffer, dma),
7097                                                dma_unmap_len(tx_buffer, len),
7098                                                DMA_TO_DEVICE);
7099                                 dma_unmap_len_set(tx_buffer, len, 0);
7100                         }
7101                 }
7102
7103                 /* move us one more past the eop_desc for start of next pkt */
7104                 tx_buffer++;
7105                 tx_desc++;
7106                 i++;
7107                 if (unlikely(!i)) {
7108                         i -= tx_ring->count;
7109                         tx_buffer = tx_ring->tx_buffer_info;
7110                         tx_desc = IGB_TX_DESC(tx_ring, 0);
7111                 }
7112
7113                 /* issue prefetch for next Tx descriptor */
7114                 prefetch(tx_desc);
7115
7116                 /* update budget accounting */
7117                 budget--;
7118         } while (likely(budget));
7119
7120         netdev_tx_completed_queue(txring_txq(tx_ring),
7121                                   total_packets, total_bytes);
7122
7123         i += tx_ring->count;
7124         tx_ring->next_to_clean = i;
7125         tx_ring->tx_stats.bytes += total_bytes;
7126         tx_ring->tx_stats.packets += total_packets;
7127         q_vector->tx.total_bytes += total_bytes;
7128         q_vector->tx.total_packets += total_packets;
7129
7130 #ifdef DEBUG
7131         if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags) &&
7132             !(adapter->disable_hw_reset && adapter->tx_hang_detected)) {
7133 #else
7134         if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
7135 #endif
7136                 struct e1000_hw *hw = &adapter->hw;
7137
7138                 /* Detect a transmit hang in hardware, this serializes the
7139                  * check with the clearing of time_stamp and movement of i */
7140                 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
7141                 if (tx_buffer->next_to_watch &&
7142                     time_after(jiffies, tx_buffer->time_stamp +
7143                                (adapter->tx_timeout_factor * HZ))
7144                     && !(E1000_READ_REG(hw, E1000_STATUS) &
7145                          E1000_STATUS_TXOFF)) {
7146
7147                         /* detected Tx unit hang */
7148 #ifdef DEBUG
7149                         adapter->tx_hang_detected = TRUE;
7150                         if (adapter->disable_hw_reset) {
7151                                 DPRINTK(DRV, WARNING,
7152                                         "Deactivating netdev watchdog timer\n");
7153                                 if (del_timer(&netdev_ring(tx_ring)->watchdog_timer))
7154                                         dev_put(netdev_ring(tx_ring));
7155 #ifndef HAVE_NET_DEVICE_OPS
7156                                 netdev_ring(tx_ring)->tx_timeout = NULL;
7157 #endif
7158                         }
7159 #endif /* DEBUG */
7160                         dev_err(tx_ring->dev,
7161                                 "Detected Tx Unit Hang\n"
7162                                 "  Tx Queue             <%d>\n"
7163                                 "  TDH                  <%x>\n"
7164                                 "  TDT                  <%x>\n"
7165                                 "  next_to_use          <%x>\n"
7166                                 "  next_to_clean        <%x>\n"
7167                                 "buffer_info[next_to_clean]\n"
7168                                 "  time_stamp           <%lx>\n"
7169                                 "  next_to_watch        <%p>\n"
7170                                 "  jiffies              <%lx>\n"
7171                                 "  desc.status          <%x>\n",
7172                                 tx_ring->queue_index,
7173                                 E1000_READ_REG(hw, E1000_TDH(tx_ring->reg_idx)),
7174                                 readl(tx_ring->tail),
7175                                 tx_ring->next_to_use,
7176                                 tx_ring->next_to_clean,
7177                                 tx_buffer->time_stamp,
7178                                 tx_buffer->next_to_watch,
7179                                 jiffies,
7180                                 tx_buffer->next_to_watch->wb.status);
7181                         if (netif_is_multiqueue(netdev_ring(tx_ring)))
7182                                 netif_stop_subqueue(netdev_ring(tx_ring),
7183                                                     ring_queue_index(tx_ring));
7184                         else
7185                                 netif_stop_queue(netdev_ring(tx_ring));
7186
7187                         /* we are about to reset, no point in enabling stuff */
7188                         return true;
7189                 }
7190         }
7191
7192 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
7193         if (unlikely(total_packets &&
7194                      netif_carrier_ok(netdev_ring(tx_ring)) &&
7195                      igb_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
7196                 /* Make sure that anybody stopping the queue after this
7197                  * sees the new next_to_clean.
7198                  */
7199                 smp_mb();
7200                 if (netif_is_multiqueue(netdev_ring(tx_ring))) {
7201                         if (__netif_subqueue_stopped(netdev_ring(tx_ring),
7202                                                      ring_queue_index(tx_ring)) &&
7203                             !(test_bit(__IGB_DOWN, &adapter->state))) {
7204                                 netif_wake_subqueue(netdev_ring(tx_ring),
7205                                                     ring_queue_index(tx_ring));
7206                                 tx_ring->tx_stats.restart_queue++;
7207                         }
7208                 } else {
7209                         if (netif_queue_stopped(netdev_ring(tx_ring)) &&
7210                             !(test_bit(__IGB_DOWN, &adapter->state))) {
7211                                 netif_wake_queue(netdev_ring(tx_ring));
7212                                 tx_ring->tx_stats.restart_queue++;
7213                         }
7214                 }
7215         }
7216
7217         return !!budget;
7218 }
7219
7220 #ifdef HAVE_VLAN_RX_REGISTER
7221 /**
7222  * igb_receive_skb - helper function to handle rx indications
7223  * @q_vector: structure containing interrupt and ring information
7224  * @skb: packet to send up
7225  **/
7226 static void igb_receive_skb(struct igb_q_vector *q_vector,
7227                             struct sk_buff *skb)
7228 {
7229         struct vlan_group **vlgrp = netdev_priv(skb->dev);
7230
7231         if (IGB_CB(skb)->vid) {
7232                 if (*vlgrp) {
7233                         vlan_gro_receive(&q_vector->napi, *vlgrp,
7234                                          IGB_CB(skb)->vid, skb);
7235                 } else {
7236                         dev_kfree_skb_any(skb);
7237                 }
7238         } else {
7239                 napi_gro_receive(&q_vector->napi, skb);
7240         }
7241 }
7242
7243 #endif /* HAVE_VLAN_RX_REGISTER */
7244 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7245 /**
7246  * igb_reuse_rx_page - page flip buffer and store it back on the ring
7247  * @rx_ring: rx descriptor ring to store buffers on
7248  * @old_buff: donor buffer to have page reused
7249  *
7250  * Synchronizes page for reuse by the adapter
7251  **/
7252 static void igb_reuse_rx_page(struct igb_ring *rx_ring,
7253                               struct igb_rx_buffer *old_buff)
7254 {
7255         struct igb_rx_buffer *new_buff;
7256         u16 nta = rx_ring->next_to_alloc;
7257
7258         new_buff = &rx_ring->rx_buffer_info[nta];
7259
7260         /* update, and store next to alloc */
7261         nta++;
7262         rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
7263
7264         /* transfer page from old buffer to new buffer */
7265         memcpy(new_buff, old_buff, sizeof(struct igb_rx_buffer));
7266
7267         /* sync the buffer for use by the device */
7268         dma_sync_single_range_for_device(rx_ring->dev, old_buff->dma,
7269                                          old_buff->page_offset,
7270                                          IGB_RX_BUFSZ,
7271                                          DMA_FROM_DEVICE);
7272 }
7273
7274 static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,
7275                                   struct page *page,
7276                                   unsigned int truesize)
7277 {
7278         /* avoid re-using remote pages */
7279         if (unlikely(page_to_nid(page) != numa_node_id()))
7280                 return false;
7281
7282 #if (PAGE_SIZE < 8192)
7283         /* if we are only owner of page we can reuse it */
7284         if (unlikely(page_count(page) != 1))
7285                 return false;
7286
7287         /* flip page offset to other buffer */
7288         rx_buffer->page_offset ^= IGB_RX_BUFSZ;
7289
7290 #else
7291         /* move offset up to the next cache line */
7292         rx_buffer->page_offset += truesize;
7293
7294         if (rx_buffer->page_offset > (PAGE_SIZE - IGB_RX_BUFSZ))
7295                 return false;
7296 #endif
7297
7298         /* bump ref count on page before it is given to the stack */
7299         get_page(page);
7300
7301         return true;
7302 }
7303
7304 /**
7305  * igb_add_rx_frag - Add contents of Rx buffer to sk_buff
7306  * @rx_ring: rx descriptor ring to transact packets on
7307  * @rx_buffer: buffer containing page to add
7308  * @rx_desc: descriptor containing length of buffer written by hardware
7309  * @skb: sk_buff to place the data into
7310  *
7311  * This function will add the data contained in rx_buffer->page to the skb.
7312  * This is done either through a direct copy if the data in the buffer is
7313  * less than the skb header size, otherwise it will just attach the page as
7314  * a frag to the skb.
7315  *
7316  * The function will then update the page offset if necessary and return
7317  * true if the buffer can be reused by the adapter.
7318  **/
7319 static bool igb_add_rx_frag(struct igb_ring *rx_ring,
7320                             struct igb_rx_buffer *rx_buffer,
7321                             union e1000_adv_rx_desc *rx_desc,
7322                             struct sk_buff *skb)
7323 {
7324         struct page *page = rx_buffer->page;
7325         unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
7326 #if (PAGE_SIZE < 8192)
7327         unsigned int truesize = IGB_RX_BUFSZ;
7328 #else
7329         unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
7330 #endif
7331
7332         if ((size <= IGB_RX_HDR_LEN) && !skb_is_nonlinear(skb)) {
7333                 unsigned char *va = page_address(page) + rx_buffer->page_offset;
7334
7335 #ifdef HAVE_PTP_1588_CLOCK
7336                 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
7337                         igb_ptp_rx_pktstamp(rx_ring->q_vector, va, skb);
7338                         va += IGB_TS_HDR_LEN;
7339                         size -= IGB_TS_HDR_LEN;
7340                 }
7341 #endif /* HAVE_PTP_1588_CLOCK */
7342
7343                 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));
7344
7345                 /* we can reuse buffer as-is, just make sure it is local */
7346                 if (likely(page_to_nid(page) == numa_node_id()))
7347                         return true;
7348
7349                 /* this page cannot be reused so discard it */
7350                 put_page(page);
7351                 return false;
7352         }
7353
7354         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
7355                         rx_buffer->page_offset, size, truesize);
7356
7357         return igb_can_reuse_rx_page(rx_buffer, page, truesize);
7358 }
7359
7360 static struct sk_buff *igb_fetch_rx_buffer(struct igb_ring *rx_ring,
7361                                            union e1000_adv_rx_desc *rx_desc,
7362                                            struct sk_buff *skb)
7363 {
7364         struct igb_rx_buffer *rx_buffer;
7365         struct page *page;
7366
7367         rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
7368
7369         page = rx_buffer->page;
7370         prefetchw(page);
7371
7372         if (likely(!skb)) {
7373                 void *page_addr = page_address(page) +
7374                                   rx_buffer->page_offset;
7375
7376                 /* prefetch first cache line of first page */
7377                 prefetch(page_addr);
7378 #if L1_CACHE_BYTES < 128
7379                 prefetch(page_addr + L1_CACHE_BYTES);
7380 #endif
7381
7382                 /* allocate a skb to store the frags */
7383                 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
7384                                                 IGB_RX_HDR_LEN);
7385                 if (unlikely(!skb)) {
7386                         rx_ring->rx_stats.alloc_failed++;
7387                         return NULL;
7388                 }
7389
7390                 /*
7391                  * we will be copying header into skb->data in
7392                  * pskb_may_pull so it is in our interest to prefetch
7393                  * it now to avoid a possible cache miss
7394                  */
7395                 prefetchw(skb->data);
7396         }
7397
7398         /* we are reusing so sync this buffer for CPU use */
7399         dma_sync_single_range_for_cpu(rx_ring->dev,
7400                                       rx_buffer->dma,
7401                                       rx_buffer->page_offset,
7402                                       IGB_RX_BUFSZ,
7403                                       DMA_FROM_DEVICE);
7404
7405         /* pull page into skb */
7406         if (igb_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) {
7407                 /* hand second half of page back to the ring */
7408                 igb_reuse_rx_page(rx_ring, rx_buffer);
7409         } else {
7410                 /* we are not reusing the buffer so unmap it */
7411                 dma_unmap_page(rx_ring->dev, rx_buffer->dma,
7412                                PAGE_SIZE, DMA_FROM_DEVICE);
7413         }
7414
7415         /* clear contents of rx_buffer */
7416         rx_buffer->page = NULL;
7417
7418         return skb;
7419 }
7420
7421 #endif
7422 static inline void igb_rx_checksum(struct igb_ring *ring,
7423                                    union e1000_adv_rx_desc *rx_desc,
7424                                    struct sk_buff *skb)
7425 {
7426         skb_checksum_none_assert(skb);
7427
7428         /* Ignore Checksum bit is set */
7429         if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
7430                 return;
7431
7432         /* Rx checksum disabled via ethtool */
7433         if (!(netdev_ring(ring)->features & NETIF_F_RXCSUM))
7434                 return;
7435
7436         /* TCP/UDP checksum error bit is set */
7437         if (igb_test_staterr(rx_desc,
7438                              E1000_RXDEXT_STATERR_TCPE |
7439                              E1000_RXDEXT_STATERR_IPE)) {
7440                 /*
7441                  * work around errata with sctp packets where the TCPE aka
7442                  * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
7443                  * packets, (aka let the stack check the crc32c)
7444                  */
7445                 if (!((skb->len == 60) &&
7446                       test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags)))
7447                         ring->rx_stats.csum_err++;
7448
7449                 /* let the stack verify checksum errors */
7450                 return;
7451         }
7452         /* It must be a TCP or UDP packet with a valid checksum */
7453         if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
7454                                       E1000_RXD_STAT_UDPCS))
7455                 skb->ip_summed = CHECKSUM_UNNECESSARY;
7456 }
7457
7458 #ifdef NETIF_F_RXHASH
7459 static inline void igb_rx_hash(struct igb_ring *ring,
7460                                union e1000_adv_rx_desc *rx_desc,
7461                                struct sk_buff *skb)
7462 {
7463         if (netdev_ring(ring)->features & NETIF_F_RXHASH)
7464                 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
7465                              PKT_HASH_TYPE_L3);
7466 }
7467
7468 #endif
7469 #ifndef IGB_NO_LRO
7470 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7471 /**
7472  * igb_merge_active_tail - merge active tail into lro skb
7473  * @tail: pointer to active tail in frag_list
7474  *
7475  * This function merges the length and data of an active tail into the
7476  * skb containing the frag_list.  It resets the tail's pointer to the head,
7477  * but it leaves the heads pointer to tail intact.
7478  **/
7479 static inline struct sk_buff *igb_merge_active_tail(struct sk_buff *tail)
7480 {
7481         struct sk_buff *head = IGB_CB(tail)->head;
7482
7483         if (!head)
7484                 return tail;
7485
7486         head->len += tail->len;
7487         head->data_len += tail->len;
7488         head->truesize += tail->len;
7489
7490         IGB_CB(tail)->head = NULL;
7491
7492         return head;
7493 }
7494
7495 /**
7496  * igb_add_active_tail - adds an active tail into the skb frag_list
7497  * @head: pointer to the start of the skb
7498  * @tail: pointer to active tail to add to frag_list
7499  *
7500  * This function adds an active tail to the end of the frag list.  This tail
7501  * will still be receiving data so we cannot yet ad it's stats to the main
7502  * skb.  That is done via igb_merge_active_tail.
7503  **/
7504 static inline void igb_add_active_tail(struct sk_buff *head, struct sk_buff *tail)
7505 {
7506         struct sk_buff *old_tail = IGB_CB(head)->tail;
7507
7508         if (old_tail) {
7509                 igb_merge_active_tail(old_tail);
7510                 old_tail->next = tail;
7511         } else {
7512                 skb_shinfo(head)->frag_list = tail;
7513         }
7514
7515         IGB_CB(tail)->head = head;
7516         IGB_CB(head)->tail = tail;
7517
7518         IGB_CB(head)->append_cnt++;
7519 }
7520
7521 /**
7522  * igb_close_active_frag_list - cleanup pointers on a frag_list skb
7523  * @head: pointer to head of an active frag list
7524  *
7525  * This function will clear the frag_tail_tracker pointer on an active
7526  * frag_list and returns true if the pointer was actually set
7527  **/
7528 static inline bool igb_close_active_frag_list(struct sk_buff *head)
7529 {
7530         struct sk_buff *tail = IGB_CB(head)->tail;
7531
7532         if (!tail)
7533                 return false;
7534
7535         igb_merge_active_tail(tail);
7536
7537         IGB_CB(head)->tail = NULL;
7538
7539         return true;
7540 }
7541
7542 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7543 /**
7544  * igb_can_lro - returns true if packet is TCP/IPV4 and LRO is enabled
7545  * @adapter: board private structure
7546  * @rx_desc: pointer to the rx descriptor
7547  * @skb: pointer to the skb to be merged
7548  *
7549  **/
7550 static inline bool igb_can_lro(struct igb_ring *rx_ring,
7551                                union e1000_adv_rx_desc *rx_desc,
7552                                struct sk_buff *skb)
7553 {
7554         struct iphdr *iph = (struct iphdr *)skb->data;
7555         __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
7556
7557         /* verify hardware indicates this is IPv4/TCP */
7558         if((!(pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_TCP)) ||
7559             !(pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_IPV4))))
7560                 return false;
7561
7562         /* .. and LRO is enabled */
7563         if (!(netdev_ring(rx_ring)->features & NETIF_F_LRO))
7564                 return false;
7565
7566         /* .. and we are not in promiscuous mode */
7567         if (netdev_ring(rx_ring)->flags & IFF_PROMISC)
7568                 return false;
7569
7570         /* .. and the header is large enough for us to read IP/TCP fields */
7571         if (!pskb_may_pull(skb, sizeof(struct igb_lrohdr)))
7572                 return false;
7573
7574         /* .. and there are no VLANs on packet */
7575         if (skb->protocol != __constant_htons(ETH_P_IP))
7576                 return false;
7577
7578         /* .. and we are version 4 with no options */
7579         if (*(u8 *)iph != 0x45)
7580                 return false;
7581
7582         /* .. and the packet is not fragmented */
7583         if (iph->frag_off & htons(IP_MF | IP_OFFSET))
7584                 return false;
7585
7586         /* .. and that next header is TCP */
7587         if (iph->protocol != IPPROTO_TCP)
7588                 return false;
7589
7590         return true;
7591 }
7592
7593 static inline struct igb_lrohdr *igb_lro_hdr(struct sk_buff *skb)
7594 {
7595         return (struct igb_lrohdr *)skb->data;
7596 }
7597
7598 /**
7599  * igb_lro_flush - Indicate packets to upper layer.
7600  *
7601  * Update IP and TCP header part of head skb if more than one
7602  * skb's chained and indicate packets to upper layer.
7603  **/
7604 static void igb_lro_flush(struct igb_q_vector *q_vector,
7605                           struct sk_buff *skb)
7606 {
7607         struct igb_lro_list *lrolist = &q_vector->lrolist;
7608
7609         __skb_unlink(skb, &lrolist->active);
7610
7611         if (IGB_CB(skb)->append_cnt) {
7612                 struct igb_lrohdr *lroh = igb_lro_hdr(skb);
7613
7614 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7615                 /* close any active lro contexts */
7616                 igb_close_active_frag_list(skb);
7617
7618 #endif
7619                 /* incorporate ip header and re-calculate checksum */
7620                 lroh->iph.tot_len = ntohs(skb->len);
7621                 lroh->iph.check = 0;
7622
7623                 /* header length is 5 since we know no options exist */
7624                 lroh->iph.check = ip_fast_csum((u8 *)lroh, 5);
7625
7626                 /* clear TCP checksum to indicate we are an LRO frame */
7627                 lroh->th.check = 0;
7628
7629                 /* incorporate latest timestamp into the tcp header */
7630                 if (IGB_CB(skb)->tsecr) {
7631                         lroh->ts[2] = IGB_CB(skb)->tsecr;
7632                         lroh->ts[1] = htonl(IGB_CB(skb)->tsval);
7633                 }
7634 #ifdef NETIF_F_GSO
7635
7636                 skb_shinfo(skb)->gso_size = IGB_CB(skb)->mss;
7637                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
7638 #endif
7639         }
7640
7641 #ifdef HAVE_VLAN_RX_REGISTER
7642         igb_receive_skb(q_vector, skb);
7643 #else
7644         napi_gro_receive(&q_vector->napi, skb);
7645 #endif
7646         lrolist->stats.flushed++;
7647 }
7648
7649 static void igb_lro_flush_all(struct igb_q_vector *q_vector)
7650 {
7651         struct igb_lro_list *lrolist = &q_vector->lrolist;
7652         struct sk_buff *skb, *tmp;
7653
7654         skb_queue_reverse_walk_safe(&lrolist->active, skb, tmp)
7655                 igb_lro_flush(q_vector, skb);
7656 }
7657
7658 /*
7659  * igb_lro_header_ok - Main LRO function.
7660  **/
7661 static void igb_lro_header_ok(struct sk_buff *skb)
7662 {
7663         struct igb_lrohdr *lroh = igb_lro_hdr(skb);
7664         u16 opt_bytes, data_len;
7665
7666 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7667         IGB_CB(skb)->tail = NULL;
7668 #endif
7669         IGB_CB(skb)->tsecr = 0;
7670         IGB_CB(skb)->append_cnt = 0;
7671         IGB_CB(skb)->mss = 0;
7672
7673         /* ensure that the checksum is valid */
7674         if (skb->ip_summed != CHECKSUM_UNNECESSARY)
7675                 return;
7676
7677         /* If we see CE codepoint in IP header, packet is not mergeable */
7678         if (INET_ECN_is_ce(ipv4_get_dsfield(&lroh->iph)))
7679                 return;
7680
7681         /* ensure no bits set besides ack or psh */
7682         if (lroh->th.fin || lroh->th.syn || lroh->th.rst ||
7683             lroh->th.urg || lroh->th.ece || lroh->th.cwr ||
7684             !lroh->th.ack)
7685                 return;
7686
7687         /* store the total packet length */
7688         data_len = ntohs(lroh->iph.tot_len);
7689
7690         /* remove any padding from the end of the skb */
7691         __pskb_trim(skb, data_len);
7692
7693         /* remove header length from data length */
7694         data_len -= sizeof(struct igb_lrohdr);
7695
7696         /*
7697          * check for timestamps. Since the only option we handle are timestamps,
7698          * we only have to handle the simple case of aligned timestamps
7699          */
7700         opt_bytes = (lroh->th.doff << 2) - sizeof(struct tcphdr);
7701         if (opt_bytes != 0) {
7702                 if ((opt_bytes != TCPOLEN_TSTAMP_ALIGNED) ||
7703                     !pskb_may_pull(skb, sizeof(struct igb_lrohdr) +
7704                                         TCPOLEN_TSTAMP_ALIGNED) ||
7705                     (lroh->ts[0] != htonl((TCPOPT_NOP << 24) |
7706                                              (TCPOPT_NOP << 16) |
7707                                              (TCPOPT_TIMESTAMP << 8) |
7708                                               TCPOLEN_TIMESTAMP)) ||
7709                     (lroh->ts[2] == 0)) {
7710                         return;
7711                 }
7712
7713                 IGB_CB(skb)->tsval = ntohl(lroh->ts[1]);
7714                 IGB_CB(skb)->tsecr = lroh->ts[2];
7715
7716                 data_len -= TCPOLEN_TSTAMP_ALIGNED;
7717         }
7718
7719         /* record data_len as mss for the packet */
7720         IGB_CB(skb)->mss = data_len;
7721         IGB_CB(skb)->next_seq = ntohl(lroh->th.seq);
7722 }
7723
7724 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7725 static void igb_merge_frags(struct sk_buff *lro_skb, struct sk_buff *new_skb)
7726 {
7727         struct skb_shared_info *sh_info;
7728         struct skb_shared_info *new_skb_info;
7729         unsigned int data_len;
7730
7731         sh_info = skb_shinfo(lro_skb);
7732         new_skb_info = skb_shinfo(new_skb);
7733
7734         /* copy frags into the last skb */
7735         memcpy(sh_info->frags + sh_info->nr_frags,
7736                new_skb_info->frags,
7737                new_skb_info->nr_frags * sizeof(skb_frag_t));
7738
7739         /* copy size data over */
7740         sh_info->nr_frags += new_skb_info->nr_frags;
7741         data_len = IGB_CB(new_skb)->mss;
7742         lro_skb->len += data_len;
7743         lro_skb->data_len += data_len;
7744         lro_skb->truesize += data_len;
7745
7746         /* wipe record of data from new_skb */
7747         new_skb_info->nr_frags = 0;
7748         new_skb->len = new_skb->data_len = 0;
7749         dev_kfree_skb_any(new_skb);
7750 }
7751
7752 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7753 /**
7754  * igb_lro_receive - if able, queue skb into lro chain
7755  * @q_vector: structure containing interrupt and ring information
7756  * @new_skb: pointer to current skb being checked
7757  *
7758  * Checks whether the skb given is eligible for LRO and if that's
7759  * fine chains it to the existing lro_skb based on flowid. If an LRO for
7760  * the flow doesn't exist create one.
7761  **/
7762 static void igb_lro_receive(struct igb_q_vector *q_vector,
7763                             struct sk_buff *new_skb)
7764 {
7765         struct sk_buff *lro_skb;
7766         struct igb_lro_list *lrolist = &q_vector->lrolist;
7767         struct igb_lrohdr *lroh = igb_lro_hdr(new_skb);
7768         __be32 saddr = lroh->iph.saddr;
7769         __be32 daddr = lroh->iph.daddr;
7770         __be32 tcp_ports = *(__be32 *)&lroh->th;
7771         u16 data_len;
7772 #ifdef HAVE_VLAN_RX_REGISTER
7773         u16 vid = IGB_CB(new_skb)->vid;
7774 #else
7775         u16 vid = new_skb->vlan_tci;
7776 #endif
7777
7778         igb_lro_header_ok(new_skb);
7779
7780         /*
7781          * we have a packet that might be eligible for LRO,
7782          * so see if it matches anything we might expect
7783          */
7784         skb_queue_walk(&lrolist->active, lro_skb) {
7785                 if (*(__be32 *)&igb_lro_hdr(lro_skb)->th != tcp_ports ||
7786                     igb_lro_hdr(lro_skb)->iph.saddr != saddr ||
7787                     igb_lro_hdr(lro_skb)->iph.daddr != daddr)
7788                         continue;
7789
7790 #ifdef HAVE_VLAN_RX_REGISTER
7791                 if (IGB_CB(lro_skb)->vid != vid)
7792 #else
7793                 if (lro_skb->vlan_tci != vid)
7794 #endif
7795                         continue;
7796
7797                 /* out of order packet */
7798                 if (IGB_CB(lro_skb)->next_seq != IGB_CB(new_skb)->next_seq) {
7799                         igb_lro_flush(q_vector, lro_skb);
7800                         IGB_CB(new_skb)->mss = 0;
7801                         break;
7802                 }
7803
7804                 /* TCP timestamp options have changed */
7805                 if (!IGB_CB(lro_skb)->tsecr != !IGB_CB(new_skb)->tsecr) {
7806                         igb_lro_flush(q_vector, lro_skb);
7807                         break;
7808                 }
7809
7810                 /* make sure timestamp values are increasing */
7811                 if (IGB_CB(lro_skb)->tsecr &&
7812                     IGB_CB(lro_skb)->tsval > IGB_CB(new_skb)->tsval) {
7813                         igb_lro_flush(q_vector, lro_skb);
7814                         IGB_CB(new_skb)->mss = 0;
7815                         break;
7816                 }
7817
7818                 data_len = IGB_CB(new_skb)->mss;
7819
7820                 /* Check for all of the above below
7821                  *   malformed header
7822                  *   no tcp data
7823                  *   resultant packet would be too large
7824                  *   new skb is larger than our current mss
7825                  *   data would remain in header
7826                  *   we would consume more frags then the sk_buff contains
7827                  *   ack sequence numbers changed
7828                  *   window size has changed
7829                  */
7830                 if (data_len == 0 ||
7831                     data_len > IGB_CB(lro_skb)->mss ||
7832                     data_len > IGB_CB(lro_skb)->free ||
7833 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7834                     data_len != new_skb->data_len ||
7835                     skb_shinfo(new_skb)->nr_frags >=
7836                     (MAX_SKB_FRAGS - skb_shinfo(lro_skb)->nr_frags) ||
7837 #endif
7838                     igb_lro_hdr(lro_skb)->th.ack_seq != lroh->th.ack_seq ||
7839                     igb_lro_hdr(lro_skb)->th.window != lroh->th.window) {
7840                         igb_lro_flush(q_vector, lro_skb);
7841                         break;
7842                 }
7843
7844                 /* Remove IP and TCP header*/
7845                 skb_pull(new_skb, new_skb->len - data_len);
7846
7847                 /* update timestamp and timestamp echo response */
7848                 IGB_CB(lro_skb)->tsval = IGB_CB(new_skb)->tsval;
7849                 IGB_CB(lro_skb)->tsecr = IGB_CB(new_skb)->tsecr;
7850
7851                 /* update sequence and free space */
7852                 IGB_CB(lro_skb)->next_seq += data_len;
7853                 IGB_CB(lro_skb)->free -= data_len;
7854
7855                 /* update append_cnt */
7856                 IGB_CB(lro_skb)->append_cnt++;
7857
7858 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7859                 /* if header is empty pull pages into current skb */
7860                 igb_merge_frags(lro_skb, new_skb);
7861 #else
7862                 /* chain this new skb in frag_list */
7863                 igb_add_active_tail(lro_skb, new_skb);
7864 #endif
7865
7866                 if ((data_len < IGB_CB(lro_skb)->mss) || lroh->th.psh ||
7867                     skb_shinfo(lro_skb)->nr_frags == MAX_SKB_FRAGS) {
7868                         igb_lro_hdr(lro_skb)->th.psh |= lroh->th.psh;
7869                         igb_lro_flush(q_vector, lro_skb);
7870                 }
7871
7872                 lrolist->stats.coal++;
7873                 return;
7874         }
7875
7876         if (IGB_CB(new_skb)->mss && !lroh->th.psh) {
7877                 /* if we are at capacity flush the tail */
7878                 if (skb_queue_len(&lrolist->active) >= IGB_LRO_MAX) {
7879                         lro_skb = skb_peek_tail(&lrolist->active);
7880                         if (lro_skb)
7881                                 igb_lro_flush(q_vector, lro_skb);
7882                 }
7883
7884                 /* update sequence and free space */
7885                 IGB_CB(new_skb)->next_seq += IGB_CB(new_skb)->mss;
7886                 IGB_CB(new_skb)->free = 65521 - new_skb->len;
7887
7888                 /* .. and insert at the front of the active list */
7889                 __skb_queue_head(&lrolist->active, new_skb);
7890
7891                 lrolist->stats.coal++;
7892                 return;
7893         }
7894
7895         /* packet not handled by any of the above, pass it to the stack */
7896 #ifdef HAVE_VLAN_RX_REGISTER
7897         igb_receive_skb(q_vector, new_skb);
7898 #else
7899         napi_gro_receive(&q_vector->napi, new_skb);
7900 #endif
7901 }
7902
7903 #endif /* IGB_NO_LRO */
7904 /**
7905  * igb_process_skb_fields - Populate skb header fields from Rx descriptor
7906  * @rx_ring: rx descriptor ring packet is being transacted on
7907  * @rx_desc: pointer to the EOP Rx descriptor
7908  * @skb: pointer to current skb being populated
7909  *
7910  * This function checks the ring, descriptor, and packet information in
7911  * order to populate the hash, checksum, VLAN, timestamp, protocol, and
7912  * other fields within the skb.
7913  **/
7914 static void igb_process_skb_fields(struct igb_ring *rx_ring,
7915                                    union e1000_adv_rx_desc *rx_desc,
7916                                    struct sk_buff *skb)
7917 {
7918         struct net_device *dev = rx_ring->netdev;
7919         __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
7920
7921 #ifdef NETIF_F_RXHASH
7922         igb_rx_hash(rx_ring, rx_desc, skb);
7923
7924 #endif
7925         igb_rx_checksum(rx_ring, rx_desc, skb);
7926
7927     /* update packet type stats */
7928         if (pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_IPV4))
7929                 rx_ring->rx_stats.ipv4_packets++;
7930         else if (pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_IPV4_EX))
7931                 rx_ring->rx_stats.ipv4e_packets++;
7932         else if (pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_IPV6))
7933                 rx_ring->rx_stats.ipv6_packets++;
7934         else if (pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_IPV6_EX))
7935                 rx_ring->rx_stats.ipv6e_packets++;
7936         else if (pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_TCP))
7937                 rx_ring->rx_stats.tcp_packets++;
7938         else if (pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_UDP))
7939                 rx_ring->rx_stats.udp_packets++;
7940         else if (pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_SCTP))
7941                 rx_ring->rx_stats.sctp_packets++;
7942         else if (pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_NFS))
7943                 rx_ring->rx_stats.nfs_packets++;
7944
7945 #ifdef HAVE_PTP_1588_CLOCK
7946         igb_ptp_rx_hwtstamp(rx_ring, rx_desc, skb);
7947 #endif /* HAVE_PTP_1588_CLOCK */
7948
7949 #ifdef NETIF_F_HW_VLAN_CTAG_RX
7950         if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
7951 #else
7952         if ((dev->features & NETIF_F_HW_VLAN_RX) &&
7953 #endif
7954             igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
7955                 u16 vid = 0;
7956                 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
7957                     test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags))
7958                         vid = be16_to_cpu(rx_desc->wb.upper.vlan);
7959                 else
7960                         vid = le16_to_cpu(rx_desc->wb.upper.vlan);
7961 #ifdef HAVE_VLAN_RX_REGISTER
7962                 IGB_CB(skb)->vid = vid;
7963         } else {
7964                 IGB_CB(skb)->vid = 0;
7965 #else
7966
7967 #ifdef HAVE_VLAN_PROTOCOL
7968                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
7969 #else
7970                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
7971 #endif
7972
7973
7974 #endif
7975         }
7976
7977         skb_record_rx_queue(skb, rx_ring->queue_index);
7978
7979         skb->protocol = eth_type_trans(skb, dev);
7980 }
7981
7982 /**
7983  * igb_is_non_eop - process handling of non-EOP buffers
7984  * @rx_ring: Rx ring being processed
7985  * @rx_desc: Rx descriptor for current buffer
7986  *
7987  * This function updates next to clean.  If the buffer is an EOP buffer
7988  * this function exits returning false, otherwise it will place the
7989  * sk_buff in the next buffer to be chained and return true indicating
7990  * that this is in fact a non-EOP buffer.
7991  **/
7992 static bool igb_is_non_eop(struct igb_ring *rx_ring,
7993                            union e1000_adv_rx_desc *rx_desc)
7994 {
7995         u32 ntc = rx_ring->next_to_clean + 1;
7996
7997         /* fetch, update, and store next to clean */
7998         ntc = (ntc < rx_ring->count) ? ntc : 0;
7999         rx_ring->next_to_clean = ntc;
8000
8001         prefetch(IGB_RX_DESC(rx_ring, ntc));
8002
8003         if (likely(igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)))
8004                 return false;
8005
8006         return true;
8007 }
8008
8009 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
8010 /* igb_clean_rx_irq -- * legacy */
8011 static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
8012 {
8013         struct igb_ring *rx_ring = q_vector->rx.ring;
8014         unsigned int total_bytes = 0, total_packets = 0;
8015         u16 cleaned_count = igb_desc_unused(rx_ring);
8016
8017         do {
8018                 struct igb_rx_buffer *rx_buffer;
8019                 union e1000_adv_rx_desc *rx_desc;
8020                 struct sk_buff *skb;
8021                 u16 ntc;
8022
8023                 /* return some buffers to hardware, one at a time is too slow */
8024                 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
8025                         igb_alloc_rx_buffers(rx_ring, cleaned_count);
8026                         cleaned_count = 0;
8027                 }
8028
8029                 ntc = rx_ring->next_to_clean;
8030                 rx_desc = IGB_RX_DESC(rx_ring, ntc);
8031                 rx_buffer = &rx_ring->rx_buffer_info[ntc];
8032
8033                 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_DD))
8034                         break;
8035
8036                 /*
8037                  * This memory barrier is needed to keep us from reading
8038                  * any other fields out of the rx_desc until we know the
8039                  * RXD_STAT_DD bit is set
8040                  */
8041                 rmb();
8042
8043                 skb = rx_buffer->skb;
8044
8045                 prefetch(skb->data);
8046
8047                 /* pull the header of the skb in */
8048                 __skb_put(skb, le16_to_cpu(rx_desc->wb.upper.length));
8049
8050                 /* clear skb reference in buffer info structure */
8051                 rx_buffer->skb = NULL;
8052
8053                 cleaned_count++;
8054
8055                 BUG_ON(igb_is_non_eop(rx_ring, rx_desc));
8056
8057                 dma_unmap_single(rx_ring->dev, rx_buffer->dma,
8058                                  rx_ring->rx_buffer_len,
8059                                  DMA_FROM_DEVICE);
8060                 rx_buffer->dma = 0;
8061
8062                 if (igb_test_staterr(rx_desc,
8063                                      E1000_RXDEXT_ERR_FRAME_ERR_MASK)) {
8064                         dev_kfree_skb_any(skb);
8065                         continue;
8066                 }
8067
8068                 total_bytes += skb->len;
8069
8070                 /* populate checksum, timestamp, VLAN, and protocol */
8071                 igb_process_skb_fields(rx_ring, rx_desc, skb);
8072
8073 #ifndef IGB_NO_LRO
8074                 if (igb_can_lro(rx_ring, rx_desc, skb))
8075                         igb_lro_receive(q_vector, skb);
8076                 else
8077 #endif
8078 #ifdef HAVE_VLAN_RX_REGISTER
8079                         igb_receive_skb(q_vector, skb);
8080 #else
8081                         napi_gro_receive(&q_vector->napi, skb);
8082 #endif
8083
8084 #ifndef NETIF_F_GRO
8085                 netdev_ring(rx_ring)->last_rx = jiffies;
8086
8087 #endif
8088                 /* update budget accounting */
8089                 total_packets++;
8090         } while (likely(total_packets < budget));
8091
8092         rx_ring->rx_stats.packets += total_packets;
8093         rx_ring->rx_stats.bytes += total_bytes;
8094         q_vector->rx.total_packets += total_packets;
8095         q_vector->rx.total_bytes += total_bytes;
8096
8097         if (cleaned_count)
8098                 igb_alloc_rx_buffers(rx_ring, cleaned_count);
8099
8100 #ifndef IGB_NO_LRO
8101         igb_lro_flush_all(q_vector);
8102
8103 #endif /* IGB_NO_LRO */
8104         return total_packets < budget;
8105 }
8106 #else /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
8107 /**
8108  * igb_get_headlen - determine size of header for LRO/GRO
8109  * @data: pointer to the start of the headers
8110  * @max_len: total length of section to find headers in
8111  *
8112  * This function is meant to determine the length of headers that will
8113  * be recognized by hardware for LRO, and GRO offloads.  The main
8114  * motivation of doing this is to only perform one pull for IPv4 TCP
8115  * packets so that we can do basic things like calculating the gso_size
8116  * based on the average data per packet.
8117  **/
8118 static unsigned int igb_get_headlen(unsigned char *data,
8119                                     unsigned int max_len)
8120 {
8121         union {
8122                 unsigned char *network;
8123                 /* l2 headers */
8124                 struct ethhdr *eth;
8125                 struct vlan_hdr *vlan;
8126                 /* l3 headers */
8127                 struct iphdr *ipv4;
8128                 struct ipv6hdr *ipv6;
8129         } hdr;
8130         __be16 protocol;
8131         u8 nexthdr = 0; /* default to not TCP */
8132         u8 hlen;
8133
8134         /* this should never happen, but better safe than sorry */
8135         if (max_len < ETH_HLEN)
8136                 return max_len;
8137
8138         /* initialize network frame pointer */
8139         hdr.network = data;
8140
8141         /* set first protocol and move network header forward */
8142         protocol = hdr.eth->h_proto;
8143         hdr.network += ETH_HLEN;
8144
8145         /* handle any vlan tag if present */
8146         if (protocol == __constant_htons(ETH_P_8021Q)) {
8147                 if ((hdr.network - data) > (max_len - VLAN_HLEN))
8148                         return max_len;
8149
8150                 protocol = hdr.vlan->h_vlan_encapsulated_proto;
8151                 hdr.network += VLAN_HLEN;
8152         }
8153
8154         /* handle L3 protocols */
8155         if (protocol == __constant_htons(ETH_P_IP)) {
8156                 if ((hdr.network - data) > (max_len - sizeof(struct iphdr)))
8157                         return max_len;
8158
8159                 /* access ihl as a u8 to avoid unaligned access on ia64 */
8160                 hlen = (hdr.network[0] & 0x0F) << 2;
8161
8162                 /* verify hlen meets minimum size requirements */
8163                 if (hlen < sizeof(struct iphdr))
8164                         return hdr.network - data;
8165
8166                 /* record next protocol if header is present */
8167                 if (!(hdr.ipv4->frag_off & htons(IP_OFFSET)))
8168                         nexthdr = hdr.ipv4->protocol;
8169 #ifdef NETIF_F_TSO6
8170         } else if (protocol == __constant_htons(ETH_P_IPV6)) {
8171                 if ((hdr.network - data) > (max_len - sizeof(struct ipv6hdr)))
8172                         return max_len;
8173
8174                 /* record next protocol */
8175                 nexthdr = hdr.ipv6->nexthdr;
8176                 hlen = sizeof(struct ipv6hdr);
8177 #endif /* NETIF_F_TSO6 */
8178         } else {
8179                 return hdr.network - data;
8180         }
8181
8182         /* relocate pointer to start of L4 header */
8183         hdr.network += hlen;
8184
8185         /* finally sort out TCP */
8186         if (nexthdr == IPPROTO_TCP) {
8187                 if ((hdr.network - data) > (max_len - sizeof(struct tcphdr)))
8188                         return max_len;
8189
8190                 /* access doff as a u8 to avoid unaligned access on ia64 */
8191                 hlen = (hdr.network[12] & 0xF0) >> 2;
8192
8193                 /* verify hlen meets minimum size requirements */
8194                 if (hlen < sizeof(struct tcphdr))
8195                         return hdr.network - data;
8196
8197                 hdr.network += hlen;
8198         } else if (nexthdr == IPPROTO_UDP) {
8199                 if ((hdr.network - data) > (max_len - sizeof(struct udphdr)))
8200                         return max_len;
8201
8202                 hdr.network += sizeof(struct udphdr);
8203         }
8204
8205         /*
8206          * If everything has gone correctly hdr.network should be the
8207          * data section of the packet and will be the end of the header.
8208          * If not then it probably represents the end of the last recognized
8209          * header.
8210          */
8211         if ((hdr.network - data) < max_len)
8212                 return hdr.network - data;
8213         else
8214                 return max_len;
8215 }
8216
8217 /**
8218  * igb_pull_tail - igb specific version of skb_pull_tail
8219  * @rx_ring: rx descriptor ring packet is being transacted on
8220  * @rx_desc: pointer to the EOP Rx descriptor
8221  * @skb: pointer to current skb being adjusted
8222  *
8223  * This function is an igb specific version of __pskb_pull_tail.  The
8224  * main difference between this version and the original function is that
8225  * this function can make several assumptions about the state of things
8226  * that allow for significant optimizations versus the standard function.
8227  * As a result we can do things like drop a frag and maintain an accurate
8228  * truesize for the skb.
8229  */
8230 static void igb_pull_tail(struct igb_ring *rx_ring,
8231                           union e1000_adv_rx_desc *rx_desc,
8232                           struct sk_buff *skb)
8233 {
8234         struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
8235         unsigned char *va;
8236         unsigned int pull_len;
8237
8238         /*
8239          * it is valid to use page_address instead of kmap since we are
8240          * working with pages allocated out of the lomem pool per
8241          * alloc_page(GFP_ATOMIC)
8242          */
8243         va = skb_frag_address(frag);
8244
8245 #ifdef HAVE_PTP_1588_CLOCK
8246         if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
8247                 /* retrieve timestamp from buffer */
8248                 igb_ptp_rx_pktstamp(rx_ring->q_vector, va, skb);
8249
8250                 /* update pointers to remove timestamp header */
8251                 skb_frag_size_sub(frag, IGB_TS_HDR_LEN);
8252                 frag->page_offset += IGB_TS_HDR_LEN;
8253                 skb->data_len -= IGB_TS_HDR_LEN;
8254                 skb->len -= IGB_TS_HDR_LEN;
8255
8256                 /* move va to start of packet data */
8257                 va += IGB_TS_HDR_LEN;
8258         }
8259 #endif /* HAVE_PTP_1588_CLOCK */
8260
8261         /*
8262          * we need the header to contain the greater of either ETH_HLEN or
8263          * 60 bytes if the skb->len is less than 60 for skb_pad.
8264          */
8265         pull_len = igb_get_headlen(va, IGB_RX_HDR_LEN);
8266
8267         /* align pull length to size of long to optimize memcpy performance */
8268         skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));
8269
8270         /* update all of the pointers */
8271         skb_frag_size_sub(frag, pull_len);
8272         frag->page_offset += pull_len;
8273         skb->data_len -= pull_len;
8274         skb->tail += pull_len;
8275 }
8276
8277 /**
8278  * igb_cleanup_headers - Correct corrupted or empty headers
8279  * @rx_ring: rx descriptor ring packet is being transacted on
8280  * @rx_desc: pointer to the EOP Rx descriptor
8281  * @skb: pointer to current skb being fixed
8282  *
8283  * Address the case where we are pulling data in on pages only
8284  * and as such no data is present in the skb header.
8285  *
8286  * In addition if skb is not at least 60 bytes we need to pad it so that
8287  * it is large enough to qualify as a valid Ethernet frame.
8288  *
8289  * Returns true if an error was encountered and skb was freed.
8290  **/
8291 static bool igb_cleanup_headers(struct igb_ring *rx_ring,
8292                                 union e1000_adv_rx_desc *rx_desc,
8293                                 struct sk_buff *skb)
8294 {
8295
8296         if (unlikely((igb_test_staterr(rx_desc,
8297                                        E1000_RXDEXT_ERR_FRAME_ERR_MASK)))) {
8298                 struct net_device *netdev = rx_ring->netdev;
8299                 if (!(netdev->features & NETIF_F_RXALL)) {
8300                         dev_kfree_skb_any(skb);
8301                         return true;
8302                 }
8303         }
8304
8305         /* place header in linear portion of buffer */
8306         if (skb_is_nonlinear(skb))
8307                 igb_pull_tail(rx_ring, rx_desc, skb);
8308
8309         /* if skb_pad returns an error the skb was freed */
8310         if (unlikely(skb->len < 60)) {
8311                 int pad_len = 60 - skb->len;
8312
8313                 if (skb_pad(skb, pad_len))
8314                         return true;
8315                 __skb_put(skb, pad_len);
8316         }
8317
8318         return false;
8319 }
8320
8321 /* igb_clean_rx_irq -- * packet split */
8322 static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
8323 {
8324         struct igb_ring *rx_ring = q_vector->rx.ring;
8325         struct sk_buff *skb = rx_ring->skb;
8326         unsigned int total_bytes = 0, total_packets = 0;
8327         u16 cleaned_count = igb_desc_unused(rx_ring);
8328
8329         do {
8330                 union e1000_adv_rx_desc *rx_desc;
8331
8332                 /* return some buffers to hardware, one at a time is too slow */
8333                 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
8334                         igb_alloc_rx_buffers(rx_ring, cleaned_count);
8335                         cleaned_count = 0;
8336                 }
8337
8338                 rx_desc = IGB_RX_DESC(rx_ring, rx_ring->next_to_clean);
8339
8340                 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_DD))
8341                         break;
8342
8343                 /*
8344                  * This memory barrier is needed to keep us from reading
8345                  * any other fields out of the rx_desc until we know the
8346                  * RXD_STAT_DD bit is set
8347                  */
8348                 rmb();
8349
8350                 /* retrieve a buffer from the ring */
8351                 skb = igb_fetch_rx_buffer(rx_ring, rx_desc, skb);
8352
8353                 /* exit if we failed to retrieve a buffer */
8354                 if (!skb)
8355                         break;
8356
8357                 cleaned_count++;
8358
8359                 /* fetch next buffer in frame if non-eop */
8360                 if (igb_is_non_eop(rx_ring, rx_desc))
8361                         continue;
8362
8363                 /* verify the packet layout is correct */
8364                 if (igb_cleanup_headers(rx_ring, rx_desc, skb)) {
8365                         skb = NULL;
8366                         continue;
8367                 }
8368
8369                 /* probably a little skewed due to removing CRC */
8370                 total_bytes += skb->len;
8371
8372                 /* populate checksum, timestamp, VLAN, and protocol */
8373                 igb_process_skb_fields(rx_ring, rx_desc, skb);
8374
8375 #ifndef IGB_NO_LRO
8376                 if (igb_can_lro(rx_ring, rx_desc, skb))
8377                         igb_lro_receive(q_vector, skb);
8378                 else
8379 #endif
8380 #ifdef HAVE_VLAN_RX_REGISTER
8381                         igb_receive_skb(q_vector, skb);
8382 #else
8383                         napi_gro_receive(&q_vector->napi, skb);
8384 #endif
8385 #ifndef NETIF_F_GRO
8386
8387                 netdev_ring(rx_ring)->last_rx = jiffies;
8388 #endif
8389
8390                 /* reset skb pointer */
8391                 skb = NULL;
8392
8393                 /* update budget accounting */
8394                 total_packets++;
8395         } while (likely(total_packets < budget));
8396
8397         /* place incomplete frames back on ring for completion */
8398         rx_ring->skb = skb;
8399
8400         rx_ring->rx_stats.packets += total_packets;
8401         rx_ring->rx_stats.bytes += total_bytes;
8402         q_vector->rx.total_packets += total_packets;
8403         q_vector->rx.total_bytes += total_bytes;
8404
8405         if (cleaned_count)
8406                 igb_alloc_rx_buffers(rx_ring, cleaned_count);
8407
8408 #ifndef IGB_NO_LRO
8409         igb_lro_flush_all(q_vector);
8410
8411 #endif /* IGB_NO_LRO */
8412         return total_packets < budget;
8413 }
8414 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
8415
8416 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
8417 static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
8418                                  struct igb_rx_buffer *bi)
8419 {
8420         struct sk_buff *skb = bi->skb;
8421         dma_addr_t dma = bi->dma;
8422
8423         if (dma)
8424                 return true;
8425
8426         if (likely(!skb)) {
8427                 skb = netdev_alloc_skb_ip_align(netdev_ring(rx_ring),
8428                                                 rx_ring->rx_buffer_len);
8429                 bi->skb = skb;
8430                 if (!skb) {
8431                         rx_ring->rx_stats.alloc_failed++;
8432                         return false;
8433                 }
8434
8435                 /* initialize skb for ring */
8436                 skb_record_rx_queue(skb, ring_queue_index(rx_ring));
8437         }
8438
8439         dma = dma_map_single(rx_ring->dev, skb->data,
8440                              rx_ring->rx_buffer_len, DMA_FROM_DEVICE);
8441
8442         /* if mapping failed free memory back to system since
8443          * there isn't much point in holding memory we can't use
8444          */
8445         if (dma_mapping_error(rx_ring->dev, dma)) {
8446                 dev_kfree_skb_any(skb);
8447                 bi->skb = NULL;
8448
8449                 rx_ring->rx_stats.alloc_failed++;
8450                 return false;
8451         }
8452
8453         bi->dma = dma;
8454         return true;
8455 }
8456
8457 #else /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
8458 static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
8459                                   struct igb_rx_buffer *bi)
8460 {
8461         struct page *page = bi->page;
8462         dma_addr_t dma;
8463
8464         /* since we are recycling buffers we should seldom need to alloc */
8465         if (likely(page))
8466                 return true;
8467
8468         /* alloc new page for storage */
8469         page = alloc_page(GFP_ATOMIC | __GFP_COLD);
8470         if (unlikely(!page)) {
8471                 rx_ring->rx_stats.alloc_failed++;
8472                 return false;
8473         }
8474
8475         /* map page for use */
8476         dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
8477
8478         /*
8479          * if mapping failed free memory back to system since
8480          * there isn't much point in holding memory we can't use
8481          */
8482         if (dma_mapping_error(rx_ring->dev, dma)) {
8483                 __free_page(page);
8484
8485                 rx_ring->rx_stats.alloc_failed++;
8486                 return false;
8487         }
8488
8489         bi->dma = dma;
8490         bi->page = page;
8491         bi->page_offset = 0;
8492
8493         return true;
8494 }
8495
8496 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
8497 /**
8498  * igb_alloc_rx_buffers - Replace used receive buffers; packet split
8499  * @adapter: address of board private structure
8500  **/
8501 void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
8502 {
8503         union e1000_adv_rx_desc *rx_desc;
8504         struct igb_rx_buffer *bi;
8505         u16 i = rx_ring->next_to_use;
8506
8507         /* nothing to do */
8508         if (!cleaned_count)
8509                 return;
8510
8511         rx_desc = IGB_RX_DESC(rx_ring, i);
8512         bi = &rx_ring->rx_buffer_info[i];
8513         i -= rx_ring->count;
8514
8515         do {
8516 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
8517                 if (!igb_alloc_mapped_skb(rx_ring, bi))
8518 #else
8519                 if (!igb_alloc_mapped_page(rx_ring, bi))
8520 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
8521                         break;
8522
8523                 /*
8524                  * Refresh the desc even if buffer_addrs didn't change
8525                  * because each write-back erases this info.
8526                  */
8527 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
8528                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
8529 #else
8530                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
8531 #endif
8532
8533                 rx_desc++;
8534                 bi++;
8535                 i++;
8536                 if (unlikely(!i)) {
8537                         rx_desc = IGB_RX_DESC(rx_ring, 0);
8538                         bi = rx_ring->rx_buffer_info;
8539                         i -= rx_ring->count;
8540                 }
8541
8542                 /* clear the hdr_addr for the next_to_use descriptor */
8543                 rx_desc->read.hdr_addr = 0;
8544
8545                 cleaned_count--;
8546         } while (cleaned_count);
8547
8548         i += rx_ring->count;
8549
8550         if (rx_ring->next_to_use != i) {
8551                 /* record the next descriptor to use */
8552                 rx_ring->next_to_use = i;
8553
8554 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
8555                 /* update next to alloc since we have filled the ring */
8556                 rx_ring->next_to_alloc = i;
8557
8558 #endif
8559                 /*
8560                  * Force memory writes to complete before letting h/w
8561                  * know there are new descriptors to fetch.  (Only
8562                  * applicable for weak-ordered memory model archs,
8563                  * such as IA-64).
8564                  */
8565                 wmb();
8566                 writel(i, rx_ring->tail);
8567         }
8568 }
8569
8570 #ifdef SIOCGMIIPHY
8571 /**
8572  * igb_mii_ioctl -
8573  * @netdev:
8574  * @ifreq:
8575  * @cmd:
8576  **/
8577 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
8578 {
8579         struct igb_adapter *adapter = netdev_priv(netdev);
8580         struct mii_ioctl_data *data = if_mii(ifr);
8581
8582         if (adapter->hw.phy.media_type != e1000_media_type_copper)
8583                 return -EOPNOTSUPP;
8584
8585         switch (cmd) {
8586         case SIOCGMIIPHY:
8587                 data->phy_id = adapter->hw.phy.addr;
8588                 break;
8589         case SIOCGMIIREG:
8590                 if (!capable(CAP_NET_ADMIN))
8591                         return -EPERM;
8592                 if (e1000_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
8593                                    &data->val_out))
8594                         return -EIO;
8595                 break;
8596         case SIOCSMIIREG:
8597         default:
8598                 return -EOPNOTSUPP;
8599         }
8600         return E1000_SUCCESS;
8601 }
8602
8603 #endif
8604 /**
8605  * igb_ioctl -
8606  * @netdev:
8607  * @ifreq:
8608  * @cmd:
8609  **/
8610 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
8611 {
8612         switch (cmd) {
8613 #ifdef SIOCGMIIPHY
8614         case SIOCGMIIPHY:
8615         case SIOCGMIIREG:
8616         case SIOCSMIIREG:
8617                 return igb_mii_ioctl(netdev, ifr, cmd);
8618 #endif
8619 #ifdef HAVE_PTP_1588_CLOCK
8620         case SIOCSHWTSTAMP:
8621                 return igb_ptp_hwtstamp_ioctl(netdev, ifr, cmd);
8622 #endif /* HAVE_PTP_1588_CLOCK */
8623 #ifdef ETHTOOL_OPS_COMPAT
8624         case SIOCETHTOOL:
8625                 return ethtool_ioctl(ifr);
8626 #endif
8627         default:
8628                 return -EOPNOTSUPP;
8629         }
8630 }
8631
8632 s32 e1000_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
8633 {
8634         struct igb_adapter *adapter = hw->back;
8635         u16 cap_offset;
8636
8637         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
8638         if (!cap_offset)
8639                 return -E1000_ERR_CONFIG;
8640
8641         pci_read_config_word(adapter->pdev, cap_offset + reg, value);
8642
8643         return E1000_SUCCESS;
8644 }
8645
8646 s32 e1000_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
8647 {
8648         struct igb_adapter *adapter = hw->back;
8649         u16 cap_offset;
8650
8651         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
8652         if (!cap_offset)
8653                 return -E1000_ERR_CONFIG;
8654
8655         pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
8656
8657         return E1000_SUCCESS;
8658 }
8659
8660 #ifdef HAVE_VLAN_RX_REGISTER
8661 static void igb_vlan_mode(struct net_device *netdev, struct vlan_group *vlgrp)
8662 #else
8663 void igb_vlan_mode(struct net_device *netdev, u32 features)
8664 #endif
8665 {
8666         struct igb_adapter *adapter = netdev_priv(netdev);
8667         struct e1000_hw *hw = &adapter->hw;
8668         u32 ctrl, rctl;
8669         int i;
8670 #ifdef HAVE_VLAN_RX_REGISTER
8671         bool enable = !!vlgrp;
8672
8673         igb_irq_disable(adapter);
8674
8675         adapter->vlgrp = vlgrp;
8676
8677         if (!test_bit(__IGB_DOWN, &adapter->state))
8678                 igb_irq_enable(adapter);
8679 #else
8680 #ifdef NETIF_F_HW_VLAN_CTAG_RX
8681         bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
8682 #else
8683         bool enable = !!(features & NETIF_F_HW_VLAN_RX);
8684 #endif
8685 #endif
8686
8687         if (enable) {
8688                 /* enable VLAN tag insert/strip */
8689                 ctrl = E1000_READ_REG(hw, E1000_CTRL);
8690                 ctrl |= E1000_CTRL_VME;
8691                 E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
8692
8693                 /* Disable CFI check */
8694                 rctl = E1000_READ_REG(hw, E1000_RCTL);
8695                 rctl &= ~E1000_RCTL_CFIEN;
8696                 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
8697         } else {
8698                 /* disable VLAN tag insert/strip */
8699                 ctrl = E1000_READ_REG(hw, E1000_CTRL);
8700                 ctrl &= ~E1000_CTRL_VME;
8701                 E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
8702         }
8703
8704 #ifndef CONFIG_IGB_VMDQ_NETDEV
8705         for (i = 0; i < adapter->vmdq_pools; i++) {
8706                 igb_set_vf_vlan_strip(adapter,
8707                                       adapter->vfs_allocated_count + i,
8708                                       enable);
8709         }
8710
8711 #else
8712         igb_set_vf_vlan_strip(adapter,
8713                               adapter->vfs_allocated_count,
8714                               enable);
8715
8716         for (i = 1; i < adapter->vmdq_pools; i++) {
8717 #ifdef HAVE_VLAN_RX_REGISTER
8718                 struct igb_vmdq_adapter *vadapter;
8719                 vadapter = netdev_priv(adapter->vmdq_netdev[i-1]);
8720                 enable = !!vadapter->vlgrp;
8721 #else
8722                 struct net_device *vnetdev;
8723                 vnetdev = adapter->vmdq_netdev[i-1];
8724 #ifdef NETIF_F_HW_VLAN_CTAG_RX
8725                 enable = !!(vnetdev->features & NETIF_F_HW_VLAN_CTAG_RX);
8726 #else
8727                 enable = !!(vnetdev->features & NETIF_F_HW_VLAN_RX);
8728 #endif
8729 #endif
8730                 igb_set_vf_vlan_strip(adapter,
8731                                       adapter->vfs_allocated_count + i,
8732                                       enable);
8733         }
8734
8735 #endif
8736         igb_rlpml_set(adapter);
8737 }
8738
8739 #ifdef HAVE_VLAN_PROTOCOL
8740 static int igb_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
8741 #elif defined HAVE_INT_NDO_VLAN_RX_ADD_VID
8742 #ifdef NETIF_F_HW_VLAN_CTAG_RX
8743 static int igb_vlan_rx_add_vid(struct net_device *netdev,
8744                                __always_unused __be16 proto, u16 vid)
8745 #else
8746 static int igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
8747 #endif
8748 #else
8749 static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
8750 #endif
8751 {
8752         struct igb_adapter *adapter = netdev_priv(netdev);
8753         int pf_id = adapter->vfs_allocated_count;
8754
8755         /* attempt to add filter to vlvf array */
8756         igb_vlvf_set(adapter, vid, TRUE, pf_id);
8757
8758         /* add the filter since PF can receive vlans w/o entry in vlvf */
8759         igb_vfta_set(adapter, vid, TRUE);
8760 #ifndef HAVE_NETDEV_VLAN_FEATURES
8761
8762         /* Copy feature flags from netdev to the vlan netdev for this vid.
8763          * This allows things like TSO to bubble down to our vlan device.
8764          * There is no need to update netdev for vlan 0 (DCB), since it
8765          * wouldn't has v_netdev.
8766          */
8767         if (adapter->vlgrp) {
8768                 struct vlan_group *vlgrp = adapter->vlgrp;
8769                 struct net_device *v_netdev = vlan_group_get_device(vlgrp, vid);
8770                 if (v_netdev) {
8771                         v_netdev->features |= netdev->features;
8772                         vlan_group_set_device(vlgrp, vid, v_netdev);
8773                 }
8774         }
8775 #endif
8776 #ifndef HAVE_VLAN_RX_REGISTER
8777
8778         set_bit(vid, adapter->active_vlans);
8779 #endif
8780 #ifdef HAVE_INT_NDO_VLAN_RX_ADD_VID
8781         return 0;
8782 #endif
8783 }
8784
8785 #ifdef HAVE_VLAN_PROTOCOL
8786 static int igb_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
8787 #elif defined HAVE_INT_NDO_VLAN_RX_ADD_VID
8788 #ifdef NETIF_F_HW_VLAN_CTAG_RX
8789 static int igb_vlan_rx_kill_vid(struct net_device *netdev,
8790                                 __always_unused __be16 proto, u16 vid)
8791 #else
8792 static int igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
8793 #endif
8794 #else
8795 static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
8796 #endif
8797 {
8798         struct igb_adapter *adapter = netdev_priv(netdev);
8799         int pf_id = adapter->vfs_allocated_count;
8800         s32 err;
8801
8802 #ifdef HAVE_VLAN_RX_REGISTER
8803         igb_irq_disable(adapter);
8804
8805         vlan_group_set_device(adapter->vlgrp, vid, NULL);
8806
8807         if (!test_bit(__IGB_DOWN, &adapter->state))
8808                 igb_irq_enable(adapter);
8809
8810 #endif /* HAVE_VLAN_RX_REGISTER */
8811         /* remove vlan from VLVF table array */
8812         err = igb_vlvf_set(adapter, vid, FALSE, pf_id);
8813
8814         /* if vid was not present in VLVF just remove it from table */
8815         if (err)
8816                 igb_vfta_set(adapter, vid, FALSE);
8817 #ifndef HAVE_VLAN_RX_REGISTER
8818
8819         clear_bit(vid, adapter->active_vlans);
8820 #endif
8821 #ifdef HAVE_INT_NDO_VLAN_RX_ADD_VID
8822         return 0;
8823 #endif
8824 }
8825
8826 static void igb_restore_vlan(struct igb_adapter *adapter)
8827 {
8828 #ifdef HAVE_VLAN_RX_REGISTER
8829         igb_vlan_mode(adapter->netdev, adapter->vlgrp);
8830
8831         if (adapter->vlgrp) {
8832                 u16 vid;
8833                 for (vid = 0; vid < VLAN_N_VID; vid++) {
8834                         if (!vlan_group_get_device(adapter->vlgrp, vid))
8835                                 continue;
8836 #ifdef NETIF_F_HW_VLAN_CTAG_RX
8837                         igb_vlan_rx_add_vid(adapter->netdev,
8838                                             htons(ETH_P_8021Q), vid);
8839 #else
8840                         igb_vlan_rx_add_vid(adapter->netdev, vid);
8841 #endif
8842                 }
8843         }
8844 #else
8845         u16 vid;
8846
8847         igb_vlan_mode(adapter->netdev, adapter->netdev->features);
8848
8849         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
8850 #ifdef NETIF_F_HW_VLAN_CTAG_RX
8851                 igb_vlan_rx_add_vid(adapter->netdev,
8852                                     htons(ETH_P_8021Q), vid);
8853 #else
8854                 igb_vlan_rx_add_vid(adapter->netdev, vid);
8855 #endif
8856 #endif
8857 }
8858
8859 int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
8860 {
8861         struct pci_dev *pdev = adapter->pdev;
8862         struct e1000_mac_info *mac = &adapter->hw.mac;
8863
8864         mac->autoneg = 0;
8865
8866         /* SerDes device's does not support 10Mbps Full/duplex
8867          * and 100Mbps Half duplex
8868          */
8869         if (adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
8870                 switch (spddplx) {
8871                 case SPEED_10 + DUPLEX_HALF:
8872                 case SPEED_10 + DUPLEX_FULL:
8873                 case SPEED_100 + DUPLEX_HALF:
8874                         dev_err(pci_dev_to_dev(pdev),
8875                                 "Unsupported Speed/Duplex configuration\n");
8876                         return -EINVAL;
8877                 default:
8878                         break;
8879                 }
8880         }
8881
8882         switch (spddplx) {
8883         case SPEED_10 + DUPLEX_HALF:
8884                 mac->forced_speed_duplex = ADVERTISE_10_HALF;
8885                 break;
8886         case SPEED_10 + DUPLEX_FULL:
8887                 mac->forced_speed_duplex = ADVERTISE_10_FULL;
8888                 break;
8889         case SPEED_100 + DUPLEX_HALF:
8890                 mac->forced_speed_duplex = ADVERTISE_100_HALF;
8891                 break;
8892         case SPEED_100 + DUPLEX_FULL:
8893                 mac->forced_speed_duplex = ADVERTISE_100_FULL;
8894                 break;
8895         case SPEED_1000 + DUPLEX_FULL:
8896                 mac->autoneg = 1;
8897                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
8898                 break;
8899         case SPEED_1000 + DUPLEX_HALF: /* not supported */
8900         default:
8901                 dev_err(pci_dev_to_dev(pdev), "Unsupported Speed/Duplex configuration\n");
8902                 return -EINVAL;
8903         }
8904
8905         /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
8906         adapter->hw.phy.mdix = AUTO_ALL_MODES;
8907
8908         return 0;
8909 }
8910
8911 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
8912                           bool runtime)
8913 {
8914         struct net_device *netdev = pci_get_drvdata(pdev);
8915         struct igb_adapter *adapter = netdev_priv(netdev);
8916         struct e1000_hw *hw = &adapter->hw;
8917         u32 ctrl, rctl, status;
8918         u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
8919 #ifdef CONFIG_PM
8920         int retval = 0;
8921 #endif
8922
8923         netif_device_detach(netdev);
8924
8925         status = E1000_READ_REG(hw, E1000_STATUS);
8926         if (status & E1000_STATUS_LU)
8927                 wufc &= ~E1000_WUFC_LNKC;
8928
8929         if (netif_running(netdev))
8930                 __igb_close(netdev, true);
8931
8932         igb_clear_interrupt_scheme(adapter);
8933
8934 #ifdef CONFIG_PM
8935         retval = pci_save_state(pdev);
8936         if (retval)
8937                 return retval;
8938 #endif
8939
8940         if (wufc) {
8941                 igb_setup_rctl(adapter);
8942                 igb_set_rx_mode(netdev);
8943
8944                 /* turn on all-multi mode if wake on multicast is enabled */
8945                 if (wufc & E1000_WUFC_MC) {
8946                         rctl = E1000_READ_REG(hw, E1000_RCTL);
8947                         rctl |= E1000_RCTL_MPE;
8948                         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
8949                 }
8950
8951                 ctrl = E1000_READ_REG(hw, E1000_CTRL);
8952                 /* phy power management enable */
8953                 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
8954                 ctrl |= E1000_CTRL_ADVD3WUC;
8955                 E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
8956
8957                 /* Allow time for pending master requests to run */
8958                 e1000_disable_pcie_master(hw);
8959
8960                 E1000_WRITE_REG(hw, E1000_WUC, E1000_WUC_PME_EN);
8961                 E1000_WRITE_REG(hw, E1000_WUFC, wufc);
8962         } else {
8963                 E1000_WRITE_REG(hw, E1000_WUC, 0);
8964                 E1000_WRITE_REG(hw, E1000_WUFC, 0);
8965         }
8966
8967         *enable_wake = wufc || adapter->en_mng_pt;
8968         if (!*enable_wake)
8969                 igb_power_down_link(adapter);
8970         else
8971                 igb_power_up_link(adapter);
8972
8973         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
8974          * would have already happened in close and is redundant. */
8975         igb_release_hw_control(adapter);
8976
8977         pci_disable_device(pdev);
8978
8979         return 0;
8980 }
8981
8982 #ifdef CONFIG_PM
8983 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
8984 static int igb_suspend(struct device *dev)
8985 #else
8986 static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
8987 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
8988 {
8989 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
8990         struct pci_dev *pdev = to_pci_dev(dev);
8991 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
8992         int retval;
8993         bool wake;
8994
8995         retval = __igb_shutdown(pdev, &wake, 0);
8996         if (retval)
8997                 return retval;
8998
8999         if (wake) {
9000                 pci_prepare_to_sleep(pdev);
9001         } else {
9002                 pci_wake_from_d3(pdev, false);
9003                 pci_set_power_state(pdev, PCI_D3hot);
9004         }
9005
9006         return 0;
9007 }
9008
9009 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
9010 static int igb_resume(struct device *dev)
9011 #else
9012 static int igb_resume(struct pci_dev *pdev)
9013 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
9014 {
9015 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
9016         struct pci_dev *pdev = to_pci_dev(dev);
9017 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
9018         struct net_device *netdev = pci_get_drvdata(pdev);
9019         struct igb_adapter *adapter = netdev_priv(netdev);
9020         struct e1000_hw *hw = &adapter->hw;
9021         u32 err;
9022
9023         pci_set_power_state(pdev, PCI_D0);
9024         pci_restore_state(pdev);
9025         pci_save_state(pdev);
9026
9027         err = pci_enable_device_mem(pdev);
9028         if (err) {
9029                 dev_err(pci_dev_to_dev(pdev),
9030                         "igb: Cannot enable PCI device from suspend\n");
9031                 return err;
9032         }
9033         pci_set_master(pdev);
9034
9035         pci_enable_wake(pdev, PCI_D3hot, 0);
9036         pci_enable_wake(pdev, PCI_D3cold, 0);
9037
9038         if (igb_init_interrupt_scheme(adapter, true)) {
9039                 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for queues\n");
9040                 return -ENOMEM;
9041         }
9042
9043         igb_reset(adapter);
9044
9045         /* let the f/w know that the h/w is now under the control of the
9046          * driver. */
9047         igb_get_hw_control(adapter);
9048
9049         E1000_WRITE_REG(hw, E1000_WUS, ~0);
9050
9051         if (netdev->flags & IFF_UP) {
9052                 rtnl_lock();
9053                 err = __igb_open(netdev, true);
9054                 rtnl_unlock();
9055                 if (err)
9056                         return err;
9057         }
9058
9059         netif_device_attach(netdev);
9060
9061         return 0;
9062 }
9063
9064 #ifdef CONFIG_PM_RUNTIME
9065 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
9066 static int igb_runtime_idle(struct device *dev)
9067 {
9068         struct pci_dev *pdev = to_pci_dev(dev);
9069         struct net_device *netdev = pci_get_drvdata(pdev);
9070         struct igb_adapter *adapter = netdev_priv(netdev);
9071
9072         if (!igb_has_link(adapter))
9073                 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
9074
9075         return -EBUSY;
9076 }
9077
9078 static int igb_runtime_suspend(struct device *dev)
9079 {
9080         struct pci_dev *pdev = to_pci_dev(dev);
9081         int retval;
9082         bool wake;
9083
9084         retval = __igb_shutdown(pdev, &wake, 1);
9085         if (retval)
9086                 return retval;
9087
9088         if (wake) {
9089                 pci_prepare_to_sleep(pdev);
9090         } else {
9091                 pci_wake_from_d3(pdev, false);
9092                 pci_set_power_state(pdev, PCI_D3hot);
9093         }
9094
9095         return 0;
9096 }
9097
9098 static int igb_runtime_resume(struct device *dev)
9099 {
9100         return igb_resume(dev);
9101 }
9102 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
9103 #endif /* CONFIG_PM_RUNTIME */
9104 #endif /* CONFIG_PM */
9105
9106 #ifdef USE_REBOOT_NOTIFIER
9107 /* only want to do this for 2.4 kernels? */
9108 static int igb_notify_reboot(struct notifier_block *nb, unsigned long event,
9109                              void *p)
9110 {
9111         struct pci_dev *pdev = NULL;
9112         bool wake;
9113
9114         switch (event) {
9115         case SYS_DOWN:
9116         case SYS_HALT:
9117         case SYS_POWER_OFF:
9118                 while ((pdev = pci_find_device(PCI_ANY_ID, PCI_ANY_ID, pdev))) {
9119                         if (pci_dev_driver(pdev) == &igb_driver) {
9120                                 __igb_shutdown(pdev, &wake, 0);
9121                                 if (event == SYS_POWER_OFF) {
9122                                         pci_wake_from_d3(pdev, wake);
9123                                         pci_set_power_state(pdev, PCI_D3hot);
9124                                 }
9125                         }
9126                 }
9127         }
9128         return NOTIFY_DONE;
9129 }
9130 #else
9131 static void igb_shutdown(struct pci_dev *pdev)
9132 {
9133         bool wake = false;
9134
9135         __igb_shutdown(pdev, &wake, 0);
9136
9137         if (system_state == SYSTEM_POWER_OFF) {
9138                 pci_wake_from_d3(pdev, wake);
9139                 pci_set_power_state(pdev, PCI_D3hot);
9140         }
9141 }
9142 #endif /* USE_REBOOT_NOTIFIER */
9143
9144 #ifdef CONFIG_NET_POLL_CONTROLLER
9145 /*
9146  * Polling 'interrupt' - used by things like netconsole to send skbs
9147  * without having to re-enable interrupts. It's not called while
9148  * the interrupt routine is executing.
9149  */
9150 static void igb_netpoll(struct net_device *netdev)
9151 {
9152         struct igb_adapter *adapter = netdev_priv(netdev);
9153         struct e1000_hw *hw = &adapter->hw;
9154         struct igb_q_vector *q_vector;
9155         int i;
9156
9157         for (i = 0; i < adapter->num_q_vectors; i++) {
9158                 q_vector = adapter->q_vector[i];
9159                 if (adapter->msix_entries)
9160                         E1000_WRITE_REG(hw, E1000_EIMC, q_vector->eims_value);
9161                 else
9162                         igb_irq_disable(adapter);
9163                 napi_schedule(&q_vector->napi);
9164         }
9165 }
9166 #endif /* CONFIG_NET_POLL_CONTROLLER */
9167
9168 #ifdef HAVE_PCI_ERS
9169 #define E1000_DEV_ID_82576_VF 0x10CA
9170 /**
9171  * igb_io_error_detected - called when PCI error is detected
9172  * @pdev: Pointer to PCI device
9173  * @state: The current pci connection state
9174  *
9175  * This function is called after a PCI bus error affecting
9176  * this device has been detected.
9177  */
9178 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
9179                                               pci_channel_state_t state)
9180 {
9181         struct net_device *netdev = pci_get_drvdata(pdev);
9182         struct igb_adapter *adapter = netdev_priv(netdev);
9183
9184 #ifdef CONFIG_PCI_IOV__UNUSED
9185         struct pci_dev *bdev, *vfdev;
9186         u32 dw0, dw1, dw2, dw3;
9187         int vf, pos;
9188         u16 req_id, pf_func;
9189
9190         if (!(adapter->flags & IGB_FLAG_DETECT_BAD_DMA))
9191                 goto skip_bad_vf_detection;
9192
9193         bdev = pdev->bus->self;
9194         while (bdev && (pci_pcie_type(bdev) != PCI_EXP_TYPE_ROOT_PORT))
9195                 bdev = bdev->bus->self;
9196
9197         if (!bdev)
9198                 goto skip_bad_vf_detection;
9199
9200         pos = pci_find_ext_capability(bdev, PCI_EXT_CAP_ID_ERR);
9201         if (!pos)
9202                 goto skip_bad_vf_detection;
9203
9204         pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG, &dw0);
9205         pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 4, &dw1);
9206         pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 8, &dw2);
9207         pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 12, &dw3);
9208
9209         req_id = dw1 >> 16;
9210         /* On the 82576 if bit 7 of the requestor ID is set then it's a VF */
9211         if (!(req_id & 0x0080))
9212                 goto skip_bad_vf_detection;
9213
9214         pf_func = req_id & 0x01;
9215         if ((pf_func & 1) == (pdev->devfn & 1)) {
9216
9217                 vf = (req_id & 0x7F) >> 1;
9218                 dev_err(pci_dev_to_dev(pdev),
9219                         "VF %d has caused a PCIe error\n", vf);
9220                 dev_err(pci_dev_to_dev(pdev),
9221                         "TLP: dw0: %8.8x\tdw1: %8.8x\tdw2: "
9222                         "%8.8x\tdw3: %8.8x\n",
9223                         dw0, dw1, dw2, dw3);
9224
9225                 /* Find the pci device of the offending VF */
9226                 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL,
9227                                        E1000_DEV_ID_82576_VF, NULL);
9228                 while (vfdev) {
9229                         if (vfdev->devfn == (req_id & 0xFF))
9230                                 break;
9231                         vfdev = pci_get_device(PCI_VENDOR_ID_INTEL,
9232                                                E1000_DEV_ID_82576_VF, vfdev);
9233                 }
9234                 /*
9235                  * There's a slim chance the VF could have been hot plugged,
9236                  * so if it is no longer present we don't need to issue the
9237                  * VFLR.  Just clean up the AER in that case.
9238                  */
9239                 if (vfdev) {
9240                         dev_err(pci_dev_to_dev(pdev),
9241                                 "Issuing VFLR to VF %d\n", vf);
9242                         pci_write_config_dword(vfdev, 0xA8, 0x00008000);
9243                 }
9244
9245                 pci_cleanup_aer_uncorrect_error_status(pdev);
9246         }
9247
9248         /*
9249          * Even though the error may have occurred on the other port
9250          * we still need to increment the vf error reference count for
9251          * both ports because the I/O resume function will be called
9252          * for both of them.
9253          */
9254         adapter->vferr_refcount++;
9255
9256         return PCI_ERS_RESULT_RECOVERED;
9257
9258 skip_bad_vf_detection:
9259 #endif /* CONFIG_PCI_IOV */
9260
9261         netif_device_detach(netdev);
9262
9263         if (state == pci_channel_io_perm_failure)
9264                 return PCI_ERS_RESULT_DISCONNECT;
9265
9266         if (netif_running(netdev))
9267                 igb_down(adapter);
9268         pci_disable_device(pdev);
9269
9270         /* Request a slot slot reset. */
9271         return PCI_ERS_RESULT_NEED_RESET;
9272 }
9273
9274 /**
9275  * igb_io_slot_reset - called after the pci bus has been reset.
9276  * @pdev: Pointer to PCI device
9277  *
9278  * Restart the card from scratch, as if from a cold-boot. Implementation
9279  * resembles the first-half of the igb_resume routine.
9280  */
9281 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
9282 {
9283         struct net_device *netdev = pci_get_drvdata(pdev);
9284         struct igb_adapter *adapter = netdev_priv(netdev);
9285         struct e1000_hw *hw = &adapter->hw;
9286         pci_ers_result_t result;
9287
9288         if (pci_enable_device_mem(pdev)) {
9289                 dev_err(pci_dev_to_dev(pdev),
9290                         "Cannot re-enable PCI device after reset.\n");
9291                 result = PCI_ERS_RESULT_DISCONNECT;
9292         } else {
9293                 pci_set_master(pdev);
9294                 pci_restore_state(pdev);
9295                 pci_save_state(pdev);
9296
9297                 pci_enable_wake(pdev, PCI_D3hot, 0);
9298                 pci_enable_wake(pdev, PCI_D3cold, 0);
9299
9300                 schedule_work(&adapter->reset_task);
9301                 E1000_WRITE_REG(hw, E1000_WUS, ~0);
9302                 result = PCI_ERS_RESULT_RECOVERED;
9303         }
9304
9305         pci_cleanup_aer_uncorrect_error_status(pdev);
9306
9307         return result;
9308 }
9309
9310 /**
9311  * igb_io_resume - called when traffic can start flowing again.
9312  * @pdev: Pointer to PCI device
9313  *
9314  * This callback is called when the error recovery driver tells us that
9315  * its OK to resume normal operation. Implementation resembles the
9316  * second-half of the igb_resume routine.
9317  */
9318 static void igb_io_resume(struct pci_dev *pdev)
9319 {
9320         struct net_device *netdev = pci_get_drvdata(pdev);
9321         struct igb_adapter *adapter = netdev_priv(netdev);
9322
9323         if (adapter->vferr_refcount) {
9324                 dev_info(pci_dev_to_dev(pdev), "Resuming after VF err\n");
9325                 adapter->vferr_refcount--;
9326                 return;
9327         }
9328
9329         if (netif_running(netdev)) {
9330                 if (igb_up(adapter)) {
9331                         dev_err(pci_dev_to_dev(pdev), "igb_up failed after reset\n");
9332                         return;
9333                 }
9334         }
9335
9336         netif_device_attach(netdev);
9337
9338         /* let the f/w know that the h/w is now under the control of the
9339          * driver. */
9340         igb_get_hw_control(adapter);
9341 }
9342
9343 #endif /* HAVE_PCI_ERS */
9344
9345 int igb_add_mac_filter(struct igb_adapter *adapter, u8 *addr, u16 queue)
9346 {
9347         struct e1000_hw *hw = &adapter->hw;
9348         int i;
9349
9350         if (is_zero_ether_addr(addr))
9351                 return 0;
9352
9353         for (i = 0; i < hw->mac.rar_entry_count; i++) {
9354                 if (adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE)
9355                         continue;
9356                 adapter->mac_table[i].state = (IGB_MAC_STATE_MODIFIED |
9357                                                    IGB_MAC_STATE_IN_USE);
9358                 memcpy(adapter->mac_table[i].addr, addr, ETH_ALEN);
9359                 adapter->mac_table[i].queue = queue;
9360                 igb_sync_mac_table(adapter);
9361                 return 0;
9362         }
9363         return -ENOMEM;
9364 }
9365 int igb_del_mac_filter(struct igb_adapter *adapter, u8* addr, u16 queue)
9366 {
9367         /* search table for addr, if found, set to 0 and sync */
9368         int i;
9369         struct e1000_hw *hw = &adapter->hw;
9370
9371         if (is_zero_ether_addr(addr))
9372                 return 0;
9373         for (i = 0; i < hw->mac.rar_entry_count; i++) {
9374                 if (ether_addr_equal(addr, adapter->mac_table[i].addr) &&
9375                     adapter->mac_table[i].queue == queue) {
9376                         adapter->mac_table[i].state = IGB_MAC_STATE_MODIFIED;
9377                         memset(adapter->mac_table[i].addr, 0, ETH_ALEN);
9378                         adapter->mac_table[i].queue = 0;
9379                         igb_sync_mac_table(adapter);
9380                         return 0;
9381                 }
9382         }
9383         return -ENOMEM;
9384 }
9385 static int igb_set_vf_mac(struct igb_adapter *adapter,
9386                           int vf, unsigned char *mac_addr)
9387 {
9388         igb_del_mac_filter(adapter, adapter->vf_data[vf].vf_mac_addresses, vf);
9389         memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
9390
9391         igb_add_mac_filter(adapter, mac_addr, vf);
9392
9393         return 0;
9394 }
9395
9396 #ifdef IFLA_VF_MAX
9397 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
9398 {
9399         struct igb_adapter *adapter = netdev_priv(netdev);
9400         if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
9401                 return -EINVAL;
9402         adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
9403         dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
9404         dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
9405                                       " change effective.\n");
9406         if (test_bit(__IGB_DOWN, &adapter->state)) {
9407                 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
9408                          " but the PF device is not up.\n");
9409                 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
9410                          " attempting to use the VF device.\n");
9411         }
9412         return igb_set_vf_mac(adapter, vf, mac);
9413 }
9414
9415 static int igb_link_mbps(int internal_link_speed)
9416 {
9417         switch (internal_link_speed) {
9418         case SPEED_100:
9419                 return 100;
9420         case SPEED_1000:
9421                 return 1000;
9422         case SPEED_2500:
9423                 return 2500;
9424         default:
9425                 return 0;
9426         }
9427 }
9428
9429 static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
9430                         int link_speed)
9431 {
9432         int rf_dec, rf_int;
9433         u32 bcnrc_val;
9434
9435         if (tx_rate != 0) {
9436                 /* Calculate the rate factor values to set */
9437                 rf_int = link_speed / tx_rate;
9438                 rf_dec = (link_speed - (rf_int * tx_rate));
9439                 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
9440
9441                 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
9442                 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
9443                                 E1000_RTTBCNRC_RF_INT_MASK);
9444                 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
9445         } else {
9446                 bcnrc_val = 0;
9447         }
9448
9449         E1000_WRITE_REG(hw, E1000_RTTDQSEL, vf); /* vf X uses queue X */
9450         /*
9451          * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
9452          * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
9453          */
9454         E1000_WRITE_REG(hw, E1000_RTTBCNRM(0), 0x14);
9455         E1000_WRITE_REG(hw, E1000_RTTBCNRC, bcnrc_val);
9456 }
9457
9458 static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
9459 {
9460         int actual_link_speed, i;
9461         bool reset_rate = false;
9462
9463         /* VF TX rate limit was not set */
9464         if ((adapter->vf_rate_link_speed == 0) ||
9465                 (adapter->hw.mac.type != e1000_82576))
9466                 return;
9467
9468         actual_link_speed = igb_link_mbps(adapter->link_speed);
9469         if (actual_link_speed != adapter->vf_rate_link_speed) {
9470                 reset_rate = true;
9471                 adapter->vf_rate_link_speed = 0;
9472                 dev_info(&adapter->pdev->dev,
9473                 "Link speed has been changed. VF Transmit rate is disabled\n");
9474         }
9475
9476         for (i = 0; i < adapter->vfs_allocated_count; i++) {
9477                 if (reset_rate)
9478                         adapter->vf_data[i].tx_rate = 0;
9479
9480                 igb_set_vf_rate_limit(&adapter->hw, i,
9481                         adapter->vf_data[i].tx_rate, actual_link_speed);
9482         }
9483 }
9484
9485 #ifdef HAVE_VF_MIN_MAX_TXRATE
9486 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int min_tx_rate,
9487                              int tx_rate)
9488 #else /* HAVE_VF_MIN_MAX_TXRATE */
9489 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
9490 #endif /* HAVE_VF_MIN_MAX_TXRATE */
9491 {
9492         struct igb_adapter *adapter = netdev_priv(netdev);
9493         struct e1000_hw *hw = &adapter->hw;
9494         int actual_link_speed;
9495
9496         if (hw->mac.type != e1000_82576)
9497                 return -EOPNOTSUPP;
9498
9499 #ifdef HAVE_VF_MIN_MAX_TXRATE
9500         if (min_tx_rate)
9501                 return -EINVAL;
9502 #endif /* HAVE_VF_MIN_MAX_TXRATE */
9503
9504         actual_link_speed = igb_link_mbps(adapter->link_speed);
9505         if ((vf >= adapter->vfs_allocated_count) ||
9506                 (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) ||
9507                 (tx_rate < 0) || (tx_rate > actual_link_speed))
9508                 return -EINVAL;
9509
9510         adapter->vf_rate_link_speed = actual_link_speed;
9511         adapter->vf_data[vf].tx_rate = (u16)tx_rate;
9512         igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
9513
9514         return 0;
9515 }
9516
9517 static int igb_ndo_get_vf_config(struct net_device *netdev,
9518                                  int vf, struct ifla_vf_info *ivi)
9519 {
9520         struct igb_adapter *adapter = netdev_priv(netdev);
9521         if (vf >= adapter->vfs_allocated_count)
9522                 return -EINVAL;
9523         ivi->vf = vf;
9524         memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
9525 #ifdef HAVE_VF_MIN_MAX_TXRATE
9526         ivi->max_tx_rate = adapter->vf_data[vf].tx_rate;
9527         ivi->min_tx_rate = 0;
9528 #else /* HAVE_VF_MIN_MAX_TXRATE */
9529         ivi->tx_rate = adapter->vf_data[vf].tx_rate;
9530 #endif /* HAVE_VF_MIN_MAX_TXRATE */
9531         ivi->vlan = adapter->vf_data[vf].pf_vlan;
9532         ivi->qos = adapter->vf_data[vf].pf_qos;
9533 #ifdef HAVE_VF_SPOOFCHK_CONFIGURE
9534         ivi->spoofchk = adapter->vf_data[vf].spoofchk_enabled;
9535 #endif
9536         return 0;
9537 }
9538 #endif
9539 static void igb_vmm_control(struct igb_adapter *adapter)
9540 {
9541         struct e1000_hw *hw = &adapter->hw;
9542         int count;
9543         u32 reg;
9544
9545         switch (hw->mac.type) {
9546         case e1000_82575:
9547         default:
9548                 /* replication is not supported for 82575 */
9549                 return;
9550         case e1000_82576:
9551                 /* notify HW that the MAC is adding vlan tags */
9552                 reg = E1000_READ_REG(hw, E1000_DTXCTL);
9553                 reg |= (E1000_DTXCTL_VLAN_ADDED |
9554                         E1000_DTXCTL_SPOOF_INT);
9555                 E1000_WRITE_REG(hw, E1000_DTXCTL, reg);
9556         case e1000_82580:
9557                 /* enable replication vlan tag stripping */
9558                 reg = E1000_READ_REG(hw, E1000_RPLOLR);
9559                 reg |= E1000_RPLOLR_STRVLAN;
9560                 E1000_WRITE_REG(hw, E1000_RPLOLR, reg);
9561         case e1000_i350:
9562         case e1000_i354:
9563                 /* none of the above registers are supported by i350 */
9564                 break;
9565         }
9566
9567         /* Enable Malicious Driver Detection */
9568         if ((adapter->vfs_allocated_count) &&
9569             (adapter->mdd)) {
9570                 if (hw->mac.type == e1000_i350)
9571                         igb_enable_mdd(adapter);
9572         }
9573
9574                 /* enable replication and loopback support */
9575                 count = adapter->vfs_allocated_count || adapter->vmdq_pools;
9576                 if (adapter->flags & IGB_FLAG_LOOPBACK_ENABLE && count)
9577                         e1000_vmdq_set_loopback_pf(hw, 1);
9578                 e1000_vmdq_set_anti_spoofing_pf(hw,
9579                         adapter->vfs_allocated_count || adapter->vmdq_pools,
9580                         adapter->vfs_allocated_count);
9581         e1000_vmdq_set_replication_pf(hw, adapter->vfs_allocated_count ||
9582                                       adapter->vmdq_pools);
9583 }
9584
9585 static void igb_init_fw(struct igb_adapter *adapter)
9586 {
9587         struct e1000_fw_drv_info fw_cmd;
9588         struct e1000_hw *hw = &adapter->hw;
9589         int i;
9590         u16 mask;
9591
9592         if (hw->mac.type == e1000_i210)
9593                 mask = E1000_SWFW_EEP_SM;
9594         else
9595                 mask = E1000_SWFW_PHY0_SM;
9596         /* i211 parts do not support this feature */
9597         if (hw->mac.type == e1000_i211)
9598                 hw->mac.arc_subsystem_valid = false;
9599
9600         if (!hw->mac.ops.acquire_swfw_sync(hw, mask)) {
9601                 for (i = 0; i <= FW_MAX_RETRIES; i++) {
9602                         E1000_WRITE_REG(hw, E1000_FWSTS, E1000_FWSTS_FWRI);
9603                         fw_cmd.hdr.cmd = FW_CMD_DRV_INFO;
9604                         fw_cmd.hdr.buf_len = FW_CMD_DRV_INFO_LEN;
9605                         fw_cmd.hdr.cmd_or_resp.cmd_resv = FW_CMD_RESERVED;
9606                         fw_cmd.port_num = hw->bus.func;
9607                         fw_cmd.drv_version = FW_FAMILY_DRV_VER;
9608                         fw_cmd.hdr.checksum = 0;
9609                         fw_cmd.hdr.checksum = e1000_calculate_checksum((u8 *)&fw_cmd,
9610                                                                    (FW_HDR_LEN +
9611                                                                     fw_cmd.hdr.buf_len));
9612                          e1000_host_interface_command(hw, (u8*)&fw_cmd,
9613                                                      sizeof(fw_cmd));
9614                         if (fw_cmd.hdr.cmd_or_resp.ret_status == FW_STATUS_SUCCESS)
9615                                 break;
9616                 }
9617         } else
9618                 dev_warn(pci_dev_to_dev(adapter->pdev),
9619                          "Unable to get semaphore, firmware init failed.\n");
9620         hw->mac.ops.release_swfw_sync(hw, mask);
9621 }
9622
9623 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
9624 {
9625         struct e1000_hw *hw = &adapter->hw;
9626         u32 dmac_thr;
9627         u16 hwm;
9628         u32 status;
9629
9630         if (hw->mac.type == e1000_i211)
9631                 return;
9632
9633         if (hw->mac.type > e1000_82580) {
9634                 if (adapter->dmac != IGB_DMAC_DISABLE) {
9635                         u32 reg;
9636
9637                         /* force threshold to 0.  */
9638                         E1000_WRITE_REG(hw, E1000_DMCTXTH, 0);
9639
9640                         /*
9641                          * DMA Coalescing high water mark needs to be greater
9642                          * than the Rx threshold. Set hwm to PBA - max frame
9643                          * size in 16B units, capping it at PBA - 6KB.
9644                          */
9645                         hwm = 64 * pba - adapter->max_frame_size / 16;
9646                         if (hwm < 64 * (pba - 6))
9647                                 hwm = 64 * (pba - 6);
9648                         reg = E1000_READ_REG(hw, E1000_FCRTC);
9649                         reg &= ~E1000_FCRTC_RTH_COAL_MASK;
9650                         reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
9651                                 & E1000_FCRTC_RTH_COAL_MASK);
9652                         E1000_WRITE_REG(hw, E1000_FCRTC, reg);
9653
9654                         /*
9655                          * Set the DMA Coalescing Rx threshold to PBA - 2 * max
9656                          * frame size, capping it at PBA - 10KB.
9657                          */
9658                         dmac_thr = pba - adapter->max_frame_size / 512;
9659                         if (dmac_thr < pba - 10)
9660                                 dmac_thr = pba - 10;
9661                         reg = E1000_READ_REG(hw, E1000_DMACR);
9662                         reg &= ~E1000_DMACR_DMACTHR_MASK;
9663                         reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
9664                                 & E1000_DMACR_DMACTHR_MASK);
9665
9666                         /* transition to L0x or L1 if available..*/
9667                         reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
9668
9669                         /* Check if status is 2.5Gb backplane connection
9670                          * before configuration of watchdog timer, which is
9671                          * in msec values in 12.8usec intervals
9672                          * watchdog timer= msec values in 32usec intervals
9673                          * for non 2.5Gb connection
9674                          */
9675                         if (hw->mac.type == e1000_i354) {
9676                                 status = E1000_READ_REG(hw, E1000_STATUS);
9677                                 if ((status & E1000_STATUS_2P5_SKU) &&
9678                                     (!(status & E1000_STATUS_2P5_SKU_OVER)))
9679                                         reg |= ((adapter->dmac * 5) >> 6);
9680                                 else
9681                                         reg |= ((adapter->dmac) >> 5);
9682                         } else {
9683                                 reg |= ((adapter->dmac) >> 5);
9684                         }
9685
9686                         /*
9687                          * Disable BMC-to-OS Watchdog enable
9688                          * on devices that support OS-to-BMC
9689                          */
9690                         if (hw->mac.type != e1000_i354)
9691                                 reg &= ~E1000_DMACR_DC_BMC2OSW_EN;
9692                         E1000_WRITE_REG(hw, E1000_DMACR, reg);
9693
9694                         /* no lower threshold to disable coalescing(smart fifb)-UTRESH=0*/
9695                         E1000_WRITE_REG(hw, E1000_DMCRTRH, 0);
9696
9697                         /* This sets the time to wait before requesting
9698                          * transition to low power state to number of usecs
9699                          * needed to receive 1 512 byte frame at gigabit
9700                          * line rate. On i350 device, time to make transition
9701                          * to Lx state is delayed by 4 usec with flush disable
9702                          * bit set to avoid losing mailbox interrupts
9703                          */
9704                         reg = E1000_READ_REG(hw, E1000_DMCTLX);
9705                         if (hw->mac.type == e1000_i350)
9706                                 reg |= IGB_DMCTLX_DCFLUSH_DIS;
9707
9708                         /* in 2.5Gb connection, TTLX unit is 0.4 usec
9709                          * which is 0x4*2 = 0xA. But delay is still 4 usec
9710                          */
9711                         if (hw->mac.type == e1000_i354) {
9712                                 status = E1000_READ_REG(hw, E1000_STATUS);
9713                                 if ((status & E1000_STATUS_2P5_SKU) &&
9714                                     (!(status & E1000_STATUS_2P5_SKU_OVER)))
9715                                         reg |= 0xA;
9716                                 else
9717                                         reg |= 0x4;
9718                         } else {
9719                                 reg |= 0x4;
9720                         }
9721                         E1000_WRITE_REG(hw, E1000_DMCTLX, reg);
9722
9723                         /* free space in tx packet buffer to wake from DMA coal */
9724                         E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
9725                                 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
9726
9727                         /* make low power state decision controlled by DMA coal */
9728                         reg = E1000_READ_REG(hw, E1000_PCIEMISC);
9729                         reg &= ~E1000_PCIEMISC_LX_DECISION;
9730                         E1000_WRITE_REG(hw, E1000_PCIEMISC, reg);
9731                 } /* endif adapter->dmac is not disabled */
9732         } else if (hw->mac.type == e1000_82580) {
9733                 u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
9734                 E1000_WRITE_REG(hw, E1000_PCIEMISC,
9735                                 reg & ~E1000_PCIEMISC_LX_DECISION);
9736                 E1000_WRITE_REG(hw, E1000_DMACR, 0);
9737         }
9738 }
9739
9740 #ifdef HAVE_I2C_SUPPORT
9741 /*  igb_read_i2c_byte - Reads 8 bit word over I2C
9742  *  @hw: pointer to hardware structure
9743  *  @byte_offset: byte offset to read
9744  *  @dev_addr: device address
9745  *  @data: value read
9746  *
9747  *  Performs byte read operation over I2C interface at
9748  *  a specified device address.
9749  */
9750 s32 igb_read_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
9751                                 u8 dev_addr, u8 *data)
9752 {
9753         struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
9754         struct i2c_client *this_client = adapter->i2c_client;
9755         s32 status;
9756         u16 swfw_mask = 0;
9757
9758         if (!this_client)
9759                 return E1000_ERR_I2C;
9760
9761         swfw_mask = E1000_SWFW_PHY0_SM;
9762
9763         if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask)
9764             != E1000_SUCCESS)
9765                 return E1000_ERR_SWFW_SYNC;
9766
9767         status = i2c_smbus_read_byte_data(this_client, byte_offset);
9768         hw->mac.ops.release_swfw_sync(hw, swfw_mask);
9769
9770         if (status < 0)
9771                 return E1000_ERR_I2C;
9772         else {
9773                 *data = status;
9774                 return E1000_SUCCESS;
9775         }
9776 }
9777
9778 /*  igb_write_i2c_byte - Writes 8 bit word over I2C
9779  *  @hw: pointer to hardware structure
9780  *  @byte_offset: byte offset to write
9781  *  @dev_addr: device address
9782  *  @data: value to write
9783  *
9784  *  Performs byte write operation over I2C interface at
9785  *  a specified device address.
9786  */
9787 s32 igb_write_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
9788                                  u8 dev_addr, u8 data)
9789 {
9790         struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
9791         struct i2c_client *this_client = adapter->i2c_client;
9792         s32 status;
9793         u16 swfw_mask = E1000_SWFW_PHY0_SM;
9794
9795         if (!this_client)
9796                 return E1000_ERR_I2C;
9797
9798         if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask) != E1000_SUCCESS)
9799                 return E1000_ERR_SWFW_SYNC;
9800         status = i2c_smbus_write_byte_data(this_client, byte_offset, data);
9801         hw->mac.ops.release_swfw_sync(hw, swfw_mask);
9802
9803         if (status)
9804                 return E1000_ERR_I2C;
9805         else
9806                 return E1000_SUCCESS;
9807 }
9808 #endif /*  HAVE_I2C_SUPPORT */
9809 /* igb_main.c */
9810
9811
9812 /**
9813  * igb_probe - Device Initialization Routine
9814  * @pdev: PCI device information struct
9815  * @ent: entry in igb_pci_tbl
9816  *
9817  * Returns 0 on success, negative on failure
9818  *
9819  * igb_probe initializes an adapter identified by a pci_dev structure.
9820  * The OS initialization, configuring of the adapter private structure,
9821  * and a hardware reset occur.
9822  **/
9823 int igb_kni_probe(struct pci_dev *pdev,
9824                                struct net_device **lad_dev)
9825 {
9826         struct net_device *netdev;
9827         struct igb_adapter *adapter;
9828         struct e1000_hw *hw;
9829         u16 eeprom_data = 0;
9830         u8 pba_str[E1000_PBANUM_LENGTH];
9831         s32 ret_val;
9832         static int global_quad_port_a; /* global quad port a indication */
9833         int i, err, pci_using_dac = 0;
9834         static int cards_found;
9835
9836         err = pci_enable_device_mem(pdev);
9837         if (err)
9838                 return err;
9839
9840 #ifdef NO_KNI
9841         pci_using_dac = 0;
9842         err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
9843         if (!err) {
9844                 err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
9845                 if (!err)
9846                         pci_using_dac = 1;
9847         } else {
9848                 err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
9849                 if (err) {
9850                         err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
9851                         if (err) {
9852                                 IGB_ERR("No usable DMA configuration, "
9853                                         "aborting\n");
9854                                 goto err_dma;
9855                         }
9856                 }
9857         }
9858
9859 #ifndef HAVE_ASPM_QUIRKS
9860         /* 82575 requires that the pci-e link partner disable the L0s state */
9861         switch (pdev->device) {
9862         case E1000_DEV_ID_82575EB_COPPER:
9863         case E1000_DEV_ID_82575EB_FIBER_SERDES:
9864         case E1000_DEV_ID_82575GB_QUAD_COPPER:
9865                 pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S);
9866         default:
9867                 break;
9868         }
9869
9870 #endif /* HAVE_ASPM_QUIRKS */
9871         err = pci_request_selected_regions(pdev,
9872                                            pci_select_bars(pdev,
9873                                                            IORESOURCE_MEM),
9874                                            igb_driver_name);
9875         if (err)
9876                 goto err_pci_reg;
9877
9878         pci_enable_pcie_error_reporting(pdev);
9879
9880         pci_set_master(pdev);
9881
9882         err = -ENOMEM;
9883 #endif /* NO_KNI */
9884 #ifdef HAVE_TX_MQ
9885         netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
9886                                    IGB_MAX_TX_QUEUES);
9887 #else
9888         netdev = alloc_etherdev(sizeof(struct igb_adapter));
9889 #endif /* HAVE_TX_MQ */
9890         if (!netdev)
9891                 goto err_alloc_etherdev;
9892
9893         SET_MODULE_OWNER(netdev);
9894         SET_NETDEV_DEV(netdev, &pdev->dev);
9895
9896         //pci_set_drvdata(pdev, netdev);
9897         adapter = netdev_priv(netdev);
9898         adapter->netdev = netdev;
9899         adapter->pdev = pdev;
9900         hw = &adapter->hw;
9901         hw->back = adapter;
9902         adapter->port_num = hw->bus.func;
9903         adapter->msg_enable = (1 << debug) - 1;
9904
9905 #ifdef HAVE_PCI_ERS
9906         err = pci_save_state(pdev);
9907         if (err)
9908                 goto err_ioremap;
9909 #endif
9910         err = -EIO;
9911         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
9912                               pci_resource_len(pdev, 0));
9913         if (!hw->hw_addr)
9914                 goto err_ioremap;
9915
9916 #ifdef HAVE_NET_DEVICE_OPS
9917         netdev->netdev_ops = &igb_netdev_ops;
9918 #else /* HAVE_NET_DEVICE_OPS */
9919         netdev->open = &igb_open;
9920         netdev->stop = &igb_close;
9921         netdev->get_stats = &igb_get_stats;
9922 #ifdef HAVE_SET_RX_MODE
9923         netdev->set_rx_mode = &igb_set_rx_mode;
9924 #endif
9925         netdev->set_multicast_list = &igb_set_rx_mode;
9926         netdev->set_mac_address = &igb_set_mac;
9927         netdev->change_mtu = &igb_change_mtu;
9928         netdev->do_ioctl = &igb_ioctl;
9929 #ifdef HAVE_TX_TIMEOUT
9930         netdev->tx_timeout = &igb_tx_timeout;
9931 #endif
9932         netdev->vlan_rx_register = igb_vlan_mode;
9933         netdev->vlan_rx_add_vid = igb_vlan_rx_add_vid;
9934         netdev->vlan_rx_kill_vid = igb_vlan_rx_kill_vid;
9935 #ifdef CONFIG_NET_POLL_CONTROLLER
9936         netdev->poll_controller = igb_netpoll;
9937 #endif
9938         netdev->hard_start_xmit = &igb_xmit_frame;
9939 #endif /* HAVE_NET_DEVICE_OPS */
9940         igb_set_ethtool_ops(netdev);
9941 #ifdef HAVE_TX_TIMEOUT
9942         netdev->watchdog_timeo = 5 * HZ;
9943 #endif
9944
9945         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
9946
9947         adapter->bd_number = cards_found;
9948
9949         /* setup the private structure */
9950         err = igb_sw_init(adapter);
9951         if (err)
9952                 goto err_sw_init;
9953
9954         e1000_get_bus_info(hw);
9955
9956         hw->phy.autoneg_wait_to_complete = FALSE;
9957         hw->mac.adaptive_ifs = FALSE;
9958
9959         /* Copper options */
9960         if (hw->phy.media_type == e1000_media_type_copper) {
9961                 hw->phy.mdix = AUTO_ALL_MODES;
9962                 hw->phy.disable_polarity_correction = FALSE;
9963                 hw->phy.ms_type = e1000_ms_hw_default;
9964         }
9965
9966         if (e1000_check_reset_block(hw))
9967                 dev_info(pci_dev_to_dev(pdev),
9968                         "PHY reset is blocked due to SOL/IDER session.\n");
9969
9970         /*
9971          * features is initialized to 0 in allocation, it might have bits
9972          * set by igb_sw_init so we should use an or instead of an
9973          * assignment.
9974          */
9975         netdev->features |= NETIF_F_SG |
9976                             NETIF_F_IP_CSUM |
9977 #ifdef NETIF_F_IPV6_CSUM
9978                             NETIF_F_IPV6_CSUM |
9979 #endif
9980 #ifdef NETIF_F_TSO
9981                             NETIF_F_TSO |
9982 #ifdef NETIF_F_TSO6
9983                             NETIF_F_TSO6 |
9984 #endif
9985 #endif /* NETIF_F_TSO */
9986 #ifdef NETIF_F_RXHASH
9987                             NETIF_F_RXHASH |
9988 #endif
9989                             NETIF_F_RXCSUM |
9990 #ifdef NETIF_F_HW_VLAN_CTAG_RX
9991                             NETIF_F_HW_VLAN_CTAG_RX |
9992                             NETIF_F_HW_VLAN_CTAG_TX;
9993 #else
9994                             NETIF_F_HW_VLAN_RX |
9995                             NETIF_F_HW_VLAN_TX;
9996 #endif
9997
9998         if (hw->mac.type >= e1000_82576)
9999                 netdev->features |= NETIF_F_SCTP_CSUM;
10000
10001 #ifdef HAVE_NDO_SET_FEATURES
10002         /* copy netdev features into list of user selectable features */
10003         netdev->hw_features |= netdev->features;
10004 #ifndef IGB_NO_LRO
10005
10006         /* give us the option of enabling LRO later */
10007         netdev->hw_features |= NETIF_F_LRO;
10008 #endif
10009 #else
10010 #ifdef NETIF_F_GRO
10011
10012         /* this is only needed on kernels prior to 2.6.39 */
10013         netdev->features |= NETIF_F_GRO;
10014 #endif
10015 #endif
10016
10017         /* set this bit last since it cannot be part of hw_features */
10018 #ifdef NETIF_F_HW_VLAN_CTAG_FILTER
10019         netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
10020 #else
10021         netdev->features |= NETIF_F_HW_VLAN_FILTER;
10022 #endif
10023
10024 #ifdef HAVE_NETDEV_VLAN_FEATURES
10025         netdev->vlan_features |= NETIF_F_TSO |
10026                                  NETIF_F_TSO6 |
10027                                  NETIF_F_IP_CSUM |
10028                                  NETIF_F_IPV6_CSUM |
10029                                  NETIF_F_SG;
10030
10031 #endif
10032         if (pci_using_dac)
10033                 netdev->features |= NETIF_F_HIGHDMA;
10034
10035 #ifdef NO_KNI
10036         adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
10037 #ifdef DEBUG
10038         if (adapter->dmac != IGB_DMAC_DISABLE)
10039                 printk("%s: DMA Coalescing is enabled..\n", netdev->name);
10040 #endif
10041
10042         /* before reading the NVM, reset the controller to put the device in a
10043          * known good starting state */
10044         e1000_reset_hw(hw);
10045 #endif /* NO_KNI */
10046
10047         /* make sure the NVM is good */
10048         if (e1000_validate_nvm_checksum(hw) < 0) {
10049                 dev_err(pci_dev_to_dev(pdev), "The NVM Checksum Is Not"
10050                         " Valid\n");
10051                 err = -EIO;
10052                 goto err_eeprom;
10053         }
10054
10055         /* copy the MAC address out of the NVM */
10056         if (e1000_read_mac_addr(hw))
10057                 dev_err(pci_dev_to_dev(pdev), "NVM Read Error\n");
10058         memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
10059 #ifdef ETHTOOL_GPERMADDR
10060         memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
10061
10062         if (!is_valid_ether_addr(netdev->perm_addr)) {
10063 #else
10064         if (!is_valid_ether_addr(netdev->dev_addr)) {
10065 #endif
10066                 dev_err(pci_dev_to_dev(pdev), "Invalid MAC Address\n");
10067                 err = -EIO;
10068                 goto err_eeprom;
10069         }
10070
10071         memcpy(&adapter->mac_table[0].addr, hw->mac.addr, netdev->addr_len);
10072         adapter->mac_table[0].queue = adapter->vfs_allocated_count;
10073         adapter->mac_table[0].state = (IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE);
10074         igb_rar_set(adapter, 0);
10075
10076         /* get firmware version for ethtool -i */
10077         igb_set_fw_version(adapter);
10078
10079         /* Check if Media Autosense is enabled */
10080         if (hw->mac.type == e1000_82580)
10081                 igb_init_mas(adapter);
10082
10083 #ifdef NO_KNI
10084 #ifdef HAVE_TIMER_SETUP
10085         timer_setup(&adapter->watchdog_timer, &igb_watchdog, 0);
10086         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
10087                 timer_setup(&adapter->dma_err_timer, &igb_dma_err_timer, 0);
10088         timer_setup(&adapter->phy_info_timer, &igb_update_phy_info, 0);
10089 #else
10090         setup_timer(&adapter->watchdog_timer, &igb_watchdog,
10091                     (unsigned long) adapter);
10092         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
10093                 setup_timer(&adapter->dma_err_timer, &igb_dma_err_timer,
10094                             (unsigned long) adapter);
10095         setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
10096                     (unsigned long) adapter);
10097 #endif
10098
10099         INIT_WORK(&adapter->reset_task, igb_reset_task);
10100         INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
10101         if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
10102                 INIT_WORK(&adapter->dma_err_task, igb_dma_err_task);
10103 #endif
10104
10105         /* Initialize link properties that are user-changeable */
10106         adapter->fc_autoneg = true;
10107         hw->mac.autoneg = true;
10108         hw->phy.autoneg_advertised = 0x2f;
10109
10110         hw->fc.requested_mode = e1000_fc_default;
10111         hw->fc.current_mode = e1000_fc_default;
10112
10113         e1000_validate_mdi_setting(hw);
10114
10115         /* By default, support wake on port A */
10116         if (hw->bus.func == 0)
10117                 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
10118
10119         /* Check the NVM for wake support for non-port A ports */
10120         if (hw->mac.type >= e1000_82580)
10121                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
10122                                  NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
10123                                  &eeprom_data);
10124         else if (hw->bus.func == 1)
10125                 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
10126
10127         if (eeprom_data & IGB_EEPROM_APME)
10128                 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
10129
10130         /* now that we have the eeprom settings, apply the special cases where
10131          * the eeprom may be wrong or the board simply won't support wake on
10132          * lan on a particular port */
10133         switch (pdev->device) {
10134         case E1000_DEV_ID_82575GB_QUAD_COPPER:
10135                 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
10136                 break;
10137         case E1000_DEV_ID_82575EB_FIBER_SERDES:
10138         case E1000_DEV_ID_82576_FIBER:
10139         case E1000_DEV_ID_82576_SERDES:
10140                 /* Wake events only supported on port A for dual fiber
10141                  * regardless of eeprom setting */
10142                 if (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_FUNC_1)
10143                         adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
10144                 break;
10145         case E1000_DEV_ID_82576_QUAD_COPPER:
10146         case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
10147                 /* if quad port adapter, disable WoL on all but port A */
10148                 if (global_quad_port_a != 0)
10149                         adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
10150                 else
10151                         adapter->flags |= IGB_FLAG_QUAD_PORT_A;
10152                 /* Reset for multiple quad port adapters */
10153                 if (++global_quad_port_a == 4)
10154                         global_quad_port_a = 0;
10155                 break;
10156         default:
10157                 /* If the device can't wake, don't set software support */
10158                 if (!device_can_wakeup(&adapter->pdev->dev))
10159                         adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
10160                 break;
10161         }
10162
10163         /* initialize the wol settings based on the eeprom settings */
10164         if (adapter->flags & IGB_FLAG_WOL_SUPPORTED)
10165                 adapter->wol |= E1000_WUFC_MAG;
10166
10167         /* Some vendors want WoL disabled by default, but still supported */
10168         if ((hw->mac.type == e1000_i350) &&
10169             (pdev->subsystem_vendor == PCI_VENDOR_ID_HP)) {
10170                 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
10171                 adapter->wol = 0;
10172         }
10173
10174 #ifdef NO_KNI
10175         device_set_wakeup_enable(pci_dev_to_dev(adapter->pdev),
10176                                  adapter->flags & IGB_FLAG_WOL_SUPPORTED);
10177
10178         /* reset the hardware with the new settings */
10179         igb_reset(adapter);
10180         adapter->devrc = 0;
10181
10182 #ifdef HAVE_I2C_SUPPORT
10183         /* Init the I2C interface */
10184         err = igb_init_i2c(adapter);
10185         if (err) {
10186                 dev_err(&pdev->dev, "failed to init i2c interface\n");
10187                 goto err_eeprom;
10188         }
10189 #endif /* HAVE_I2C_SUPPORT */
10190
10191         /* let the f/w know that the h/w is now under the control of the
10192          * driver. */
10193         igb_get_hw_control(adapter);
10194
10195         strncpy(netdev->name, "eth%d", IFNAMSIZ);
10196         err = register_netdev(netdev);
10197         if (err)
10198                 goto err_register;
10199
10200 #ifdef CONFIG_IGB_VMDQ_NETDEV
10201         err = igb_init_vmdq_netdevs(adapter);
10202         if (err)
10203                 goto err_register;
10204 #endif
10205         /* carrier off reporting is important to ethtool even BEFORE open */
10206         netif_carrier_off(netdev);
10207
10208 #ifdef IGB_DCA
10209         if (dca_add_requester(&pdev->dev) == E1000_SUCCESS) {
10210                 adapter->flags |= IGB_FLAG_DCA_ENABLED;
10211                 dev_info(pci_dev_to_dev(pdev), "DCA enabled\n");
10212                 igb_setup_dca(adapter);
10213         }
10214
10215 #endif
10216 #ifdef HAVE_PTP_1588_CLOCK
10217         /* do hw tstamp init after resetting */
10218         igb_ptp_init(adapter);
10219 #endif /* HAVE_PTP_1588_CLOCK */
10220
10221 #endif /* NO_KNI */
10222         dev_info(pci_dev_to_dev(pdev), "Intel(R) Gigabit Ethernet Network Connection\n");
10223         /* print bus type/speed/width info */
10224         dev_info(pci_dev_to_dev(pdev), "%s: (PCIe:%s:%s) ",
10225                  netdev->name,
10226                  ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5GT/s" :
10227                   (hw->bus.speed == e1000_bus_speed_5000) ? "5.0GT/s" :
10228                   (hw->mac.type == e1000_i354) ? "integrated" :
10229                                                             "unknown"),
10230                  ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
10231                   (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
10232                   (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
10233                   (hw->mac.type == e1000_i354) ? "integrated" :
10234                    "unknown"));
10235         dev_info(pci_dev_to_dev(pdev), "%s: MAC: ", netdev->name);
10236         for (i = 0; i < 6; i++)
10237                 printk("%2.2x%c", netdev->dev_addr[i], i == 5 ? '\n' : ':');
10238
10239         ret_val = e1000_read_pba_string(hw, pba_str, E1000_PBANUM_LENGTH);
10240         if (ret_val)
10241                 strncpy(pba_str, "Unknown", sizeof(pba_str) - 1);
10242         dev_info(pci_dev_to_dev(pdev), "%s: PBA No: %s\n", netdev->name,
10243                  pba_str);
10244
10245
10246         /* Initialize the thermal sensor on i350 devices. */
10247         if (hw->mac.type == e1000_i350) {
10248                 if (hw->bus.func == 0) {
10249                         u16 ets_word;
10250
10251                         /*
10252                          * Read the NVM to determine if this i350 device
10253                          * supports an external thermal sensor.
10254                          */
10255                         e1000_read_nvm(hw, NVM_ETS_CFG, 1, &ets_word);
10256                         if (ets_word != 0x0000 && ets_word != 0xFFFF)
10257                                 adapter->ets = true;
10258                         else
10259                                 adapter->ets = false;
10260                 }
10261 #ifdef NO_KNI
10262 #ifdef IGB_HWMON
10263
10264                 igb_sysfs_init(adapter);
10265 #else
10266 #ifdef IGB_PROCFS
10267
10268                 igb_procfs_init(adapter);
10269 #endif /* IGB_PROCFS */
10270 #endif /* IGB_HWMON */
10271 #endif /* NO_KNI */
10272         } else {
10273                 adapter->ets = false;
10274         }
10275
10276         if (hw->phy.media_type == e1000_media_type_copper) {
10277                 switch (hw->mac.type) {
10278                 case e1000_i350:
10279                 case e1000_i210:
10280                 case e1000_i211:
10281                         /* Enable EEE for internal copper PHY devices */
10282                         err = e1000_set_eee_i350(hw);
10283                         if ((!err) &&
10284                             (adapter->flags & IGB_FLAG_EEE))
10285                                 adapter->eee_advert =
10286                                         MDIO_EEE_100TX | MDIO_EEE_1000T;
10287                         break;
10288                 case e1000_i354:
10289                         if ((E1000_READ_REG(hw, E1000_CTRL_EXT)) &
10290                             (E1000_CTRL_EXT_LINK_MODE_SGMII)) {
10291                                 err = e1000_set_eee_i354(hw);
10292                                 if ((!err) &&
10293                                     (adapter->flags & IGB_FLAG_EEE))
10294                                         adapter->eee_advert =
10295                                            MDIO_EEE_100TX | MDIO_EEE_1000T;
10296                         }
10297                         break;
10298                 default:
10299                         break;
10300                 }
10301         }
10302
10303         /* send driver version info to firmware */
10304         if (hw->mac.type >= e1000_i350)
10305                 igb_init_fw(adapter);
10306
10307 #ifndef IGB_NO_LRO
10308         if (netdev->features & NETIF_F_LRO)
10309                 dev_info(pci_dev_to_dev(pdev), "Internal LRO is enabled \n");
10310         else
10311                 dev_info(pci_dev_to_dev(pdev), "LRO is disabled \n");
10312 #endif
10313         dev_info(pci_dev_to_dev(pdev),
10314                  "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
10315                  adapter->msix_entries ? "MSI-X" :
10316                  (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
10317                  adapter->num_rx_queues, adapter->num_tx_queues);
10318
10319         cards_found++;
10320         *lad_dev = netdev;
10321
10322         pm_runtime_put_noidle(&pdev->dev);
10323         return 0;
10324
10325 //err_register:
10326 //      igb_release_hw_control(adapter);
10327 #ifdef HAVE_I2C_SUPPORT
10328         memset(&adapter->i2c_adap, 0, sizeof(adapter->i2c_adap));
10329 #endif /* HAVE_I2C_SUPPORT */
10330 err_eeprom:
10331 //      if (!e1000_check_reset_block(hw))
10332 //              e1000_phy_hw_reset(hw);
10333
10334         if (hw->flash_address)
10335                 iounmap(hw->flash_address);
10336 err_sw_init:
10337 //      igb_clear_interrupt_scheme(adapter);
10338 //      igb_reset_sriov_capability(adapter);
10339         iounmap(hw->hw_addr);
10340 err_ioremap:
10341         free_netdev(netdev);
10342 err_alloc_etherdev:
10343 //      pci_release_selected_regions(pdev,
10344 //                                   pci_select_bars(pdev, IORESOURCE_MEM));
10345 //err_pci_reg:
10346 //err_dma:
10347         pci_disable_device(pdev);
10348         return err;
10349 }
10350
10351
10352 void igb_kni_remove(struct pci_dev *pdev)
10353 {
10354         pci_disable_device(pdev);
10355 }