1 /*******************************************************************************
3 Intel(R) Gigabit Ethernet Linux driver
4 Copyright(c) 2007-2012 Intel Corporation.
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
26 *******************************************************************************/
28 #include <linux/module.h>
29 #include <linux/types.h>
30 #include <linux/init.h>
31 #include <linux/vmalloc.h>
32 #include <linux/pagemap.h>
33 #include <linux/netdevice.h>
34 #include <linux/tcp.h>
36 #include <net/checksum.h>
38 #include <linux/ipv6.h>
39 #include <net/ip6_checksum.h>
43 #include <linux/mii.h>
46 #include <linux/ethtool.h>
48 #include <linux/if_vlan.h>
49 #ifdef CONFIG_PM_RUNTIME
50 #include <linux/pm_runtime.h>
51 #endif /* CONFIG_PM_RUNTIME */
56 #include <linux/uio_driver.h>
60 #define VERSION_SUFFIX
65 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." __stringify(BUILD) VERSION_SUFFIX DRV_DEBUG DRV_HW_PERF
67 char igb_driver_name[] = "igb";
68 char igb_driver_version[] = DRV_VERSION;
69 static const char igb_driver_string[] =
70 "Intel(R) Gigabit Ethernet Network Driver";
71 static const char igb_copyright[] = "Copyright (c) 2007-2011 Intel Corporation.";
73 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER) },
75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER) },
76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES) },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII) },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER) },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER) },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER) },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES) },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII) },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL) },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII) },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES) },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE) },
87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP) },
88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576) },
89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS) },
90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES) },
91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER) },
92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES) },
93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD) },
94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2) },
95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER) },
96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER) },
97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES) },
98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER) },
99 /* required last entry */
103 //MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
104 static void igb_set_sriov_capability(struct igb_adapter *adapter) __attribute__((__unused__));
105 void igb_reset(struct igb_adapter *);
106 static int igb_setup_all_tx_resources(struct igb_adapter *);
107 static int igb_setup_all_rx_resources(struct igb_adapter *);
108 static void igb_free_all_tx_resources(struct igb_adapter *);
109 static void igb_free_all_rx_resources(struct igb_adapter *);
110 static void igb_setup_mrqc(struct igb_adapter *);
111 void igb_update_stats(struct igb_adapter *);
112 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
113 static void __devexit igb_remove(struct pci_dev *pdev);
114 #ifdef HAVE_HW_TIME_STAMP
115 static void igb_init_hw_timer(struct igb_adapter *adapter);
117 static int igb_sw_init(struct igb_adapter *);
118 static int igb_open(struct net_device *);
119 static int igb_close(struct net_device *);
120 static void igb_configure_tx(struct igb_adapter *);
121 static void igb_configure_rx(struct igb_adapter *);
122 static void igb_clean_all_tx_rings(struct igb_adapter *);
123 static void igb_clean_all_rx_rings(struct igb_adapter *);
124 static void igb_clean_tx_ring(struct igb_ring *);
125 static void igb_set_rx_mode(struct net_device *);
126 static void igb_update_phy_info(unsigned long);
127 static void igb_watchdog(unsigned long);
128 static void igb_watchdog_task(struct work_struct *);
129 static void igb_dma_err_task(struct work_struct *);
130 static void igb_dma_err_timer(unsigned long data);
131 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
132 static struct net_device_stats *igb_get_stats(struct net_device *);
133 static int igb_change_mtu(struct net_device *, int);
134 void igb_full_sync_mac_table(struct igb_adapter *adapter);
135 static int igb_set_mac(struct net_device *, void *);
136 static void igb_set_uta(struct igb_adapter *adapter);
137 static irqreturn_t igb_intr(int irq, void *);
138 static irqreturn_t igb_intr_msi(int irq, void *);
139 static irqreturn_t igb_msix_other(int irq, void *);
140 static irqreturn_t igb_msix_ring(int irq, void *);
142 static void igb_update_dca(struct igb_q_vector *);
143 static void igb_setup_dca(struct igb_adapter *);
145 static int igb_poll(struct napi_struct *, int);
146 static bool igb_clean_tx_irq(struct igb_q_vector *);
147 static bool igb_clean_rx_irq(struct igb_q_vector *, int);
148 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
149 static void igb_tx_timeout(struct net_device *);
150 static void igb_reset_task(struct work_struct *);
151 #ifdef HAVE_VLAN_RX_REGISTER
152 static void igb_vlan_mode(struct net_device *, struct vlan_group *);
154 #ifdef HAVE_VLAN_PROTOCOL
155 static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16);
156 static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16);
157 #elif defined HAVE_INT_NDO_VLAN_RX_ADD_VID
158 static int igb_vlan_rx_add_vid(struct net_device *, u16);
159 static int igb_vlan_rx_kill_vid(struct net_device *, u16);
161 static void igb_vlan_rx_add_vid(struct net_device *, u16);
162 static void igb_vlan_rx_kill_vid(struct net_device *, u16);
164 static void igb_restore_vlan(struct igb_adapter *);
165 void igb_rar_set(struct igb_adapter *adapter, u32 index);
166 static void igb_ping_all_vfs(struct igb_adapter *);
167 static void igb_msg_task(struct igb_adapter *);
168 static void igb_vmm_control(struct igb_adapter *);
169 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
170 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
171 static void igb_process_mdd_event(struct igb_adapter *);
173 static int igb_ndo_set_vf_mac( struct net_device *netdev, int vf, u8 *mac);
174 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
175 int vf, u16 vlan, u8 qos);
176 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
177 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
178 struct ifla_vf_info *ivi);
179 static void igb_check_vf_rate_limit(struct igb_adapter *);
181 static int igb_vf_configure(struct igb_adapter *adapter, int vf);
182 static int igb_check_vf_assignment(struct igb_adapter *adapter);
183 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
184 static int igb_find_enabled_vfs(struct igb_adapter *adapter);
187 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
188 static int igb_suspend(struct device *dev);
189 static int igb_resume(struct device *dev);
190 #ifdef CONFIG_PM_RUNTIME
191 static int igb_runtime_suspend(struct device *dev);
192 static int igb_runtime_resume(struct device *dev);
193 static int igb_runtime_idle(struct device *dev);
194 #endif /* CONFIG_PM_RUNTIME */
195 static const struct dev_pm_ops igb_pm_ops = {
196 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,34)
197 .suspend = igb_suspend,
198 .resume = igb_resume,
199 .freeze = igb_suspend,
201 .poweroff = igb_suspend,
202 .restore = igb_resume,
203 #ifdef CONFIG_PM_RUNTIME
204 .runtime_suspend = igb_runtime_suspend,
205 .runtime_resume = igb_runtime_resume,
206 .runtime_idle = igb_runtime_idle,
208 #else /* Linux >= 2.6.34 */
209 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
210 #ifdef CONFIG_PM_RUNTIME
211 SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
213 #endif /* CONFIG_PM_RUNTIME */
214 #endif /* Linux version */
216 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
217 #endif /* CONFIG_PM */
218 #ifndef USE_REBOOT_NOTIFIER
219 static void igb_shutdown(struct pci_dev *);
221 static int igb_notify_reboot(struct notifier_block *, unsigned long, void *);
222 static struct notifier_block igb_notifier_reboot = {
223 .notifier_call = igb_notify_reboot,
229 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
230 static struct notifier_block dca_notifier = {
231 .notifier_call = igb_notify_dca,
236 #ifdef CONFIG_NET_POLL_CONTROLLER
237 /* for netdump / net console */
238 static void igb_netpoll(struct net_device *);
242 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
243 pci_channel_state_t);
244 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
245 static void igb_io_resume(struct pci_dev *);
247 static struct pci_error_handlers igb_err_handler = {
248 .error_detected = igb_io_error_detected,
249 .slot_reset = igb_io_slot_reset,
250 .resume = igb_io_resume,
254 static void igb_init_fw(struct igb_adapter *adapter);
255 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
257 static struct pci_driver igb_driver = {
258 .name = igb_driver_name,
259 .id_table = igb_pci_tbl,
261 .remove = __devexit_p(igb_remove),
263 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
264 .driver.pm = &igb_pm_ops,
265 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
266 #endif /* CONFIG_PM */
267 #ifndef USE_REBOOT_NOTIFIER
268 .shutdown = igb_shutdown,
271 .err_handler = &igb_err_handler
275 //MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
276 //MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
277 //MODULE_LICENSE("GPL");
278 //MODULE_VERSION(DRV_VERSION);
280 static void igb_vfta_set(struct igb_adapter *adapter, u32 vid, bool add)
282 struct e1000_hw *hw = &adapter->hw;
283 struct e1000_host_mng_dhcp_cookie *mng_cookie = &hw->mng_cookie;
284 u32 index = (vid >> E1000_VFTA_ENTRY_SHIFT) & E1000_VFTA_ENTRY_MASK;
285 u32 mask = 1 << (vid & E1000_VFTA_ENTRY_BIT_SHIFT_MASK);
289 * if this is the management vlan the only option is to add it in so
290 * that the management pass through will continue to work
292 if ((mng_cookie->status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
293 (vid == mng_cookie->vlan_id))
296 vfta = adapter->shadow_vfta[index];
303 e1000_write_vfta(hw, index, vfta);
304 adapter->shadow_vfta[index] = vfta;
307 #ifdef HAVE_HW_TIME_STAMP
309 * igb_read_clock - read raw cycle counter (to be used by time counter)
311 static cycle_t igb_read_clock(const struct cyclecounter *tc)
313 struct igb_adapter *adapter =
314 container_of(tc, struct igb_adapter, cycles);
315 struct e1000_hw *hw = &adapter->hw;
320 * The timestamp latches on lowest register read. For the 82580
321 * the lowest register is SYSTIMR instead of SYSTIML. However we never
322 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
324 if (hw->mac.type >= e1000_82580) {
325 stamp = E1000_READ_REG(hw, E1000_SYSTIMR) >> 8;
326 shift = IGB_82580_TSYNC_SHIFT;
329 stamp |= (u64)E1000_READ_REG(hw, E1000_SYSTIML) << shift;
330 stamp |= (u64)E1000_READ_REG(hw, E1000_SYSTIMH) << (shift + 32);
334 #endif /* SIOCSHWTSTAMP */
335 static int debug = NETIF_MSG_DRV | NETIF_MSG_PROBE;
336 //module_param(debug, int, 0);
337 //MODULE_PARM_DESC(debug, "Debug level (0=none, ..., 16=all)");
340 * igb_init_module - Driver Registration Routine
342 * igb_init_module is the first routine called when the driver is
343 * loaded. All it does is register with the PCI subsystem.
345 static int __init igb_init_module(void)
349 printk(KERN_INFO "%s - version %s\n",
350 igb_driver_string, igb_driver_version);
352 printk(KERN_INFO "%s\n", igb_copyright);
354 /* only use IGB_PROCFS if IGB_SYSFS is not defined */
357 if (igb_procfs_topdir_init())
358 printk(KERN_INFO "Procfs failed to initialize topdir\n");
359 #endif /* IGB_PROCFS */
360 #endif /* IGB_SYSFS */
363 dca_register_notify(&dca_notifier);
365 ret = pci_register_driver(&igb_driver);
366 #ifdef USE_REBOOT_NOTIFIER
368 register_reboot_notifier(&igb_notifier_reboot);
375 #define module_init(x) static int x(void) __attribute__((__unused__));
376 module_init(igb_init_module);
379 * igb_exit_module - Driver Exit Cleanup Routine
381 * igb_exit_module is called just before the driver is removed
384 static void __exit igb_exit_module(void)
387 dca_unregister_notify(&dca_notifier);
389 #ifdef USE_REBOOT_NOTIFIER
390 unregister_reboot_notifier(&igb_notifier_reboot);
392 pci_unregister_driver(&igb_driver);
395 /* only compile IGB_PROCFS if IGB_SYSFS is not defined */
398 igb_procfs_topdir_exit();
399 #endif /* IGB_PROCFS */
400 #endif /* IGB_SYSFS */
404 #define module_exit(x) static void x(void) __attribute__((__unused__));
405 module_exit(igb_exit_module);
407 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
409 * igb_cache_ring_register - Descriptor ring to register mapping
410 * @adapter: board private structure to initialize
412 * Once we know the feature-set enabled for the device, we'll cache
413 * the register offset the descriptor ring is assigned to.
415 static void igb_cache_ring_register(struct igb_adapter *adapter)
418 u32 rbase_offset = adapter->vfs_allocated_count;
420 switch (adapter->hw.mac.type) {
422 /* The queues are allocated for virtualization such that VF 0
423 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
424 * In order to avoid collision we start at the first free queue
425 * and continue consuming queues in the same sequence
427 if ((adapter->rss_queues > 1) && adapter->vmdq_pools) {
428 for (; i < adapter->rss_queues; i++)
429 adapter->rx_ring[i]->reg_idx = rbase_offset +
436 for (; i < adapter->num_rx_queues; i++)
437 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
438 for (; j < adapter->num_tx_queues; j++)
439 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
444 static void igb_free_queues(struct igb_adapter *adapter)
448 for (i = 0; i < adapter->num_tx_queues; i++) {
449 kfree(adapter->tx_ring[i]);
450 adapter->tx_ring[i] = NULL;
452 for (i = 0; i < adapter->num_rx_queues; i++) {
453 kfree(adapter->rx_ring[i]);
454 adapter->rx_ring[i] = NULL;
456 adapter->num_rx_queues = 0;
457 adapter->num_tx_queues = 0;
461 * igb_alloc_queues - Allocate memory for all rings
462 * @adapter: board private structure to initialize
464 * We allocate one ring per queue at run-time since we don't know the
465 * number of queues at compile-time.
467 static int igb_alloc_queues(struct igb_adapter *adapter)
469 struct igb_ring *ring;
471 #ifdef HAVE_DEVICE_NUMA_NODE
472 int orig_node = adapter->node;
473 #endif /* HAVE_DEVICE_NUMA_NODE */
475 for (i = 0; i < adapter->num_tx_queues; i++) {
476 #ifdef HAVE_DEVICE_NUMA_NODE
477 if (orig_node == -1) {
478 int cur_node = next_online_node(adapter->node);
479 if (cur_node == MAX_NUMNODES)
480 cur_node = first_online_node;
481 adapter->node = cur_node;
483 #endif /* HAVE_DEVICE_NUMA_NODE */
484 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
487 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
490 ring->count = adapter->tx_ring_count;
491 ring->queue_index = i;
492 ring->dev = pci_dev_to_dev(adapter->pdev);
493 ring->netdev = adapter->netdev;
494 ring->numa_node = adapter->node;
495 /* For 82575, context index must be unique per ring. */
496 if (adapter->hw.mac.type == e1000_82575)
497 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
498 adapter->tx_ring[i] = ring;
500 #ifdef HAVE_DEVICE_NUMA_NODE
501 /* Restore the adapter's original node */
502 adapter->node = orig_node;
503 #endif /* HAVE_DEVICE_NUMA_NODE */
505 for (i = 0; i < adapter->num_rx_queues; i++) {
506 #ifdef HAVE_DEVICE_NUMA_NODE
507 if (orig_node == -1) {
508 int cur_node = next_online_node(adapter->node);
509 if (cur_node == MAX_NUMNODES)
510 cur_node = first_online_node;
511 adapter->node = cur_node;
513 #endif /* HAVE_DEVICE_NUMA_NODE */
514 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
517 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
520 ring->count = adapter->rx_ring_count;
521 ring->queue_index = i;
522 ring->dev = pci_dev_to_dev(adapter->pdev);
523 ring->netdev = adapter->netdev;
524 ring->numa_node = adapter->node;
525 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
526 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
528 #ifndef HAVE_NDO_SET_FEATURES
529 /* enable rx checksum */
530 set_bit(IGB_RING_FLAG_RX_CSUM, &ring->flags);
533 /* set flag indicating ring supports SCTP checksum offload */
534 if (adapter->hw.mac.type >= e1000_82576)
535 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
537 /* On i350, loopback VLAN packets have the tag byte-swapped. */
538 if (adapter->hw.mac.type == e1000_i350)
539 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
541 adapter->rx_ring[i] = ring;
543 #ifdef HAVE_DEVICE_NUMA_NODE
544 /* Restore the adapter's original node */
545 adapter->node = orig_node;
546 #endif /* HAVE_DEVICE_NUMA_NODE */
548 igb_cache_ring_register(adapter);
550 return E1000_SUCCESS;
553 #ifdef HAVE_DEVICE_NUMA_NODE
554 /* Restore the adapter's original node */
555 adapter->node = orig_node;
556 #endif /* HAVE_DEVICE_NUMA_NODE */
557 igb_free_queues(adapter);
562 static void igb_configure_lli(struct igb_adapter *adapter)
564 struct e1000_hw *hw = &adapter->hw;
567 /* LLI should only be enabled for MSI-X or MSI interrupts */
568 if (!adapter->msix_entries && !(adapter->flags & IGB_FLAG_HAS_MSI))
571 if (adapter->lli_port) {
572 /* use filter 0 for port */
573 port = htons((u16)adapter->lli_port);
574 E1000_WRITE_REG(hw, E1000_IMIR(0),
575 (port | E1000_IMIR_PORT_IM_EN));
576 E1000_WRITE_REG(hw, E1000_IMIREXT(0),
577 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
580 if (adapter->flags & IGB_FLAG_LLI_PUSH) {
581 /* use filter 1 for push flag */
582 E1000_WRITE_REG(hw, E1000_IMIR(1),
583 (E1000_IMIR_PORT_BP | E1000_IMIR_PORT_IM_EN));
584 E1000_WRITE_REG(hw, E1000_IMIREXT(1),
585 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_PSH));
588 if (adapter->lli_size) {
589 /* use filter 2 for size */
590 E1000_WRITE_REG(hw, E1000_IMIR(2),
591 (E1000_IMIR_PORT_BP | E1000_IMIR_PORT_IM_EN));
592 E1000_WRITE_REG(hw, E1000_IMIREXT(2),
593 (adapter->lli_size | E1000_IMIREXT_CTRL_BP));
599 * igb_write_ivar - configure ivar for given MSI-X vector
600 * @hw: pointer to the HW structure
601 * @msix_vector: vector number we are allocating to a given ring
602 * @index: row index of IVAR register to write within IVAR table
603 * @offset: column offset of in IVAR, should be multiple of 8
605 * This function is intended to handle the writing of the IVAR register
606 * for adapters 82576 and newer. The IVAR table consists of 2 columns,
607 * each containing an cause allocation for an Rx and Tx ring, and a
608 * variable number of rows depending on the number of queues supported.
610 static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
611 int index, int offset)
613 u32 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
615 /* clear any bits that are currently set */
616 ivar &= ~((u32)0xFF << offset);
618 /* write vector and valid bit */
619 ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
621 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
624 #define IGB_N0_QUEUE -1
625 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
627 struct igb_adapter *adapter = q_vector->adapter;
628 struct e1000_hw *hw = &adapter->hw;
629 int rx_queue = IGB_N0_QUEUE;
630 int tx_queue = IGB_N0_QUEUE;
633 if (q_vector->rx.ring)
634 rx_queue = q_vector->rx.ring->reg_idx;
635 if (q_vector->tx.ring)
636 tx_queue = q_vector->tx.ring->reg_idx;
638 switch (hw->mac.type) {
640 /* The 82575 assigns vectors using a bitmask, which matches the
641 bitmask for the EICR/EIMS/EIMC registers. To assign one
642 or more queues to a vector, we write the appropriate bits
643 into the MSIXBM register for that vector. */
644 if (rx_queue > IGB_N0_QUEUE)
645 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
646 if (tx_queue > IGB_N0_QUEUE)
647 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
648 if (!adapter->msix_entries && msix_vector == 0)
649 msixbm |= E1000_EIMS_OTHER;
650 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), msix_vector, msixbm);
651 q_vector->eims_value = msixbm;
655 * 82576 uses a table that essentially consists of 2 columns
656 * with 8 rows. The ordering is column-major so we use the
657 * lower 3 bits as the row index, and the 4th bit as the
660 if (rx_queue > IGB_N0_QUEUE)
661 igb_write_ivar(hw, msix_vector,
663 (rx_queue & 0x8) << 1);
664 if (tx_queue > IGB_N0_QUEUE)
665 igb_write_ivar(hw, msix_vector,
667 ((tx_queue & 0x8) << 1) + 8);
668 q_vector->eims_value = 1 << msix_vector;
673 * On 82580 and newer adapters the scheme is similar to 82576
674 * however instead of ordering column-major we have things
675 * ordered row-major. So we traverse the table by using
676 * bit 0 as the column offset, and the remaining bits as the
679 if (rx_queue > IGB_N0_QUEUE)
680 igb_write_ivar(hw, msix_vector,
682 (rx_queue & 0x1) << 4);
683 if (tx_queue > IGB_N0_QUEUE)
684 igb_write_ivar(hw, msix_vector,
686 ((tx_queue & 0x1) << 4) + 8);
687 q_vector->eims_value = 1 << msix_vector;
694 /* add q_vector eims value to global eims_enable_mask */
695 adapter->eims_enable_mask |= q_vector->eims_value;
697 /* configure q_vector to set itr on first interrupt */
698 q_vector->set_itr = 1;
702 * igb_configure_msix - Configure MSI-X hardware
704 * igb_configure_msix sets up the hardware to properly
705 * generate MSI-X interrupts.
707 static void igb_configure_msix(struct igb_adapter *adapter)
711 struct e1000_hw *hw = &adapter->hw;
713 adapter->eims_enable_mask = 0;
715 /* set vector for other causes, i.e. link changes */
716 switch (hw->mac.type) {
718 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
719 /* enable MSI-X PBA support*/
720 tmp |= E1000_CTRL_EXT_PBA_CLR;
722 /* Auto-Mask interrupts upon ICR read. */
723 tmp |= E1000_CTRL_EXT_EIAME;
724 tmp |= E1000_CTRL_EXT_IRCA;
726 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
728 /* enable msix_other interrupt */
729 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), vector++,
731 adapter->eims_other = E1000_EIMS_OTHER;
738 /* Turn on MSI-X capability first, or our settings
739 * won't stick. And it will take days to debug. */
740 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE |
741 E1000_GPIE_PBA | E1000_GPIE_EIAME |
744 /* enable msix_other interrupt */
745 adapter->eims_other = 1 << vector;
746 tmp = (vector++ | E1000_IVAR_VALID) << 8;
748 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmp);
751 /* do nothing, since nothing else supports MSI-X */
753 } /* switch (hw->mac.type) */
755 adapter->eims_enable_mask |= adapter->eims_other;
757 for (i = 0; i < adapter->num_q_vectors; i++)
758 igb_assign_vector(adapter->q_vector[i], vector++);
760 E1000_WRITE_FLUSH(hw);
764 * igb_request_msix - Initialize MSI-X interrupts
766 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
769 static int igb_request_msix(struct igb_adapter *adapter)
771 struct net_device *netdev = adapter->netdev;
772 struct e1000_hw *hw = &adapter->hw;
773 int i, err = 0, vector = 0;
775 err = request_irq(adapter->msix_entries[vector].vector,
776 &igb_msix_other, 0, netdev->name, adapter);
781 for (i = 0; i < adapter->num_q_vectors; i++) {
782 struct igb_q_vector *q_vector = adapter->q_vector[i];
784 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
786 if (q_vector->rx.ring && q_vector->tx.ring)
787 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
788 q_vector->rx.ring->queue_index);
789 else if (q_vector->tx.ring)
790 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
791 q_vector->tx.ring->queue_index);
792 else if (q_vector->rx.ring)
793 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
794 q_vector->rx.ring->queue_index);
796 sprintf(q_vector->name, "%s-unused", netdev->name);
798 err = request_irq(adapter->msix_entries[vector].vector,
799 igb_msix_ring, 0, q_vector->name,
806 igb_configure_msix(adapter);
812 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
814 if (adapter->msix_entries) {
815 pci_disable_msix(adapter->pdev);
816 kfree(adapter->msix_entries);
817 adapter->msix_entries = NULL;
818 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
819 pci_disable_msi(adapter->pdev);
824 * igb_free_q_vectors - Free memory allocated for interrupt vectors
825 * @adapter: board private structure to initialize
827 * This function frees the memory allocated to the q_vectors. In addition if
828 * NAPI is enabled it will delete any references to the NAPI struct prior
829 * to freeing the q_vector.
831 static void igb_free_q_vectors(struct igb_adapter *adapter)
835 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
836 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
837 adapter->q_vector[v_idx] = NULL;
840 netif_napi_del(&q_vector->napi);
842 if (q_vector->lrolist) {
843 __skb_queue_purge(&q_vector->lrolist->active);
844 vfree(q_vector->lrolist);
845 q_vector->lrolist = NULL;
850 adapter->num_q_vectors = 0;
854 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
856 * This function resets the device so that it has 0 rx queues, tx queues, and
857 * MSI-X interrupts allocated.
859 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
861 igb_free_queues(adapter);
862 igb_free_q_vectors(adapter);
863 igb_reset_interrupt_capability(adapter);
867 * igb_process_mdd_event
868 * @adapter - board private structure
870 * Identify a malicious VF, disable the VF TX/RX queues and log a message.
872 static void igb_process_mdd_event(struct igb_adapter *adapter)
874 struct e1000_hw *hw = &adapter->hw;
875 u32 lvmmc, vfte, vfre, mdfb;
878 lvmmc = E1000_READ_REG(hw, E1000_LVMMC);
879 vf_queue = lvmmc >> 29;
881 /* VF index cannot be bigger or equal to VFs allocated */
882 if (vf_queue >= adapter->vfs_allocated_count)
885 netdev_info(adapter->netdev,
886 "VF %d misbehaved. VF queues are disabled. "
887 "VM misbehavior code is 0x%x\n", vf_queue, lvmmc);
889 /* Disable VFTE and VFRE related bits */
890 vfte = E1000_READ_REG(hw, E1000_VFTE);
891 vfte &= ~(1 << vf_queue);
892 E1000_WRITE_REG(hw, E1000_VFTE, vfte);
894 vfre = E1000_READ_REG(hw, E1000_VFRE);
895 vfre &= ~(1 << vf_queue);
896 E1000_WRITE_REG(hw, E1000_VFRE, vfre);
898 /* Disable MDFB related bit */
899 mdfb = E1000_READ_REG(hw, E1000_MDFB);
900 mdfb &= ~(1 << vf_queue);
901 E1000_WRITE_REG(hw, E1000_MDFB, mdfb);
903 /* Reset the specific VF */
904 E1000_WRITE_REG(hw, E1000_VTCTRL(vf_queue), E1000_VTCTRL_RST);
909 * @adapter - board private structure
911 * Disable MDD behavior in the HW
913 static void igb_disable_mdd(struct igb_adapter *adapter)
915 struct e1000_hw *hw = &adapter->hw;
918 if (hw->mac.type != e1000_i350)
921 reg = E1000_READ_REG(hw, E1000_DTXCTL);
922 reg &= (~E1000_DTXCTL_MDP_EN);
923 E1000_WRITE_REG(hw, E1000_DTXCTL, reg);
928 * @adapter - board private structure
930 * Enable the HW to detect malicious driver and sends an interrupt to
933 * Only available on i350 device
935 static void igb_enable_mdd(struct igb_adapter *adapter)
937 struct e1000_hw *hw = &adapter->hw;
940 if (hw->mac.type != e1000_i350)
943 reg = E1000_READ_REG(hw, E1000_DTXCTL);
944 reg |= E1000_DTXCTL_MDP_EN;
945 E1000_WRITE_REG(hw, E1000_DTXCTL, reg);
949 * igb_reset_sriov_capability - disable SR-IOV if enabled
951 * Attempt to disable single root IO virtualization capabilites present in the
954 static void igb_reset_sriov_capability(struct igb_adapter *adapter)
956 struct pci_dev *pdev = adapter->pdev;
957 struct e1000_hw *hw = &adapter->hw;
959 /* reclaim resources allocated to VFs */
960 if (adapter->vf_data) {
961 if (!igb_check_vf_assignment(adapter)) {
963 * disable iov and allow time for transactions to
966 pci_disable_sriov(pdev);
969 dev_info(pci_dev_to_dev(pdev), "IOV Disabled\n");
971 dev_info(pci_dev_to_dev(pdev), "IOV Not Disabled\n "
972 "VF(s) are assigned to guests!\n");
974 /* Disable Malicious Driver Detection */
975 igb_disable_mdd(adapter);
977 /* free vf data storage */
978 kfree(adapter->vf_data);
979 adapter->vf_data = NULL;
981 /* switch rings back to PF ownership */
982 E1000_WRITE_REG(hw, E1000_IOVCTL,
983 E1000_IOVCTL_REUSE_VFQ);
984 E1000_WRITE_FLUSH(hw);
988 adapter->vfs_allocated_count = 0;
992 * igb_set_sriov_capability - setup SR-IOV if supported
994 * Attempt to enable single root IO virtualization capabilites present in the
997 static void igb_set_sriov_capability(struct igb_adapter *adapter)
999 struct pci_dev *pdev = adapter->pdev;
1003 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
1004 old_vfs = igb_find_enabled_vfs(adapter);
1007 dev_info(pci_dev_to_dev(pdev),
1008 "%d pre-allocated VFs found - override "
1009 "max_vfs setting of %d\n", old_vfs,
1010 adapter->vfs_allocated_count);
1011 adapter->vfs_allocated_count = old_vfs;
1013 /* no VFs requested, do nothing */
1014 if (!adapter->vfs_allocated_count)
1017 /* allocate vf data storage */
1018 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
1019 sizeof(struct vf_data_storage),
1022 if (adapter->vf_data) {
1024 if (pci_enable_sriov(pdev,
1025 adapter->vfs_allocated_count))
1028 for (i = 0; i < adapter->vfs_allocated_count; i++)
1029 igb_vf_configure(adapter, i);
1031 /* DMA Coalescing is not supported in IOV mode. */
1032 if (adapter->hw.mac.type >= e1000_i350)
1033 adapter->dmac = IGB_DMAC_DISABLE;
1034 if (adapter->hw.mac.type < e1000_i350)
1035 adapter->flags |= IGB_FLAG_DETECT_BAD_DMA;
1041 kfree(adapter->vf_data);
1042 adapter->vf_data = NULL;
1043 adapter->vfs_allocated_count = 0;
1044 dev_warn(pci_dev_to_dev(pdev),
1045 "Failed to initialize SR-IOV virtualization\n");
1049 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1051 * Attempt to configure interrupts using the best available
1052 * capabilities of the hardware and kernel.
1054 static void igb_set_interrupt_capability(struct igb_adapter *adapter)
1056 struct pci_dev *pdev = adapter->pdev;
1060 /* Number of supported queues. */
1061 adapter->num_rx_queues = adapter->rss_queues;
1063 if (adapter->vmdq_pools > 1)
1064 adapter->num_rx_queues += adapter->vmdq_pools - 1;
1067 if (adapter->vmdq_pools)
1068 adapter->num_tx_queues = adapter->vmdq_pools;
1070 adapter->num_tx_queues = adapter->num_rx_queues;
1072 adapter->num_tx_queues = max_t(u32, 1, adapter->vmdq_pools);
1075 switch (adapter->int_mode) {
1076 case IGB_INT_MODE_MSIX:
1077 /* start with one vector for every rx queue */
1078 numvecs = adapter->num_rx_queues;
1080 /* if tx handler is seperate add 1 for every tx queue */
1081 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1082 numvecs += adapter->num_tx_queues;
1084 /* store the number of vectors reserved for queues */
1085 adapter->num_q_vectors = numvecs;
1087 /* add 1 vector for link status interrupts */
1089 adapter->msix_entries = kcalloc(numvecs,
1090 sizeof(struct msix_entry),
1092 if (adapter->msix_entries) {
1093 for (i = 0; i < numvecs; i++)
1094 adapter->msix_entries[i].entry = i;
1096 err = pci_enable_msix(pdev,
1097 adapter->msix_entries, numvecs);
1101 /* MSI-X failed, so fall through and try MSI */
1102 dev_warn(pci_dev_to_dev(pdev), "Failed to initialize MSI-X interrupts. "
1103 "Falling back to MSI interrupts.\n");
1104 igb_reset_interrupt_capability(adapter);
1105 case IGB_INT_MODE_MSI:
1106 if (!pci_enable_msi(pdev))
1107 adapter->flags |= IGB_FLAG_HAS_MSI;
1109 dev_warn(pci_dev_to_dev(pdev), "Failed to initialize MSI "
1110 "interrupts. Falling back to legacy "
1113 case IGB_INT_MODE_LEGACY:
1114 /* disable advanced features and set number of queues to 1 */
1115 igb_reset_sriov_capability(adapter);
1116 adapter->vmdq_pools = 0;
1117 adapter->rss_queues = 1;
1118 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1119 adapter->num_rx_queues = 1;
1120 adapter->num_tx_queues = 1;
1121 adapter->num_q_vectors = 1;
1122 /* Don't do anything; this is system default */
1127 /* Notify the stack of the (possibly) reduced Tx Queue count. */
1128 #ifdef CONFIG_NETDEVICES_MULTIQUEUE
1129 adapter->netdev->egress_subqueue_count = adapter->num_tx_queues;
1131 adapter->netdev->real_num_tx_queues =
1132 (adapter->vmdq_pools ? 1 : adapter->num_tx_queues);
1133 #endif /* CONFIG_NETDEVICES_MULTIQUEUE */
1134 #endif /* HAVE_TX_MQ */
1138 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1139 * @adapter: board private structure to initialize
1141 * We allocate one q_vector per queue interrupt. If allocation fails we
1144 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1146 struct igb_q_vector *q_vector;
1147 struct e1000_hw *hw = &adapter->hw;
1149 #ifdef HAVE_DEVICE_NUMA_NODE
1150 int orig_node = adapter->node;
1151 #endif /* HAVE_DEVICE_NUMA_NODE */
1153 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1154 #ifdef HAVE_DEVICE_NUMA_NODE
1155 if ((adapter->num_q_vectors == (adapter->num_rx_queues +
1156 adapter->num_tx_queues)) &&
1157 (adapter->num_rx_queues == v_idx))
1158 adapter->node = orig_node;
1159 if (orig_node == -1) {
1160 int cur_node = next_online_node(adapter->node);
1161 if (cur_node == MAX_NUMNODES)
1162 cur_node = first_online_node;
1163 adapter->node = cur_node;
1165 #endif /* HAVE_DEVICE_NUMA_NODE */
1166 q_vector = kzalloc_node(sizeof(struct igb_q_vector), GFP_KERNEL,
1169 q_vector = kzalloc(sizeof(struct igb_q_vector),
1173 q_vector->adapter = adapter;
1174 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1175 q_vector->itr_val = IGB_START_ITR;
1176 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1177 adapter->q_vector[v_idx] = q_vector;
1179 if (v_idx < adapter->num_rx_queues) {
1180 int size = sizeof(struct igb_lro_list);
1181 q_vector->lrolist = vzalloc_node(size, q_vector->numa_node);
1182 if (!q_vector->lrolist)
1183 q_vector->lrolist = vzalloc(size);
1184 if (!q_vector->lrolist)
1186 __skb_queue_head_init(&q_vector->lrolist->active);
1188 #endif /* IGB_NO_LRO */
1190 #ifdef HAVE_DEVICE_NUMA_NODE
1191 /* Restore the adapter's original node */
1192 adapter->node = orig_node;
1193 #endif /* HAVE_DEVICE_NUMA_NODE */
1198 #ifdef HAVE_DEVICE_NUMA_NODE
1199 /* Restore the adapter's original node */
1200 adapter->node = orig_node;
1201 #endif /* HAVE_DEVICE_NUMA_NODE */
1202 igb_free_q_vectors(adapter);
1206 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1207 int ring_idx, int v_idx)
1209 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1211 q_vector->rx.ring = adapter->rx_ring[ring_idx];
1212 q_vector->rx.ring->q_vector = q_vector;
1213 q_vector->rx.count++;
1214 q_vector->itr_val = adapter->rx_itr_setting;
1215 if (q_vector->itr_val && q_vector->itr_val <= 3)
1216 q_vector->itr_val = IGB_START_ITR;
1219 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1220 int ring_idx, int v_idx)
1222 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1224 q_vector->tx.ring = adapter->tx_ring[ring_idx];
1225 q_vector->tx.ring->q_vector = q_vector;
1226 q_vector->tx.count++;
1227 q_vector->itr_val = adapter->tx_itr_setting;
1228 q_vector->tx.work_limit = adapter->tx_work_limit;
1229 if (q_vector->itr_val && q_vector->itr_val <= 3)
1230 q_vector->itr_val = IGB_START_ITR;
1234 * igb_map_ring_to_vector - maps allocated queues to vectors
1236 * This function maps the recently allocated queues to vectors.
1238 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1243 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1244 (adapter->num_q_vectors < adapter->num_tx_queues))
1247 if (adapter->num_q_vectors >=
1248 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1249 for (i = 0; i < adapter->num_rx_queues; i++)
1250 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1251 for (i = 0; i < adapter->num_tx_queues; i++)
1252 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1254 for (i = 0; i < adapter->num_rx_queues; i++) {
1255 if (i < adapter->num_tx_queues)
1256 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1257 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1259 for (; i < adapter->num_tx_queues; i++)
1260 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1266 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1268 * This function initializes the interrupts and allocates all of the queues.
1270 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1272 struct pci_dev *pdev = adapter->pdev;
1275 igb_set_interrupt_capability(adapter);
1277 err = igb_alloc_q_vectors(adapter);
1279 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for vectors\n");
1280 goto err_alloc_q_vectors;
1283 err = igb_alloc_queues(adapter);
1285 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for queues\n");
1286 goto err_alloc_queues;
1289 err = igb_map_ring_to_vector(adapter);
1291 dev_err(pci_dev_to_dev(pdev), "Invalid q_vector to ring mapping\n");
1292 goto err_map_queues;
1298 igb_free_queues(adapter);
1300 igb_free_q_vectors(adapter);
1301 err_alloc_q_vectors:
1302 igb_reset_interrupt_capability(adapter);
1307 * igb_request_irq - initialize interrupts
1309 * Attempts to configure interrupts using the best available
1310 * capabilities of the hardware and kernel.
1312 static int igb_request_irq(struct igb_adapter *adapter)
1314 struct net_device *netdev = adapter->netdev;
1315 struct pci_dev *pdev = adapter->pdev;
1318 if (adapter->msix_entries) {
1319 err = igb_request_msix(adapter);
1322 /* fall back to MSI */
1323 igb_clear_interrupt_scheme(adapter);
1324 igb_reset_sriov_capability(adapter);
1325 if (!pci_enable_msi(pdev))
1326 adapter->flags |= IGB_FLAG_HAS_MSI;
1327 igb_free_all_tx_resources(adapter);
1328 igb_free_all_rx_resources(adapter);
1329 adapter->num_tx_queues = 1;
1330 adapter->num_rx_queues = 1;
1331 adapter->num_q_vectors = 1;
1332 err = igb_alloc_q_vectors(adapter);
1334 dev_err(pci_dev_to_dev(pdev),
1335 "Unable to allocate memory for vectors\n");
1338 err = igb_alloc_queues(adapter);
1340 dev_err(pci_dev_to_dev(pdev),
1341 "Unable to allocate memory for queues\n");
1342 igb_free_q_vectors(adapter);
1345 igb_setup_all_tx_resources(adapter);
1346 igb_setup_all_rx_resources(adapter);
1349 igb_assign_vector(adapter->q_vector[0], 0);
1351 if (adapter->flags & IGB_FLAG_HAS_MSI) {
1352 err = request_irq(pdev->irq, &igb_intr_msi, 0,
1353 netdev->name, adapter);
1357 /* fall back to legacy interrupts */
1358 igb_reset_interrupt_capability(adapter);
1359 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1362 err = request_irq(pdev->irq, &igb_intr, IRQF_SHARED,
1363 netdev->name, adapter);
1366 dev_err(pci_dev_to_dev(pdev), "Error %d getting interrupt\n",
1373 static void igb_free_irq(struct igb_adapter *adapter)
1375 if (adapter->msix_entries) {
1378 free_irq(adapter->msix_entries[vector++].vector, adapter);
1380 for (i = 0; i < adapter->num_q_vectors; i++)
1381 free_irq(adapter->msix_entries[vector++].vector,
1382 adapter->q_vector[i]);
1384 free_irq(adapter->pdev->irq, adapter);
1389 * igb_irq_disable - Mask off interrupt generation on the NIC
1390 * @adapter: board private structure
1392 static void igb_irq_disable(struct igb_adapter *adapter)
1394 struct e1000_hw *hw = &adapter->hw;
1397 * we need to be careful when disabling interrupts. The VFs are also
1398 * mapped into these registers and so clearing the bits can cause
1399 * issues on the VF drivers so we only need to clear what we set
1401 if (adapter->msix_entries) {
1402 u32 regval = E1000_READ_REG(hw, E1000_EIAM);
1403 E1000_WRITE_REG(hw, E1000_EIAM, regval & ~adapter->eims_enable_mask);
1404 E1000_WRITE_REG(hw, E1000_EIMC, adapter->eims_enable_mask);
1405 regval = E1000_READ_REG(hw, E1000_EIAC);
1406 E1000_WRITE_REG(hw, E1000_EIAC, regval & ~adapter->eims_enable_mask);
1409 E1000_WRITE_REG(hw, E1000_IAM, 0);
1410 E1000_WRITE_REG(hw, E1000_IMC, ~0);
1411 E1000_WRITE_FLUSH(hw);
1413 if (adapter->msix_entries) {
1416 synchronize_irq(adapter->msix_entries[vector++].vector);
1418 for (i = 0; i < adapter->num_q_vectors; i++)
1419 synchronize_irq(adapter->msix_entries[vector++].vector);
1421 synchronize_irq(adapter->pdev->irq);
1426 * igb_irq_enable - Enable default interrupt generation settings
1427 * @adapter: board private structure
1429 static void igb_irq_enable(struct igb_adapter *adapter)
1431 struct e1000_hw *hw = &adapter->hw;
1433 if (adapter->msix_entries) {
1434 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1435 u32 regval = E1000_READ_REG(hw, E1000_EIAC);
1436 E1000_WRITE_REG(hw, E1000_EIAC, regval | adapter->eims_enable_mask);
1437 regval = E1000_READ_REG(hw, E1000_EIAM);
1438 E1000_WRITE_REG(hw, E1000_EIAM, regval | adapter->eims_enable_mask);
1439 E1000_WRITE_REG(hw, E1000_EIMS, adapter->eims_enable_mask);
1440 if (adapter->vfs_allocated_count) {
1441 E1000_WRITE_REG(hw, E1000_MBVFIMR, 0xFF);
1442 ims |= E1000_IMS_VMMB;
1443 /* For I350 device only enable MDD interrupts*/
1444 if ((adapter->mdd) &&
1445 (adapter->hw.mac.type == e1000_i350))
1446 ims |= E1000_IMS_MDDET;
1448 E1000_WRITE_REG(hw, E1000_IMS, ims);
1450 E1000_WRITE_REG(hw, E1000_IMS, IMS_ENABLE_MASK |
1452 E1000_WRITE_REG(hw, E1000_IAM, IMS_ENABLE_MASK |
1457 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1459 struct e1000_hw *hw = &adapter->hw;
1460 u16 vid = adapter->hw.mng_cookie.vlan_id;
1461 u16 old_vid = adapter->mng_vlan_id;
1463 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1464 /* add VID to filter table */
1465 igb_vfta_set(adapter, vid, TRUE);
1466 adapter->mng_vlan_id = vid;
1468 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1471 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1473 #ifdef HAVE_VLAN_RX_REGISTER
1474 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1476 !test_bit(old_vid, adapter->active_vlans)) {
1478 /* remove VID from filter table */
1479 igb_vfta_set(adapter, old_vid, FALSE);
1484 * igb_release_hw_control - release control of the h/w to f/w
1485 * @adapter: address of board private structure
1487 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1488 * For ASF and Pass Through versions of f/w this means that the
1489 * driver is no longer loaded.
1492 static void igb_release_hw_control(struct igb_adapter *adapter)
1494 struct e1000_hw *hw = &adapter->hw;
1497 /* Let firmware take over control of h/w */
1498 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1499 E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1500 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1504 * igb_get_hw_control - get control of the h/w from f/w
1505 * @adapter: address of board private structure
1507 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1508 * For ASF and Pass Through versions of f/w this means that
1509 * the driver is loaded.
1512 static void igb_get_hw_control(struct igb_adapter *adapter)
1514 struct e1000_hw *hw = &adapter->hw;
1517 /* Let firmware know the driver has taken over */
1518 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1519 E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1520 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1524 * igb_configure - configure the hardware for RX and TX
1525 * @adapter: private board structure
1527 static void igb_configure(struct igb_adapter *adapter)
1529 struct net_device *netdev = adapter->netdev;
1532 igb_get_hw_control(adapter);
1533 igb_set_rx_mode(netdev);
1535 igb_restore_vlan(adapter);
1537 igb_setup_tctl(adapter);
1538 igb_setup_mrqc(adapter);
1539 igb_setup_rctl(adapter);
1541 igb_configure_tx(adapter);
1542 igb_configure_rx(adapter);
1544 e1000_rx_fifo_flush_82575(&adapter->hw);
1545 #ifdef CONFIG_NETDEVICES_MULTIQUEUE
1547 if (adapter->num_tx_queues > 1)
1548 netdev->features |= NETIF_F_MULTI_QUEUE;
1550 netdev->features &= ~NETIF_F_MULTI_QUEUE;
1553 /* call igb_desc_unused which always leaves
1554 * at least 1 descriptor unused to make sure
1555 * next_to_use != next_to_clean */
1556 for (i = 0; i < adapter->num_rx_queues; i++) {
1557 struct igb_ring *ring = adapter->rx_ring[i];
1558 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
1563 * igb_power_up_link - Power up the phy/serdes link
1564 * @adapter: address of board private structure
1566 void igb_power_up_link(struct igb_adapter *adapter)
1568 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1569 e1000_power_up_phy(&adapter->hw);
1571 e1000_power_up_fiber_serdes_link(&adapter->hw);
1573 e1000_phy_hw_reset(&adapter->hw);
1577 * igb_power_down_link - Power down the phy/serdes link
1578 * @adapter: address of board private structure
1580 static void igb_power_down_link(struct igb_adapter *adapter)
1582 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1583 e1000_power_down_phy(&adapter->hw);
1585 e1000_shutdown_fiber_serdes_link(&adapter->hw);
1589 * igb_up - Open the interface and prepare it to handle traffic
1590 * @adapter: board private structure
1592 int igb_up(struct igb_adapter *adapter)
1594 struct e1000_hw *hw = &adapter->hw;
1597 /* hardware has been reset, we need to reload some things */
1598 igb_configure(adapter);
1600 clear_bit(__IGB_DOWN, &adapter->state);
1602 for (i = 0; i < adapter->num_q_vectors; i++)
1603 napi_enable(&(adapter->q_vector[i]->napi));
1605 if (adapter->msix_entries)
1606 igb_configure_msix(adapter);
1608 igb_assign_vector(adapter->q_vector[0], 0);
1610 igb_configure_lli(adapter);
1612 /* Clear any pending interrupts. */
1613 E1000_READ_REG(hw, E1000_ICR);
1614 igb_irq_enable(adapter);
1616 /* notify VFs that reset has been completed */
1617 if (adapter->vfs_allocated_count) {
1618 u32 reg_data = E1000_READ_REG(hw, E1000_CTRL_EXT);
1619 reg_data |= E1000_CTRL_EXT_PFRSTD;
1620 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg_data);
1623 netif_tx_start_all_queues(adapter->netdev);
1625 if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
1626 schedule_work(&adapter->dma_err_task);
1627 /* start the watchdog. */
1628 hw->mac.get_link_status = 1;
1629 schedule_work(&adapter->watchdog_task);
1634 void igb_down(struct igb_adapter *adapter)
1636 struct net_device *netdev = adapter->netdev;
1637 struct e1000_hw *hw = &adapter->hw;
1641 /* signal that we're down so the interrupt handler does not
1642 * reschedule our watchdog timer */
1643 set_bit(__IGB_DOWN, &adapter->state);
1645 /* disable receives in the hardware */
1646 rctl = E1000_READ_REG(hw, E1000_RCTL);
1647 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
1648 /* flush and sleep below */
1650 netif_tx_stop_all_queues(netdev);
1652 /* disable transmits in the hardware */
1653 tctl = E1000_READ_REG(hw, E1000_TCTL);
1654 tctl &= ~E1000_TCTL_EN;
1655 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
1656 /* flush both disables and wait for them to finish */
1657 E1000_WRITE_FLUSH(hw);
1658 usleep_range(10000, 20000);
1660 for (i = 0; i < adapter->num_q_vectors; i++)
1661 napi_disable(&(adapter->q_vector[i]->napi));
1663 igb_irq_disable(adapter);
1665 del_timer_sync(&adapter->watchdog_timer);
1666 if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
1667 del_timer_sync(&adapter->dma_err_timer);
1668 del_timer_sync(&adapter->phy_info_timer);
1670 netif_carrier_off(netdev);
1672 /* record the stats before reset*/
1673 igb_update_stats(adapter);
1675 adapter->link_speed = 0;
1676 adapter->link_duplex = 0;
1679 if (!pci_channel_offline(adapter->pdev))
1684 igb_clean_all_tx_rings(adapter);
1685 igb_clean_all_rx_rings(adapter);
1688 /* since we reset the hardware DCA settings were cleared */
1689 igb_setup_dca(adapter);
1693 void igb_reinit_locked(struct igb_adapter *adapter)
1695 WARN_ON(in_interrupt());
1696 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1697 usleep_range(1000, 2000);
1700 clear_bit(__IGB_RESETTING, &adapter->state);
1703 void igb_reset(struct igb_adapter *adapter)
1705 struct pci_dev *pdev = adapter->pdev;
1706 struct e1000_hw *hw = &adapter->hw;
1707 struct e1000_mac_info *mac = &hw->mac;
1708 struct e1000_fc_info *fc = &hw->fc;
1709 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1712 /* Repartition Pba for greater than 9k mtu
1713 * To take effect CTRL.RST is required.
1715 switch (mac->type) {
1718 pba = E1000_READ_REG(hw, E1000_RXPBS);
1719 pba = e1000_rxpbs_adjust_82580(pba);
1722 pba = E1000_READ_REG(hw, E1000_RXPBS);
1723 pba &= E1000_RXPBS_SIZE_MASK_82576;
1727 pba = E1000_PBA_34K;
1731 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1732 (mac->type < e1000_82576)) {
1733 /* adjust PBA for jumbo frames */
1734 E1000_WRITE_REG(hw, E1000_PBA, pba);
1736 /* To maintain wire speed transmits, the Tx FIFO should be
1737 * large enough to accommodate two full transmit packets,
1738 * rounded up to the next 1KB and expressed in KB. Likewise,
1739 * the Rx FIFO should be large enough to accommodate at least
1740 * one full receive packet and is similarly rounded up and
1741 * expressed in KB. */
1742 pba = E1000_READ_REG(hw, E1000_PBA);
1743 /* upper 16 bits has Tx packet buffer allocation size in KB */
1744 tx_space = pba >> 16;
1745 /* lower 16 bits has Rx packet buffer allocation size in KB */
1747 /* the tx fifo also stores 16 bytes of information about the tx
1748 * but don't include ethernet FCS because hardware appends it */
1749 min_tx_space = (adapter->max_frame_size +
1750 sizeof(union e1000_adv_tx_desc) -
1752 min_tx_space = ALIGN(min_tx_space, 1024);
1753 min_tx_space >>= 10;
1754 /* software strips receive CRC, so leave room for it */
1755 min_rx_space = adapter->max_frame_size;
1756 min_rx_space = ALIGN(min_rx_space, 1024);
1757 min_rx_space >>= 10;
1759 /* If current Tx allocation is less than the min Tx FIFO size,
1760 * and the min Tx FIFO size is less than the current Rx FIFO
1761 * allocation, take space away from current Rx allocation */
1762 if (tx_space < min_tx_space &&
1763 ((min_tx_space - tx_space) < pba)) {
1764 pba = pba - (min_tx_space - tx_space);
1766 /* if short on rx space, rx wins and must trump tx
1768 if (pba < min_rx_space)
1771 E1000_WRITE_REG(hw, E1000_PBA, pba);
1774 /* flow control settings */
1775 /* The high water mark must be low enough to fit one full frame
1776 * (or the size used for early receive) above it in the Rx FIFO.
1777 * Set it to the lower of:
1778 * - 90% of the Rx FIFO size, or
1779 * - the full Rx FIFO size minus one full frame */
1780 hwm = min(((pba << 10) * 9 / 10),
1781 ((pba << 10) - 2 * adapter->max_frame_size));
1783 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1784 fc->low_water = fc->high_water - 16;
1785 fc->pause_time = 0xFFFF;
1787 fc->current_mode = fc->requested_mode;
1789 /* disable receive for all VFs and wait one second */
1790 if (adapter->vfs_allocated_count) {
1793 * Clear all flags except indication that the PF has set
1794 * the VF MAC addresses administratively
1796 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1797 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
1799 /* ping all the active vfs to let them know we are going down */
1800 igb_ping_all_vfs(adapter);
1802 /* disable transmits and receives */
1803 E1000_WRITE_REG(hw, E1000_VFRE, 0);
1804 E1000_WRITE_REG(hw, E1000_VFTE, 0);
1807 /* Allow time for pending master requests to run */
1809 E1000_WRITE_REG(hw, E1000_WUC, 0);
1811 if (e1000_init_hw(hw))
1812 dev_err(pci_dev_to_dev(pdev), "Hardware Error\n");
1814 igb_init_dmac(adapter, pba);
1815 /* Re-initialize the thermal sensor on i350 devices. */
1816 if (mac->type == e1000_i350 && hw->bus.func == 0) {
1818 * If present, re-initialize the external thermal sensor
1822 e1000_set_i2c_bb(hw);
1823 e1000_init_thermal_sensor_thresh(hw);
1825 if (!netif_running(adapter->netdev))
1826 igb_power_down_link(adapter);
1828 igb_update_mng_vlan(adapter);
1830 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1831 E1000_WRITE_REG(hw, E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1833 e1000_get_phy_info(hw);
1836 #ifdef HAVE_NDO_SET_FEATURES
1837 static netdev_features_t igb_fix_features(struct net_device *netdev,
1838 netdev_features_t features)
1841 * Since there is no support for separate tx vlan accel
1842 * enabled make sure tx flag is cleared if rx is.
1844 if (!(features & NETIF_F_HW_VLAN_RX))
1845 features &= ~NETIF_F_HW_VLAN_TX;
1847 /* If Rx checksum is disabled, then LRO should also be disabled */
1848 if (!(features & NETIF_F_RXCSUM))
1849 features &= ~NETIF_F_LRO;
1854 static int igb_set_features(struct net_device *netdev,
1855 netdev_features_t features)
1857 u32 changed = netdev->features ^ features;
1859 if (changed & NETIF_F_HW_VLAN_RX)
1860 igb_vlan_mode(netdev, features);
1865 #endif /* HAVE_NDO_SET_FEATURES */
1866 #ifdef HAVE_NET_DEVICE_OPS
1867 static const struct net_device_ops igb_netdev_ops = {
1868 .ndo_open = igb_open,
1869 .ndo_stop = igb_close,
1870 .ndo_start_xmit = igb_xmit_frame,
1871 .ndo_get_stats = igb_get_stats,
1872 .ndo_set_rx_mode = igb_set_rx_mode,
1873 .ndo_set_mac_address = igb_set_mac,
1874 .ndo_change_mtu = igb_change_mtu,
1875 .ndo_do_ioctl = igb_ioctl,
1876 .ndo_tx_timeout = igb_tx_timeout,
1877 .ndo_validate_addr = eth_validate_addr,
1878 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1879 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1881 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1882 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1883 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1884 .ndo_get_vf_config = igb_ndo_get_vf_config,
1886 #ifdef CONFIG_NET_POLL_CONTROLLER
1887 .ndo_poll_controller = igb_netpoll,
1889 #ifdef HAVE_NDO_SET_FEATURES
1890 .ndo_fix_features = igb_fix_features,
1891 .ndo_set_features = igb_set_features,
1893 #ifdef HAVE_VLAN_RX_REGISTER
1894 .ndo_vlan_rx_register = igb_vlan_mode,
1898 #ifdef CONFIG_IGB_VMDQ_NETDEV
1899 static const struct net_device_ops igb_vmdq_ops = {
1900 .ndo_open = &igb_vmdq_open,
1901 .ndo_stop = &igb_vmdq_close,
1902 .ndo_start_xmit = &igb_vmdq_xmit_frame,
1903 .ndo_get_stats = &igb_vmdq_get_stats,
1904 .ndo_set_rx_mode = &igb_vmdq_set_rx_mode,
1905 .ndo_validate_addr = eth_validate_addr,
1906 .ndo_set_mac_address = &igb_vmdq_set_mac,
1907 .ndo_change_mtu = &igb_vmdq_change_mtu,
1908 .ndo_tx_timeout = &igb_vmdq_tx_timeout,
1909 .ndo_vlan_rx_register = &igb_vmdq_vlan_rx_register,
1910 .ndo_vlan_rx_add_vid = &igb_vmdq_vlan_rx_add_vid,
1911 .ndo_vlan_rx_kill_vid = &igb_vmdq_vlan_rx_kill_vid,
1914 #endif /* CONFIG_IGB_VMDQ_NETDEV */
1915 #endif /* HAVE_NET_DEVICE_OPS */
1916 #ifdef CONFIG_IGB_VMDQ_NETDEV
1917 void igb_assign_vmdq_netdev_ops(struct net_device *vnetdev)
1919 #ifdef HAVE_NET_DEVICE_OPS
1920 vnetdev->netdev_ops = &igb_vmdq_ops;
1922 dev->open = &igb_vmdq_open;
1923 dev->stop = &igb_vmdq_close;
1924 dev->hard_start_xmit = &igb_vmdq_xmit_frame;
1925 dev->get_stats = &igb_vmdq_get_stats;
1926 #ifdef HAVE_SET_RX_MODE
1927 dev->set_rx_mode = &igb_vmdq_set_rx_mode;
1929 dev->set_multicast_list = &igb_vmdq_set_rx_mode;
1930 dev->set_mac_address = &igb_vmdq_set_mac;
1931 dev->change_mtu = &igb_vmdq_change_mtu;
1932 #ifdef HAVE_TX_TIMEOUT
1933 dev->tx_timeout = &igb_vmdq_tx_timeout;
1935 #ifdef NETIF_F_HW_VLAN_TX
1936 dev->vlan_rx_register = &igb_vmdq_vlan_rx_register;
1937 dev->vlan_rx_add_vid = &igb_vmdq_vlan_rx_add_vid;
1938 dev->vlan_rx_kill_vid = &igb_vmdq_vlan_rx_kill_vid;
1941 igb_vmdq_set_ethtool_ops(vnetdev);
1942 vnetdev->watchdog_timeo = 5 * HZ;
1946 int igb_init_vmdq_netdevs(struct igb_adapter *adapter)
1948 int pool, err = 0, base_queue;
1949 struct net_device *vnetdev;
1950 struct igb_vmdq_adapter *vmdq_adapter;
1952 for (pool = 1; pool < adapter->vmdq_pools; pool++) {
1953 int qpp = (!adapter->rss_queues ? 1 : adapter->rss_queues);
1954 base_queue = pool * qpp;
1955 vnetdev = alloc_etherdev(sizeof(struct igb_vmdq_adapter));
1960 vmdq_adapter = netdev_priv(vnetdev);
1961 vmdq_adapter->vnetdev = vnetdev;
1962 vmdq_adapter->real_adapter = adapter;
1963 vmdq_adapter->rx_ring = adapter->rx_ring[base_queue];
1964 vmdq_adapter->tx_ring = adapter->tx_ring[base_queue];
1965 igb_assign_vmdq_netdev_ops(vnetdev);
1966 snprintf(vnetdev->name, IFNAMSIZ, "%sv%d",
1967 adapter->netdev->name, pool);
1968 vnetdev->features = adapter->netdev->features;
1969 #ifdef HAVE_NETDEV_VLAN_FEATURES
1970 vnetdev->vlan_features = adapter->netdev->vlan_features;
1972 adapter->vmdq_netdev[pool-1] = vnetdev;
1973 err = register_netdev(vnetdev);
1980 int igb_remove_vmdq_netdevs(struct igb_adapter *adapter)
1984 for (pool = 1; pool < adapter->vmdq_pools; pool++) {
1985 unregister_netdev(adapter->vmdq_netdev[pool-1]);
1986 free_netdev(adapter->vmdq_netdev[pool-1]);
1987 adapter->vmdq_netdev[pool-1] = NULL;
1991 #endif /* CONFIG_IGB_VMDQ_NETDEV */
1994 * igb_probe - Device Initialization Routine
1995 * @pdev: PCI device information struct
1996 * @ent: entry in igb_pci_tbl
1998 * Returns 0 on success, negative on failure
2000 * igb_probe initializes an adapter identified by a pci_dev structure.
2001 * The OS initialization, configuring of the adapter private structure,
2002 * and a hardware reset occur.
2004 static int __devinit igb_probe(struct pci_dev *pdev,
2005 const struct pci_device_id *ent)
2007 struct net_device *netdev;
2008 struct igb_adapter *adapter;
2009 struct e1000_hw *hw;
2010 u16 eeprom_data = 0;
2011 u8 pba_str[E1000_PBANUM_LENGTH];
2013 static int global_quad_port_a; /* global quad port a indication */
2014 int i, err, pci_using_dac;
2015 static int cards_found;
2017 err = pci_enable_device_mem(pdev);
2022 err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
2024 err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
2028 err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
2030 err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
2032 IGB_ERR("No usable DMA configuration, "
2039 #ifndef HAVE_ASPM_QUIRKS
2040 /* 82575 requires that the pci-e link partner disable the L0s state */
2041 switch (pdev->device) {
2042 case E1000_DEV_ID_82575EB_COPPER:
2043 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2044 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2045 pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S);
2050 #endif /* HAVE_ASPM_QUIRKS */
2051 err = pci_request_selected_regions(pdev,
2052 pci_select_bars(pdev,
2058 pci_enable_pcie_error_reporting(pdev);
2060 pci_set_master(pdev);
2064 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
2067 netdev = alloc_etherdev(sizeof(struct igb_adapter));
2068 #endif /* HAVE_TX_MQ */
2070 goto err_alloc_etherdev;
2072 SET_MODULE_OWNER(netdev);
2073 SET_NETDEV_DEV(netdev, &pdev->dev);
2075 pci_set_drvdata(pdev, netdev);
2076 adapter = netdev_priv(netdev);
2077 adapter->netdev = netdev;
2078 adapter->pdev = pdev;
2081 adapter->port_num = hw->bus.func;
2082 adapter->msg_enable = (1 << debug) - 1;
2085 err = pci_save_state(pdev);
2090 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
2091 pci_resource_len(pdev, 0));
2095 #ifdef HAVE_NET_DEVICE_OPS
2096 netdev->netdev_ops = &igb_netdev_ops;
2097 #else /* HAVE_NET_DEVICE_OPS */
2098 netdev->open = &igb_open;
2099 netdev->stop = &igb_close;
2100 netdev->get_stats = &igb_get_stats;
2101 #ifdef HAVE_SET_RX_MODE
2102 netdev->set_rx_mode = &igb_set_rx_mode;
2104 netdev->set_multicast_list = &igb_set_rx_mode;
2105 netdev->set_mac_address = &igb_set_mac;
2106 netdev->change_mtu = &igb_change_mtu;
2107 netdev->do_ioctl = &igb_ioctl;
2108 #ifdef HAVE_TX_TIMEOUT
2109 netdev->tx_timeout = &igb_tx_timeout;
2111 netdev->vlan_rx_register = igb_vlan_mode;
2112 netdev->vlan_rx_add_vid = igb_vlan_rx_add_vid;
2113 netdev->vlan_rx_kill_vid = igb_vlan_rx_kill_vid;
2114 #ifdef CONFIG_NET_POLL_CONTROLLER
2115 netdev->poll_controller = igb_netpoll;
2117 netdev->hard_start_xmit = &igb_xmit_frame;
2118 #endif /* HAVE_NET_DEVICE_OPS */
2119 igb_set_ethtool_ops(netdev);
2120 #ifdef HAVE_TX_TIMEOUT
2121 netdev->watchdog_timeo = 5 * HZ;
2124 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
2126 adapter->bd_number = cards_found;
2128 /* setup the private structure */
2129 err = igb_sw_init(adapter);
2133 e1000_get_bus_info(hw);
2135 hw->phy.autoneg_wait_to_complete = FALSE;
2136 hw->mac.adaptive_ifs = FALSE;
2138 /* Copper options */
2139 if (hw->phy.media_type == e1000_media_type_copper) {
2141 hw->phy.mdix = ETH_TP_MDI_INVALID;
2143 hw->phy.mdix = AUTO_ALL_MODES;
2144 #endif /* ETH_TP_MDI_X */
2145 hw->phy.disable_polarity_correction = FALSE;
2146 hw->phy.ms_type = e1000_ms_hw_default;
2149 if (e1000_check_reset_block(hw))
2150 dev_info(pci_dev_to_dev(pdev),
2151 "PHY reset is blocked due to SOL/IDER session.\n");
2154 * features is initialized to 0 in allocation, it might have bits
2155 * set by igb_sw_init so we should use an or instead of an
2158 netdev->features |= NETIF_F_SG |
2160 #ifdef NETIF_F_IPV6_CSUM
2168 #endif /* NETIF_F_TSO */
2169 #ifdef NETIF_F_RXHASH
2172 #ifdef HAVE_NDO_SET_FEATURES
2175 NETIF_F_HW_VLAN_RX |
2178 #ifdef HAVE_NDO_SET_FEATURES
2179 /* copy netdev features into list of user selectable features */
2180 netdev->hw_features |= netdev->features;
2183 /* give us the option of enabling LRO later */
2184 netdev->hw_features |= NETIF_F_LRO;
2189 /* this is only needed on kernels prior to 2.6.39 */
2190 netdev->features |= NETIF_F_GRO;
2194 /* set this bit last since it cannot be part of hw_features */
2195 netdev->features |= NETIF_F_HW_VLAN_FILTER;
2197 #ifdef HAVE_NETDEV_VLAN_FEATURES
2198 netdev->vlan_features |= NETIF_F_TSO |
2206 netdev->features |= NETIF_F_HIGHDMA;
2208 if (hw->mac.type >= e1000_82576)
2209 netdev->features |= NETIF_F_SCTP_CSUM;
2211 adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
2213 /* before reading the NVM, reset the controller to put the device in a
2214 * known good starting state */
2217 /* make sure the NVM is good */
2218 if (e1000_validate_nvm_checksum(hw) < 0) {
2219 dev_err(pci_dev_to_dev(pdev), "The NVM Checksum Is Not"
2225 /* copy the MAC address out of the NVM */
2226 if (e1000_read_mac_addr(hw))
2227 dev_err(pci_dev_to_dev(pdev), "NVM Read Error\n");
2228 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2229 #ifdef ETHTOOL_GPERMADDR
2230 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
2232 if (!is_valid_ether_addr(netdev->perm_addr)) {
2234 if (!is_valid_ether_addr(netdev->dev_addr)) {
2236 dev_err(pci_dev_to_dev(pdev), "Invalid MAC Address\n");
2241 memcpy(&adapter->mac_table[0].addr, hw->mac.addr, netdev->addr_len);
2242 adapter->mac_table[0].queue = adapter->vfs_allocated_count;
2243 adapter->mac_table[0].state = (IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE);
2244 igb_rar_set(adapter, 0);
2246 /* get firmware version for ethtool -i */
2247 e1000_read_nvm(&adapter->hw, 5, 1, &adapter->fw_version);
2248 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
2249 (unsigned long) adapter);
2250 if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2251 setup_timer(&adapter->dma_err_timer, &igb_dma_err_timer,
2252 (unsigned long) adapter);
2253 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
2254 (unsigned long) adapter);
2256 INIT_WORK(&adapter->reset_task, igb_reset_task);
2257 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2258 if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2259 INIT_WORK(&adapter->dma_err_task, igb_dma_err_task);
2261 /* Initialize link properties that are user-changeable */
2262 adapter->fc_autoneg = true;
2263 hw->mac.autoneg = true;
2264 hw->phy.autoneg_advertised = 0x2f;
2266 hw->fc.requested_mode = e1000_fc_default;
2267 hw->fc.current_mode = e1000_fc_default;
2269 e1000_validate_mdi_setting(hw);
2271 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2272 * enable the ACPI Magic Packet filter
2275 if (hw->bus.func == 0)
2276 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
2277 else if (hw->mac.type >= e1000_82580)
2278 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2279 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2281 else if (hw->bus.func == 1)
2282 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
2284 if (eeprom_data & IGB_EEPROM_APME)
2285 adapter->eeprom_wol |= E1000_WUFC_MAG;
2287 /* now that we have the eeprom settings, apply the special cases where
2288 * the eeprom may be wrong or the board simply won't support wake on
2289 * lan on a particular port */
2290 switch (pdev->device) {
2291 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2292 adapter->eeprom_wol = 0;
2294 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2295 case E1000_DEV_ID_82576_FIBER:
2296 case E1000_DEV_ID_82576_SERDES:
2297 /* Wake events only supported on port A for dual fiber
2298 * regardless of eeprom setting */
2299 if (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_FUNC_1)
2300 adapter->eeprom_wol = 0;
2302 case E1000_DEV_ID_82576_QUAD_COPPER:
2303 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
2304 /* if quad port adapter, disable WoL on all but port A */
2305 if (global_quad_port_a != 0)
2306 adapter->eeprom_wol = 0;
2308 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2309 /* Reset for multiple quad port adapters */
2310 if (++global_quad_port_a == 4)
2311 global_quad_port_a = 0;
2315 /* initialize the wol settings based on the eeprom settings */
2316 adapter->wol = adapter->eeprom_wol;
2317 device_set_wakeup_enable(pci_dev_to_dev(adapter->pdev), adapter->wol);
2319 /* reset the hardware with the new settings */
2322 /* let the f/w know that the h/w is now under the control of the
2324 igb_get_hw_control(adapter);
2326 strncpy(netdev->name, "eth%d", IFNAMSIZ);
2327 err = register_netdev(netdev);
2331 #ifdef CONFIG_IGB_VMDQ_NETDEV
2332 err = igb_init_vmdq_netdevs(adapter);
2336 /* carrier off reporting is important to ethtool even BEFORE open */
2337 netif_carrier_off(netdev);
2340 if (dca_add_requester(&pdev->dev) == E1000_SUCCESS) {
2341 adapter->flags |= IGB_FLAG_DCA_ENABLED;
2342 dev_info(pci_dev_to_dev(pdev), "DCA enabled\n");
2343 igb_setup_dca(adapter);
2347 #ifdef HAVE_HW_TIME_STAMP
2348 /* do hw tstamp init after resetting */
2349 igb_init_hw_timer(adapter);
2352 dev_info(pci_dev_to_dev(pdev), "Intel(R) Gigabit Ethernet Network Connection\n");
2353 /* print bus type/speed/width info */
2354 dev_info(pci_dev_to_dev(pdev), "%s: (PCIe:%s:%s) ",
2356 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5GT/s" :
2357 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0GT/s" :
2359 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4\n" :
2360 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2\n" :
2361 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1\n" :
2363 dev_info(pci_dev_to_dev(pdev), "%s: MAC: ", netdev->name);
2364 for (i = 0; i < 6; i++)
2365 printk("%2.2x%c", netdev->dev_addr[i], i == 5 ? '\n' : ':');
2367 ret_val = e1000_read_pba_string(hw, pba_str, E1000_PBANUM_LENGTH);
2369 strncpy(pba_str, "Unknown", sizeof(pba_str) - 1);
2370 dev_info(pci_dev_to_dev(pdev), "%s: PBA No: %s\n", netdev->name,
2374 /* Initialize the thermal sensor on i350 devices. */
2375 if (hw->mac.type == e1000_i350 && hw->bus.func == 0) {
2379 * Read the NVM to determine if this i350 device supports an
2380 * external thermal sensor.
2382 e1000_read_nvm(hw, NVM_ETS_CFG, 1, &ets_word);
2383 if (ets_word != 0x0000 && ets_word != 0xFFFF)
2384 adapter->ets = true;
2386 adapter->ets = false;
2388 igb_sysfs_init(adapter);
2391 igb_procfs_init(adapter);
2392 #endif /* IGB_PROCFS */
2393 #endif /* IGB_SYSFS */
2395 adapter->ets = false;
2398 switch (hw->mac.type) {
2400 /* Enable EEE for internal copper PHY devices */
2401 if (hw->phy.media_type == e1000_media_type_copper)
2402 e1000_set_eee_i350(hw);
2404 /* send driver version info to firmware */
2405 igb_init_fw(adapter);
2411 if (netdev->features & NETIF_F_LRO)
2412 dev_info(pci_dev_to_dev(pdev), "Internal LRO is enabled \n");
2414 dev_info(pci_dev_to_dev(pdev), "LRO is disabled \n");
2416 dev_info(pci_dev_to_dev(pdev),
2417 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2418 adapter->msix_entries ? "MSI-X" :
2419 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
2420 adapter->num_rx_queues, adapter->num_tx_queues);
2424 pm_runtime_put_noidle(&pdev->dev);
2428 igb_release_hw_control(adapter);
2430 if (!e1000_check_reset_block(hw))
2431 e1000_phy_hw_reset(hw);
2433 if (hw->flash_address)
2434 iounmap(hw->flash_address);
2436 igb_clear_interrupt_scheme(adapter);
2437 igb_reset_sriov_capability(adapter);
2438 iounmap(hw->hw_addr);
2440 free_netdev(netdev);
2442 pci_release_selected_regions(pdev,
2443 pci_select_bars(pdev, IORESOURCE_MEM));
2446 pci_disable_device(pdev);
2451 * igb_remove - Device Removal Routine
2452 * @pdev: PCI device information struct
2454 * igb_remove is called by the PCI subsystem to alert the driver
2455 * that it should release a PCI device. The could be caused by a
2456 * Hot-Plug event, or because the driver is going to be removed from
2459 static void __devexit igb_remove(struct pci_dev *pdev)
2461 struct net_device *netdev = pci_get_drvdata(pdev);
2462 struct igb_adapter *adapter = netdev_priv(netdev);
2463 struct e1000_hw *hw = &adapter->hw;
2465 pm_runtime_get_noresume(&pdev->dev);
2467 /* flush_scheduled work may reschedule our watchdog task, so
2468 * explicitly disable watchdog tasks from being rescheduled */
2469 set_bit(__IGB_DOWN, &adapter->state);
2470 del_timer_sync(&adapter->watchdog_timer);
2471 if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2472 del_timer_sync(&adapter->dma_err_timer);
2473 del_timer_sync(&adapter->phy_info_timer);
2475 flush_scheduled_work();
2478 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2479 dev_info(pci_dev_to_dev(pdev), "DCA disabled\n");
2480 dca_remove_requester(&pdev->dev);
2481 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2482 E1000_WRITE_REG(hw, E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_DISABLE);
2486 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2487 * would have already happened in close and is redundant. */
2488 igb_release_hw_control(adapter);
2490 unregister_netdev(netdev);
2491 #ifdef CONFIG_IGB_VMDQ_NETDEV
2492 igb_remove_vmdq_netdevs(adapter);
2495 igb_clear_interrupt_scheme(adapter);
2496 igb_reset_sriov_capability(adapter);
2498 iounmap(hw->hw_addr);
2499 if (hw->flash_address)
2500 iounmap(hw->flash_address);
2501 pci_release_selected_regions(pdev,
2502 pci_select_bars(pdev, IORESOURCE_MEM));
2504 kfree(adapter->mac_table);
2505 kfree(adapter->shadow_vfta);
2506 free_netdev(netdev);
2508 pci_disable_pcie_error_reporting(pdev);
2510 pci_disable_device(pdev);
2513 igb_sysfs_exit(adapter);
2516 igb_procfs_exit(adapter);
2517 #endif /* IGB_PROCFS */
2518 #endif /* IGB_SYSFS */
2521 #ifdef HAVE_HW_TIME_STAMP
2523 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2524 * @adapter: board private structure to initialize
2526 * igb_init_hw_timer initializes the function pointer and values for the hw
2527 * timer found in hardware.
2529 static void igb_init_hw_timer(struct igb_adapter *adapter)
2531 struct e1000_hw *hw = &adapter->hw;
2533 switch (hw->mac.type) {
2536 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2537 adapter->cycles.read = igb_read_clock;
2538 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2539 adapter->cycles.mult = 1;
2541 * The 82580 timesync updates the system timer every 8ns by 8ns
2542 * and the value cannot be shifted. Instead we need to shift
2543 * the registers to generate a 64bit timer value. As a result
2544 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2545 * 24 in order to generate a larger value for synchronization.
2547 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2548 /* disable system timer temporarily by setting bit 31 */
2549 E1000_WRITE_REG(hw, E1000_TSAUXC, 0x80000000);
2550 E1000_WRITE_FLUSH(hw);
2552 /* Set registers so that rollover occurs soon to test this. */
2553 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x00000000);
2554 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x80000000);
2555 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x000000FF);
2556 E1000_WRITE_FLUSH(hw);
2558 /* enable system timer by clearing bit 31 */
2559 E1000_WRITE_REG(hw, E1000_TSAUXC, 0x0);
2560 E1000_WRITE_FLUSH(hw);
2562 timecounter_init(&adapter->clock,
2564 ktime_to_ns(ktime_get_real()));
2566 * Synchronize our NIC clock against system wall clock. NIC
2567 * time stamp reading requires ~3us per sample, each sample
2568 * was pretty stable even under load => only require 10
2569 * samples for each offset comparison.
2571 memset(&adapter->compare, 0, sizeof(adapter->compare));
2572 adapter->compare.source = &adapter->clock;
2573 adapter->compare.target = ktime_get_real;
2574 adapter->compare.num_samples = 10;
2575 timecompare_update(&adapter->compare, 0);
2579 * Initialize hardware timer: we keep it running just in case
2580 * that some program needs it later on.
2582 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2583 adapter->cycles.read = igb_read_clock;
2584 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2585 adapter->cycles.mult = 1;
2587 * Scale the NIC clock cycle by a large factor so that
2588 * relatively small clock corrections can be added or
2589 * subtracted at each clock tick. The drawbacks of a large
2590 * factor are a) that the clock register overflows more quickly
2591 * (not such a big deal) and b) that the increment per tick has
2592 * to fit into 24 bits. As a result we need to use a shift of
2593 * 19 so we can fit a value of 16 into the TIMINCA register.
2595 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2596 E1000_WRITE_REG(hw, E1000_TIMINCA,
2597 (1 << E1000_TIMINCA_16NS_SHIFT) |
2598 (16 << IGB_82576_TSYNC_SHIFT));
2600 /* Set registers so that rollover occurs soon to test this. */
2601 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x00000000);
2602 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0xFF800000);
2603 E1000_WRITE_FLUSH(hw);
2605 timecounter_init(&adapter->clock,
2607 ktime_to_ns(ktime_get_real()));
2609 * Synchronize our NIC clock against system wall clock. NIC
2610 * time stamp reading requires ~3us per sample, each sample
2611 * was pretty stable even under load => only require 10
2612 * samples for each offset comparison.
2614 memset(&adapter->compare, 0, sizeof(adapter->compare));
2615 adapter->compare.source = &adapter->clock;
2616 adapter->compare.target = ktime_get_real;
2617 adapter->compare.num_samples = 10;
2618 timecompare_update(&adapter->compare, 0);
2621 /* 82575 does not support timesync */
2627 #endif /* HAVE_HW_TIME_STAMP */
2629 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2630 * @adapter: board private structure to initialize
2632 * igb_sw_init initializes the Adapter private data structure.
2633 * Fields are initialized based on PCI device information and
2634 * OS network device settings (MTU size).
2636 static int igb_sw_init(struct igb_adapter *adapter)
2638 struct e1000_hw *hw = &adapter->hw;
2639 struct net_device *netdev = adapter->netdev;
2640 struct pci_dev *pdev = adapter->pdev;
2642 /* PCI config space info */
2644 hw->vendor_id = pdev->vendor;
2645 hw->device_id = pdev->device;
2646 hw->subsystem_vendor_id = pdev->subsystem_vendor;
2647 hw->subsystem_device_id = pdev->subsystem_device;
2649 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
2651 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2653 /* set default ring sizes */
2654 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2655 adapter->rx_ring_count = IGB_DEFAULT_RXD;
2657 /* set default work limits */
2658 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2660 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2663 /* Initialize the hardware-specific values */
2664 if (e1000_setup_init_funcs(hw, TRUE)) {
2665 dev_err(pci_dev_to_dev(pdev), "Hardware Initialization Failure\n");
2669 adapter->mac_table = kzalloc(sizeof(struct igb_mac_addr) *
2670 hw->mac.rar_entry_count,
2673 /* Setup and initialize a copy of the hw vlan table array */
2674 adapter->shadow_vfta = (u32 *)kzalloc(sizeof(u32) * E1000_VFTA_ENTRIES,
2677 /* These calls may decrease the number of queues */
2678 igb_set_sriov_capability(adapter);
2680 if (igb_init_interrupt_scheme(adapter)) {
2681 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for queues\n");
2685 /* Explicitly disable IRQ since the NIC can be in any state. */
2686 igb_irq_disable(adapter);
2688 set_bit(__IGB_DOWN, &adapter->state);
2694 * igb_open - Called when a network interface is made active
2695 * @netdev: network interface device structure
2697 * Returns 0 on success, negative value on failure
2699 * The open entry point is called when a network interface is made
2700 * active by the system (IFF_UP). At this point all resources needed
2701 * for transmit and receive operations are allocated, the interrupt
2702 * handler is registered with the OS, the watchdog timer is started,
2703 * and the stack is notified that the interface is ready.
2705 static int __igb_open(struct net_device *netdev, bool resuming)
2707 struct igb_adapter *adapter = netdev_priv(netdev);
2708 struct e1000_hw *hw = &adapter->hw;
2709 #ifdef CONFIG_PM_RUNTIME
2710 struct pci_dev *pdev = adapter->pdev;
2711 #endif /* CONFIG_PM_RUNTIME */
2715 /* disallow open during test */
2716 if (test_bit(__IGB_TESTING, &adapter->state)) {
2721 #ifdef CONFIG_PM_RUNTIME
2723 pm_runtime_get_sync(&pdev->dev);
2724 #endif /* CONFIG_PM_RUNTIME */
2726 netif_carrier_off(netdev);
2728 /* allocate transmit descriptors */
2729 err = igb_setup_all_tx_resources(adapter);
2733 /* allocate receive descriptors */
2734 err = igb_setup_all_rx_resources(adapter);
2738 igb_power_up_link(adapter);
2740 /* before we allocate an interrupt, we must be ready to handle it.
2741 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2742 * as soon as we call pci_request_irq, so we have to setup our
2743 * clean_rx handler before we do so. */
2744 igb_configure(adapter);
2746 err = igb_request_irq(adapter);
2750 /* From here on the code is the same as igb_up() */
2751 clear_bit(__IGB_DOWN, &adapter->state);
2753 for (i = 0; i < adapter->num_q_vectors; i++)
2754 napi_enable(&(adapter->q_vector[i]->napi));
2755 igb_configure_lli(adapter);
2757 /* Clear any pending interrupts. */
2758 E1000_READ_REG(hw, E1000_ICR);
2760 igb_irq_enable(adapter);
2762 /* notify VFs that reset has been completed */
2763 if (adapter->vfs_allocated_count) {
2764 u32 reg_data = E1000_READ_REG(hw, E1000_CTRL_EXT);
2765 reg_data |= E1000_CTRL_EXT_PFRSTD;
2766 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg_data);
2769 netif_tx_start_all_queues(netdev);
2771 if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2772 schedule_work(&adapter->dma_err_task);
2774 /* start the watchdog. */
2775 hw->mac.get_link_status = 1;
2776 schedule_work(&adapter->watchdog_task);
2778 return E1000_SUCCESS;
2781 igb_release_hw_control(adapter);
2782 igb_power_down_link(adapter);
2783 igb_free_all_rx_resources(adapter);
2785 igb_free_all_tx_resources(adapter);
2789 #ifdef CONFIG_PM_RUNTIME
2791 pm_runtime_put(&pdev->dev);
2792 #endif /* CONFIG_PM_RUNTIME */
2797 static int igb_open(struct net_device *netdev)
2799 return __igb_open(netdev, false);
2803 * igb_close - Disables a network interface
2804 * @netdev: network interface device structure
2806 * Returns 0, this is not allowed to fail
2808 * The close entry point is called when an interface is de-activated
2809 * by the OS. The hardware is still under the driver's control, but
2810 * needs to be disabled. A global MAC reset is issued to stop the
2811 * hardware, and all transmit and receive resources are freed.
2813 static int __igb_close(struct net_device *netdev, bool suspending)
2815 struct igb_adapter *adapter = netdev_priv(netdev);
2816 #ifdef CONFIG_PM_RUNTIME
2817 struct pci_dev *pdev = adapter->pdev;
2818 #endif /* CONFIG_PM_RUNTIME */
2820 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2822 #ifdef CONFIG_PM_RUNTIME
2824 pm_runtime_get_sync(&pdev->dev);
2825 #endif /* CONFIG_PM_RUNTIME */
2829 igb_release_hw_control(adapter);
2831 igb_free_irq(adapter);
2833 igb_free_all_tx_resources(adapter);
2834 igb_free_all_rx_resources(adapter);
2836 #ifdef CONFIG_PM_RUNTIME
2838 pm_runtime_put_sync(&pdev->dev);
2839 #endif /* CONFIG_PM_RUNTIME */
2844 static int igb_close(struct net_device *netdev)
2846 return __igb_close(netdev, false);
2850 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2851 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2853 * Return 0 on success, negative on failure
2855 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2857 struct device *dev = tx_ring->dev;
2858 int orig_node = dev_to_node(dev);
2861 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
2862 tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
2863 if (!tx_ring->tx_buffer_info)
2864 tx_ring->tx_buffer_info = vzalloc(size);
2865 if (!tx_ring->tx_buffer_info)
2868 /* round up to nearest 4K */
2869 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2870 tx_ring->size = ALIGN(tx_ring->size, 4096);
2872 set_dev_node(dev, tx_ring->numa_node);
2873 tx_ring->desc = dma_alloc_coherent(dev,
2877 set_dev_node(dev, orig_node);
2879 tx_ring->desc = dma_alloc_coherent(dev,
2887 tx_ring->next_to_use = 0;
2888 tx_ring->next_to_clean = 0;
2893 vfree(tx_ring->tx_buffer_info);
2895 "Unable to allocate memory for the transmit descriptor ring\n");
2900 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2901 * (Descriptors) for all queues
2902 * @adapter: board private structure
2904 * Return 0 on success, negative on failure
2906 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2908 struct pci_dev *pdev = adapter->pdev;
2911 for (i = 0; i < adapter->num_tx_queues; i++) {
2912 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2914 dev_err(pci_dev_to_dev(pdev),
2915 "Allocation for Tx Queue %u failed\n", i);
2916 for (i--; i >= 0; i--)
2917 igb_free_tx_resources(adapter->tx_ring[i]);
2926 * igb_setup_tctl - configure the transmit control registers
2927 * @adapter: Board private structure
2929 void igb_setup_tctl(struct igb_adapter *adapter)
2931 struct e1000_hw *hw = &adapter->hw;
2934 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2935 E1000_WRITE_REG(hw, E1000_TXDCTL(0), 0);
2937 /* Program the Transmit Control Register */
2938 tctl = E1000_READ_REG(hw, E1000_TCTL);
2939 tctl &= ~E1000_TCTL_CT;
2940 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2941 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2943 e1000_config_collision_dist(hw);
2945 /* Enable transmits */
2946 tctl |= E1000_TCTL_EN;
2948 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
2952 * igb_configure_tx_ring - Configure transmit ring after Reset
2953 * @adapter: board private structure
2954 * @ring: tx ring to configure
2956 * Configure a transmit ring after a reset.
2958 void igb_configure_tx_ring(struct igb_adapter *adapter,
2959 struct igb_ring *ring)
2961 struct e1000_hw *hw = &adapter->hw;
2963 u64 tdba = ring->dma;
2964 int reg_idx = ring->reg_idx;
2966 /* disable the queue */
2967 E1000_WRITE_REG(hw, E1000_TXDCTL(reg_idx), 0);
2968 E1000_WRITE_FLUSH(hw);
2971 E1000_WRITE_REG(hw, E1000_TDLEN(reg_idx),
2972 ring->count * sizeof(union e1000_adv_tx_desc));
2973 E1000_WRITE_REG(hw, E1000_TDBAL(reg_idx),
2974 tdba & 0x00000000ffffffffULL);
2975 E1000_WRITE_REG(hw, E1000_TDBAH(reg_idx), tdba >> 32);
2977 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2978 E1000_WRITE_REG(hw, E1000_TDH(reg_idx), 0);
2979 writel(0, ring->tail);
2981 txdctl |= IGB_TX_PTHRESH;
2982 txdctl |= IGB_TX_HTHRESH << 8;
2983 txdctl |= IGB_TX_WTHRESH << 16;
2985 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2986 E1000_WRITE_REG(hw, E1000_TXDCTL(reg_idx), txdctl);
2990 * igb_configure_tx - Configure transmit Unit after Reset
2991 * @adapter: board private structure
2993 * Configure the Tx unit of the MAC after a reset.
2995 static void igb_configure_tx(struct igb_adapter *adapter)
2999 for (i = 0; i < adapter->num_tx_queues; i++)
3000 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
3004 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
3005 * @rx_ring: rx descriptor ring (for a specific queue) to setup
3007 * Returns 0 on success, negative on failure
3009 int igb_setup_rx_resources(struct igb_ring *rx_ring)
3011 struct device *dev = rx_ring->dev;
3012 int orig_node = dev_to_node(dev);
3015 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3016 rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
3017 if (!rx_ring->rx_buffer_info)
3018 rx_ring->rx_buffer_info = vzalloc(size);
3019 if (!rx_ring->rx_buffer_info)
3022 desc_len = sizeof(union e1000_adv_rx_desc);
3024 /* Round up to nearest 4K */
3025 rx_ring->size = rx_ring->count * desc_len;
3026 rx_ring->size = ALIGN(rx_ring->size, 4096);
3028 set_dev_node(dev, rx_ring->numa_node);
3029 rx_ring->desc = dma_alloc_coherent(dev,
3033 set_dev_node(dev, orig_node);
3035 rx_ring->desc = dma_alloc_coherent(dev,
3043 rx_ring->next_to_clean = 0;
3044 rx_ring->next_to_use = 0;
3049 vfree(rx_ring->rx_buffer_info);
3050 rx_ring->rx_buffer_info = NULL;
3051 dev_err(dev, "Unable to allocate memory for the receive descriptor"
3057 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
3058 * (Descriptors) for all queues
3059 * @adapter: board private structure
3061 * Return 0 on success, negative on failure
3063 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
3065 struct pci_dev *pdev = adapter->pdev;
3068 for (i = 0; i < adapter->num_rx_queues; i++) {
3069 err = igb_setup_rx_resources(adapter->rx_ring[i]);
3071 dev_err(pci_dev_to_dev(pdev),
3072 "Allocation for Rx Queue %u failed\n", i);
3073 for (i--; i >= 0; i--)
3074 igb_free_rx_resources(adapter->rx_ring[i]);
3083 * igb_setup_mrqc - configure the multiple receive queue control registers
3084 * @adapter: Board private structure
3086 static void igb_setup_mrqc(struct igb_adapter *adapter)
3088 struct e1000_hw *hw = &adapter->hw;
3090 u32 j, num_rx_queues, shift = 0, shift2 = 0;
3095 static const u8 rsshash[40] = {
3096 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
3097 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
3098 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
3099 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
3101 /* Fill out hash function seeds */
3102 for (j = 0; j < 10; j++) {
3103 u32 rsskey = rsshash[(j * 4)];
3104 rsskey |= rsshash[(j * 4) + 1] << 8;
3105 rsskey |= rsshash[(j * 4) + 2] << 16;
3106 rsskey |= rsshash[(j * 4) + 3] << 24;
3107 E1000_WRITE_REG_ARRAY(hw, E1000_RSSRK(0), j, rsskey);
3110 num_rx_queues = adapter->rss_queues;
3112 if (adapter->vfs_allocated_count || adapter->vmdq_pools) {
3113 /* 82575 and 82576 supports 2 RSS queues for VMDq */
3114 switch (hw->mac.type) {
3131 if (hw->mac.type == e1000_82575)
3135 for (j = 0; j < (32 * 4); j++) {
3136 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
3138 reta.bytes[j & 3] |= num_rx_queues << shift2;
3140 E1000_WRITE_REG(hw, E1000_RETA(j >> 2), reta.dword);
3144 * Disable raw packet checksumming so that RSS hash is placed in
3145 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
3146 * offloads as they are enabled by default
3148 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
3149 rxcsum |= E1000_RXCSUM_PCSD;
3151 if (adapter->hw.mac.type >= e1000_82576)
3152 /* Enable Receive Checksum Offload for SCTP */
3153 rxcsum |= E1000_RXCSUM_CRCOFL;
3155 /* Don't need to set TUOFL or IPOFL, they default to 1 */
3156 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
3158 /* If VMDq is enabled then we set the appropriate mode for that, else
3159 * we default to RSS so that an RSS hash is calculated per packet even
3160 * if we are only using one queue */
3161 if (adapter->vfs_allocated_count || adapter->vmdq_pools) {
3162 if (hw->mac.type > e1000_82575) {
3163 /* Set the default pool for the PF's first queue */
3164 u32 vtctl = E1000_READ_REG(hw, E1000_VT_CTL);
3165 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
3166 E1000_VT_CTL_DISABLE_DEF_POOL);
3167 vtctl |= adapter->vfs_allocated_count <<
3168 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
3169 E1000_WRITE_REG(hw, E1000_VT_CTL, vtctl);
3170 } else if (adapter->rss_queues > 1) {
3171 /* set default queue for pool 1 to queue 2 */
3172 E1000_WRITE_REG(hw, E1000_VT_CTL,
3173 adapter->rss_queues << 7);
3175 if (adapter->rss_queues > 1)
3176 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
3178 mrqc = E1000_MRQC_ENABLE_VMDQ;
3180 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
3183 igb_vmm_control(adapter);
3186 * Generate RSS hash based on TCP port numbers and/or
3187 * IPv4/v6 src and dst addresses since UDP cannot be
3188 * hashed reliably due to IP fragmentation
3190 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
3191 E1000_MRQC_RSS_FIELD_IPV4_TCP |
3192 E1000_MRQC_RSS_FIELD_IPV6 |
3193 E1000_MRQC_RSS_FIELD_IPV6_TCP |
3194 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
3196 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
3200 * igb_setup_rctl - configure the receive control registers
3201 * @adapter: Board private structure
3203 void igb_setup_rctl(struct igb_adapter *adapter)
3205 struct e1000_hw *hw = &adapter->hw;
3208 rctl = E1000_READ_REG(hw, E1000_RCTL);
3210 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3211 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
3213 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
3214 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
3217 * enable stripping of CRC. It's unlikely this will break BMC
3218 * redirection as it did with e1000. Newer features require
3219 * that the HW strips the CRC.
3221 rctl |= E1000_RCTL_SECRC;
3223 /* disable store bad packets and clear size bits. */
3224 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
3226 /* enable LPE to prevent packets larger than max_frame_size */
3227 rctl |= E1000_RCTL_LPE;
3229 /* disable queue 0 to prevent tail write w/o re-config */
3230 E1000_WRITE_REG(hw, E1000_RXDCTL(0), 0);
3232 /* Attention!!! For SR-IOV PF driver operations you must enable
3233 * queue drop for all VF and PF queues to prevent head of line blocking
3234 * if an un-trusted VF does not provide descriptors to hardware.
3236 if (adapter->vfs_allocated_count) {
3237 /* set all queue drop enable bits */
3238 E1000_WRITE_REG(hw, E1000_QDE, ALL_QUEUES);
3241 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3244 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
3247 struct e1000_hw *hw = &adapter->hw;
3250 /* if it isn't the PF check to see if VFs are enabled and
3251 * increase the size to support vlan tags */
3252 if (vfn < adapter->vfs_allocated_count &&
3253 adapter->vf_data[vfn].vlans_enabled)
3256 #ifdef CONFIG_IGB_VMDQ_NETDEV
3257 if (vfn >= adapter->vfs_allocated_count) {
3258 int queue = vfn - adapter->vfs_allocated_count;
3259 struct igb_vmdq_adapter *vadapter;
3261 vadapter = netdev_priv(adapter->vmdq_netdev[queue-1]);
3262 if (vadapter->vlgrp)
3266 vmolr = E1000_READ_REG(hw, E1000_VMOLR(vfn));
3267 vmolr &= ~E1000_VMOLR_RLPML_MASK;
3268 vmolr |= size | E1000_VMOLR_LPE;
3269 E1000_WRITE_REG(hw, E1000_VMOLR(vfn), vmolr);
3275 * igb_rlpml_set - set maximum receive packet size
3276 * @adapter: board private structure
3278 * Configure maximum receivable packet size.
3280 static void igb_rlpml_set(struct igb_adapter *adapter)
3282 u32 max_frame_size = adapter->max_frame_size;
3283 struct e1000_hw *hw = &adapter->hw;
3284 u16 pf_id = adapter->vfs_allocated_count;
3286 if (adapter->vmdq_pools && hw->mac.type != e1000_82575) {
3288 for (i = 0; i < adapter->vmdq_pools; i++)
3289 igb_set_vf_rlpml(adapter, max_frame_size, pf_id + i);
3291 * If we're in VMDQ or SR-IOV mode, then set global RLPML
3292 * to our max jumbo frame size, in case we need to enable
3293 * jumbo frames on one of the rings later.
3294 * This will not pass over-length frames into the default
3295 * queue because it's gated by the VMOLR.RLPML.
3297 max_frame_size = MAX_JUMBO_FRAME_SIZE;
3299 /* Set VF RLPML for the PF device. */
3300 if (adapter->vfs_allocated_count)
3301 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
3303 E1000_WRITE_REG(hw, E1000_RLPML, max_frame_size);
3306 static inline void igb_set_vf_vlan_strip(struct igb_adapter *adapter,
3307 int vfn, bool enable)
3309 struct e1000_hw *hw = &adapter->hw;
3313 if (hw->mac.type < e1000_82576)
3316 if (hw->mac.type == e1000_i350)
3317 reg = hw->hw_addr + E1000_DVMOLR(vfn);
3319 reg = hw->hw_addr + E1000_VMOLR(vfn);
3323 val |= E1000_VMOLR_STRVLAN;
3325 val &= ~(E1000_VMOLR_STRVLAN);
3328 static inline void igb_set_vmolr(struct igb_adapter *adapter,
3331 struct e1000_hw *hw = &adapter->hw;
3335 * This register exists only on 82576 and newer so if we are older then
3336 * we should exit and do nothing
3338 if (hw->mac.type < e1000_82576)
3341 vmolr = E1000_READ_REG(hw, E1000_VMOLR(vfn));
3344 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3346 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
3348 /* clear all bits that might not be set */
3349 vmolr &= ~E1000_VMOLR_RSSE;
3351 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
3352 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3354 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3355 vmolr |= E1000_VMOLR_LPE; /* Accept long packets */
3357 E1000_WRITE_REG(hw, E1000_VMOLR(vfn), vmolr);
3361 * igb_configure_rx_ring - Configure a receive ring after Reset
3362 * @adapter: board private structure
3363 * @ring: receive ring to be configured
3365 * Configure the Rx unit of the MAC after a reset.
3367 void igb_configure_rx_ring(struct igb_adapter *adapter,
3368 struct igb_ring *ring)
3370 struct e1000_hw *hw = &adapter->hw;
3371 u64 rdba = ring->dma;
3372 int reg_idx = ring->reg_idx;
3373 u32 srrctl = 0, rxdctl = 0;
3375 /* disable the queue */
3376 E1000_WRITE_REG(hw, E1000_RXDCTL(reg_idx), 0);
3378 /* Set DMA base address registers */
3379 E1000_WRITE_REG(hw, E1000_RDBAL(reg_idx),
3380 rdba & 0x00000000ffffffffULL);
3381 E1000_WRITE_REG(hw, E1000_RDBAH(reg_idx), rdba >> 32);
3382 E1000_WRITE_REG(hw, E1000_RDLEN(reg_idx),
3383 ring->count * sizeof(union e1000_adv_rx_desc));
3385 /* initialize head and tail */
3386 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
3387 E1000_WRITE_REG(hw, E1000_RDH(reg_idx), 0);
3388 writel(0, ring->tail);
3390 /* set descriptor configuration */
3391 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
3392 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3393 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
3394 srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3396 srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3398 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3399 #else /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
3400 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
3401 E1000_SRRCTL_BSIZEPKT_SHIFT;
3402 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
3403 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
3404 #ifdef IGB_PER_PKT_TIMESTAMP
3405 if (hw->mac.type >= e1000_82580)
3406 srrctl |= E1000_SRRCTL_TIMESTAMP;
3409 * We should set the drop enable bit if:
3412 * Flow Control is disabled and number of RX queues > 1
3414 * This allows us to avoid head of line blocking for security
3415 * and performance reasons.
3417 if (adapter->vfs_allocated_count ||
3418 (adapter->num_rx_queues > 1 &&
3419 (hw->fc.requested_mode == e1000_fc_none ||
3420 hw->fc.requested_mode == e1000_fc_rx_pause)))
3421 srrctl |= E1000_SRRCTL_DROP_EN;
3423 E1000_WRITE_REG(hw, E1000_SRRCTL(reg_idx), srrctl);
3425 /* set filtering for VMDQ pools */
3426 igb_set_vmolr(adapter, reg_idx & 0x7, true);
3428 rxdctl |= IGB_RX_PTHRESH;
3429 rxdctl |= IGB_RX_HTHRESH << 8;
3430 rxdctl |= IGB_RX_WTHRESH << 16;
3432 /* enable receive descriptor fetching */
3433 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3434 E1000_WRITE_REG(hw, E1000_RXDCTL(reg_idx), rxdctl);
3438 * igb_configure_rx - Configure receive Unit after Reset
3439 * @adapter: board private structure
3441 * Configure the Rx unit of the MAC after a reset.
3443 static void igb_configure_rx(struct igb_adapter *adapter)
3447 /* set UTA to appropriate mode */
3448 igb_set_uta(adapter);
3450 igb_full_sync_mac_table(adapter);
3451 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3452 * the Base and Length of the Rx Descriptor Ring */
3453 for (i = 0; i < adapter->num_rx_queues; i++)
3454 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
3458 * igb_free_tx_resources - Free Tx Resources per Queue
3459 * @tx_ring: Tx descriptor ring for a specific queue
3461 * Free all transmit software resources
3463 void igb_free_tx_resources(struct igb_ring *tx_ring)
3465 igb_clean_tx_ring(tx_ring);
3467 vfree(tx_ring->tx_buffer_info);
3468 tx_ring->tx_buffer_info = NULL;
3470 /* if not set, then don't free */
3474 dma_free_coherent(tx_ring->dev, tx_ring->size,
3475 tx_ring->desc, tx_ring->dma);
3477 tx_ring->desc = NULL;
3481 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3482 * @adapter: board private structure
3484 * Free all transmit software resources
3486 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3490 for (i = 0; i < adapter->num_tx_queues; i++)
3491 igb_free_tx_resources(adapter->tx_ring[i]);
3494 void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
3495 struct igb_tx_buffer *tx_buffer)
3497 if (tx_buffer->skb) {
3498 dev_kfree_skb_any(tx_buffer->skb);
3499 if (dma_unmap_len(tx_buffer, len))
3500 dma_unmap_single(ring->dev,
3501 dma_unmap_addr(tx_buffer, dma),
3502 dma_unmap_len(tx_buffer, len),
3504 } else if (dma_unmap_len(tx_buffer, len)) {
3505 dma_unmap_page(ring->dev,
3506 dma_unmap_addr(tx_buffer, dma),
3507 dma_unmap_len(tx_buffer, len),
3510 tx_buffer->next_to_watch = NULL;
3511 tx_buffer->skb = NULL;
3512 dma_unmap_len_set(tx_buffer, len, 0);
3513 /* buffer_info must be completely set up in the transmit path */
3517 * igb_clean_tx_ring - Free Tx Buffers
3518 * @tx_ring: ring to be cleaned
3520 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3522 struct igb_tx_buffer *buffer_info;
3526 if (!tx_ring->tx_buffer_info)
3528 /* Free all the Tx ring sk_buffs */
3530 for (i = 0; i < tx_ring->count; i++) {
3531 buffer_info = &tx_ring->tx_buffer_info[i];
3532 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3536 netdev_tx_reset_queue(txring_txq(tx_ring));
3537 #endif /* CONFIG_BQL */
3539 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3540 memset(tx_ring->tx_buffer_info, 0, size);
3542 /* Zero out the descriptor ring */
3543 memset(tx_ring->desc, 0, tx_ring->size);
3545 tx_ring->next_to_use = 0;
3546 tx_ring->next_to_clean = 0;
3550 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3551 * @adapter: board private structure
3553 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3557 for (i = 0; i < adapter->num_tx_queues; i++)
3558 igb_clean_tx_ring(adapter->tx_ring[i]);
3562 * igb_free_rx_resources - Free Rx Resources
3563 * @rx_ring: ring to clean the resources from
3565 * Free all receive software resources
3567 void igb_free_rx_resources(struct igb_ring *rx_ring)
3569 igb_clean_rx_ring(rx_ring);
3571 vfree(rx_ring->rx_buffer_info);
3572 rx_ring->rx_buffer_info = NULL;
3574 /* if not set, then don't free */
3578 dma_free_coherent(rx_ring->dev, rx_ring->size,
3579 rx_ring->desc, rx_ring->dma);
3581 rx_ring->desc = NULL;
3585 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3586 * @adapter: board private structure
3588 * Free all receive software resources
3590 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3594 for (i = 0; i < adapter->num_rx_queues; i++)
3595 igb_free_rx_resources(adapter->rx_ring[i]);
3599 * igb_clean_rx_ring - Free Rx Buffers per Queue
3600 * @rx_ring: ring to free buffers from
3602 void igb_clean_rx_ring(struct igb_ring *rx_ring)
3605 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
3606 const int bufsz = rx_ring->rx_buffer_len;
3608 const int bufsz = IGB_RX_HDR_LEN;
3612 if (!rx_ring->rx_buffer_info)
3615 /* Free all the Rx ring sk_buffs */
3616 for (i = 0; i < rx_ring->count; i++) {
3617 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
3618 if (buffer_info->dma) {
3619 dma_unmap_single(rx_ring->dev,
3623 buffer_info->dma = 0;
3626 if (buffer_info->skb) {
3627 dev_kfree_skb(buffer_info->skb);
3628 buffer_info->skb = NULL;
3630 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
3631 if (buffer_info->page_dma) {
3632 dma_unmap_page(rx_ring->dev,
3633 buffer_info->page_dma,
3636 buffer_info->page_dma = 0;
3638 if (buffer_info->page) {
3639 put_page(buffer_info->page);
3640 buffer_info->page = NULL;
3641 buffer_info->page_offset = 0;
3646 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3647 memset(rx_ring->rx_buffer_info, 0, size);
3649 /* Zero out the descriptor ring */
3650 memset(rx_ring->desc, 0, rx_ring->size);
3652 rx_ring->next_to_clean = 0;
3653 rx_ring->next_to_use = 0;
3657 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3658 * @adapter: board private structure
3660 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3664 for (i = 0; i < adapter->num_rx_queues; i++)
3665 igb_clean_rx_ring(adapter->rx_ring[i]);
3669 * igb_set_mac - Change the Ethernet Address of the NIC
3670 * @netdev: network interface device structure
3671 * @p: pointer to an address structure
3673 * Returns 0 on success, negative on failure
3675 static int igb_set_mac(struct net_device *netdev, void *p)
3677 struct igb_adapter *adapter = netdev_priv(netdev);
3678 struct e1000_hw *hw = &adapter->hw;
3679 struct sockaddr *addr = p;
3681 if (!is_valid_ether_addr(addr->sa_data))
3682 return -EADDRNOTAVAIL;
3684 igb_del_mac_filter(adapter, hw->mac.addr,
3685 adapter->vfs_allocated_count);
3686 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3687 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3689 /* set the correct pool for the new PF MAC address in entry 0 */
3690 return igb_add_mac_filter(adapter, hw->mac.addr,
3691 adapter->vfs_allocated_count);
3695 * igb_write_mc_addr_list - write multicast addresses to MTA
3696 * @netdev: network interface device structure
3698 * Writes multicast address list to the MTA hash table.
3699 * Returns: -ENOMEM on failure
3700 * 0 on no addresses written
3701 * X on writing X addresses to MTA
3703 int igb_write_mc_addr_list(struct net_device *netdev)
3705 struct igb_adapter *adapter = netdev_priv(netdev);
3706 struct e1000_hw *hw = &adapter->hw;
3707 #ifdef NETDEV_HW_ADDR_T_MULTICAST
3708 struct netdev_hw_addr *ha;
3710 struct dev_mc_list *ha;
3714 #ifdef CONFIG_IGB_VMDQ_NETDEV
3717 count = netdev_mc_count(netdev);
3718 #ifdef CONFIG_IGB_VMDQ_NETDEV
3719 for (vm = 1; vm < adapter->vmdq_pools; vm++) {
3720 if (!adapter->vmdq_netdev[vm])
3722 if (!netif_running(adapter->vmdq_netdev[vm]))
3724 count += netdev_mc_count(adapter->vmdq_netdev[vm]);
3729 e1000_update_mc_addr_list(hw, NULL, 0);
3732 mta_list = kzalloc(count * 6, GFP_ATOMIC);
3736 /* The shared function expects a packed array of only addresses. */
3738 netdev_for_each_mc_addr(ha, netdev)
3739 #ifdef NETDEV_HW_ADDR_T_MULTICAST
3740 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3742 memcpy(mta_list + (i++ * ETH_ALEN), ha->dmi_addr, ETH_ALEN);
3744 #ifdef CONFIG_IGB_VMDQ_NETDEV
3745 for (vm = 1; vm < adapter->vmdq_pools; vm++) {
3746 if (!adapter->vmdq_netdev[vm])
3748 if (!netif_running(adapter->vmdq_netdev[vm]) ||
3749 !netdev_mc_count(adapter->vmdq_netdev[vm]))
3751 netdev_for_each_mc_addr(ha, adapter->vmdq_netdev[vm])
3752 #ifdef NETDEV_HW_ADDR_T_MULTICAST
3753 memcpy(mta_list + (i++ * ETH_ALEN),
3754 ha->addr, ETH_ALEN);
3756 memcpy(mta_list + (i++ * ETH_ALEN),
3757 ha->dmi_addr, ETH_ALEN);
3761 e1000_update_mc_addr_list(hw, mta_list, i);
3767 void igb_rar_set(struct igb_adapter *adapter, u32 index)
3769 u32 rar_low, rar_high;
3770 struct e1000_hw *hw = &adapter->hw;
3771 u8 *addr = adapter->mac_table[index].addr;
3772 /* HW expects these in little endian so we reverse the byte order
3773 * from network order (big endian) to little endian
3775 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
3776 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
3777 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
3779 /* Indicate to hardware the Address is Valid. */
3780 if (adapter->mac_table[index].state & IGB_MAC_STATE_IN_USE)
3781 rar_high |= E1000_RAH_AV;
3783 if (hw->mac.type == e1000_82575)
3784 rar_high |= E1000_RAH_POOL_1 * adapter->mac_table[index].queue;
3786 rar_high |= E1000_RAH_POOL_1 << adapter->mac_table[index].queue;
3788 E1000_WRITE_REG(hw, E1000_RAL(index), rar_low);
3789 E1000_WRITE_FLUSH(hw);
3790 E1000_WRITE_REG(hw, E1000_RAH(index), rar_high);
3791 E1000_WRITE_FLUSH(hw);
3794 void igb_full_sync_mac_table(struct igb_adapter *adapter)
3796 struct e1000_hw *hw = &adapter->hw;
3798 for (i = 0; i < hw->mac.rar_entry_count; i++) {
3799 igb_rar_set(adapter, i);
3803 void igb_sync_mac_table(struct igb_adapter *adapter)
3805 struct e1000_hw *hw = &adapter->hw;
3807 for (i = 0; i < hw->mac.rar_entry_count; i++) {
3808 if (adapter->mac_table[i].state & IGB_MAC_STATE_MODIFIED)
3809 igb_rar_set(adapter, i);
3810 adapter->mac_table[i].state &= ~(IGB_MAC_STATE_MODIFIED);
3814 int igb_available_rars(struct igb_adapter *adapter)
3816 struct e1000_hw *hw = &adapter->hw;
3819 for (i = 0; i < hw->mac.rar_entry_count; i++) {
3820 if (adapter->mac_table[i].state == 0)
3826 #ifdef HAVE_SET_RX_MODE
3828 * igb_write_uc_addr_list - write unicast addresses to RAR table
3829 * @netdev: network interface device structure
3831 * Writes unicast address list to the RAR table.
3832 * Returns: -ENOMEM on failure/insufficient address space
3833 * 0 on no addresses written
3834 * X on writing X addresses to the RAR table
3836 static int igb_write_uc_addr_list(struct net_device *netdev)
3838 struct igb_adapter *adapter = netdev_priv(netdev);
3839 unsigned int vfn = adapter->vfs_allocated_count;
3842 /* return ENOMEM indicating insufficient memory for addresses */
3843 if (netdev_uc_count(netdev) > igb_available_rars(adapter))
3845 if (!netdev_uc_empty(netdev)) {
3846 #ifdef NETDEV_HW_ADDR_T_UNICAST
3847 struct netdev_hw_addr *ha;
3849 struct dev_mc_list *ha;
3851 netdev_for_each_uc_addr(ha, netdev) {
3852 #ifdef NETDEV_HW_ADDR_T_UNICAST
3853 igb_del_mac_filter(adapter, ha->addr, vfn);
3854 igb_add_mac_filter(adapter, ha->addr, vfn);
3856 igb_del_mac_filter(adapter, ha->da_addr, vfn);
3857 igb_add_mac_filter(adapter, ha->da_addr, vfn);
3865 #endif /* HAVE_SET_RX_MODE */
3867 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3868 * @netdev: network interface device structure
3870 * The set_rx_mode entry point is called whenever the unicast or multicast
3871 * address lists or the network interface flags are updated. This routine is
3872 * responsible for configuring the hardware for proper unicast, multicast,
3873 * promiscuous mode, and all-multi behavior.
3875 static void igb_set_rx_mode(struct net_device *netdev)
3877 struct igb_adapter *adapter = netdev_priv(netdev);
3878 struct e1000_hw *hw = &adapter->hw;
3879 unsigned int vfn = adapter->vfs_allocated_count;
3880 u32 rctl, vmolr = 0;
3883 /* Check for Promiscuous and All Multicast modes */
3884 rctl = E1000_READ_REG(hw, E1000_RCTL);
3886 /* clear the effected bits */
3887 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3889 if (netdev->flags & IFF_PROMISC) {
3890 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3891 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3893 if (netdev->flags & IFF_ALLMULTI) {
3894 rctl |= E1000_RCTL_MPE;
3895 vmolr |= E1000_VMOLR_MPME;
3898 * Write addresses to the MTA, if the attempt fails
3899 * then we should just turn on promiscuous mode so
3900 * that we can at least receive multicast traffic
3902 count = igb_write_mc_addr_list(netdev);
3904 rctl |= E1000_RCTL_MPE;
3905 vmolr |= E1000_VMOLR_MPME;
3907 vmolr |= E1000_VMOLR_ROMPE;
3910 #ifdef HAVE_SET_RX_MODE
3912 * Write addresses to available RAR registers, if there is not
3913 * sufficient space to store all the addresses then enable
3914 * unicast promiscuous mode
3916 count = igb_write_uc_addr_list(netdev);
3918 rctl |= E1000_RCTL_UPE;
3919 vmolr |= E1000_VMOLR_ROPE;
3921 #endif /* HAVE_SET_RX_MODE */
3922 rctl |= E1000_RCTL_VFE;
3924 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3927 * In order to support SR-IOV and eventually VMDq it is necessary to set
3928 * the VMOLR to enable the appropriate modes. Without this workaround
3929 * we will have issues with VLAN tag stripping not being done for frames
3930 * that are only arriving because we are the default pool
3932 if (hw->mac.type < e1000_82576)
3935 vmolr |= E1000_READ_REG(hw, E1000_VMOLR(vfn)) &
3936 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3937 E1000_WRITE_REG(hw, E1000_VMOLR(vfn), vmolr);
3938 igb_restore_vf_multicasts(adapter);
3941 static void igb_check_wvbr(struct igb_adapter *adapter)
3943 struct e1000_hw *hw = &adapter->hw;
3946 switch (hw->mac.type) {
3949 if (!(wvbr = E1000_READ_REG(hw, E1000_WVBR)))
3956 adapter->wvbr |= wvbr;
3959 #define IGB_STAGGERED_QUEUE_OFFSET 8
3961 static void igb_spoof_check(struct igb_adapter *adapter)
3968 for(j = 0; j < adapter->vfs_allocated_count; j++) {
3969 if (adapter->wvbr & (1 << j) ||
3970 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3971 DPRINTK(DRV, WARNING,
3972 "Spoof event(s) detected on VF %d\n", j);
3975 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3980 /* Need to wait a few seconds after link up to get diagnostic information from
3982 static void igb_update_phy_info(unsigned long data)
3984 struct igb_adapter *adapter = (struct igb_adapter *) data;
3985 e1000_get_phy_info(&adapter->hw);
3989 * igb_has_link - check shared code for link and determine up/down
3990 * @adapter: pointer to driver private info
3992 bool igb_has_link(struct igb_adapter *adapter)
3994 struct e1000_hw *hw = &adapter->hw;
3995 bool link_active = FALSE;
3997 /* get_link_status is set on LSC (link status) interrupt or
3998 * rx sequence error interrupt. get_link_status will stay
3999 * false until the e1000_check_for_link establishes link
4000 * for copper adapters ONLY
4002 switch (hw->phy.media_type) {
4003 case e1000_media_type_copper:
4004 if (!hw->mac.get_link_status)
4006 case e1000_media_type_internal_serdes:
4007 e1000_check_for_link(hw);
4008 link_active = !hw->mac.get_link_status;
4010 case e1000_media_type_unknown:
4019 * igb_watchdog - Timer Call-back
4020 * @data: pointer to adapter cast into an unsigned long
4022 static void igb_watchdog(unsigned long data)
4024 struct igb_adapter *adapter = (struct igb_adapter *)data;
4025 /* Do the rest outside of interrupt context */
4026 schedule_work(&adapter->watchdog_task);
4029 static void igb_watchdog_task(struct work_struct *work)
4031 struct igb_adapter *adapter = container_of(work,
4034 struct e1000_hw *hw = &adapter->hw;
4035 struct net_device *netdev = adapter->netdev;
4038 u32 thstat, ctrl_ext;
4041 link = igb_has_link(adapter);
4043 /* Cancel scheduled suspend requests. */
4044 pm_runtime_resume(netdev->dev.parent);
4046 if (!netif_carrier_ok(netdev)) {
4048 e1000_get_speed_and_duplex(hw,
4049 &adapter->link_speed,
4050 &adapter->link_duplex);
4052 ctrl = E1000_READ_REG(hw, E1000_CTRL);
4053 /* Links status message must follow this format */
4054 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
4055 "Flow Control: %s\n",
4057 adapter->link_speed,
4058 adapter->link_duplex == FULL_DUPLEX ?
4059 "Full Duplex" : "Half Duplex",
4060 ((ctrl & E1000_CTRL_TFCE) &&
4061 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX":
4062 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
4063 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
4064 /* adjust timeout factor according to speed/duplex */
4065 adapter->tx_timeout_factor = 1;
4066 switch (adapter->link_speed) {
4068 adapter->tx_timeout_factor = 14;
4071 /* maybe add some timeout factor ? */
4075 netif_carrier_on(netdev);
4076 netif_tx_wake_all_queues(netdev);
4078 igb_ping_all_vfs(adapter);
4080 igb_check_vf_rate_limit(adapter);
4081 #endif /* IFLA_VF_MAX */
4083 /* link state has changed, schedule phy info update */
4084 if (!test_bit(__IGB_DOWN, &adapter->state))
4085 mod_timer(&adapter->phy_info_timer,
4086 round_jiffies(jiffies + 2 * HZ));
4089 if (netif_carrier_ok(netdev)) {
4090 adapter->link_speed = 0;
4091 adapter->link_duplex = 0;
4092 /* check for thermal sensor event on i350 */
4093 if (hw->mac.type == e1000_i350) {
4094 thstat = E1000_READ_REG(hw, E1000_THSTAT);
4095 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
4096 if ((hw->phy.media_type ==
4097 e1000_media_type_copper) &&
4099 E1000_CTRL_EXT_LINK_MODE_SGMII)) {
4100 if (thstat & E1000_THSTAT_PWR_DOWN) {
4101 printk(KERN_ERR "igb: %s The "
4102 "network adapter was stopped "
4103 "because it overheated.\n",
4106 if (thstat & E1000_THSTAT_LINK_THROTTLE) {
4108 "igb: %s The network "
4109 "adapter supported "
4119 /* Links status message must follow this format */
4120 printk(KERN_INFO "igb: %s NIC Link is Down\n",
4122 netif_carrier_off(netdev);
4123 netif_tx_stop_all_queues(netdev);
4125 igb_ping_all_vfs(adapter);
4127 /* link state has changed, schedule phy info update */
4128 if (!test_bit(__IGB_DOWN, &adapter->state))
4129 mod_timer(&adapter->phy_info_timer,
4130 round_jiffies(jiffies + 2 * HZ));
4132 pm_schedule_suspend(netdev->dev.parent,
4137 igb_update_stats(adapter);
4139 for (i = 0; i < adapter->num_tx_queues; i++) {
4140 struct igb_ring *tx_ring = adapter->tx_ring[i];
4141 if (!netif_carrier_ok(netdev)) {
4142 /* We've lost link, so the controller stops DMA,
4143 * but we've got queued Tx work that's never going
4144 * to get done, so reset controller to flush Tx.
4145 * (Do the reset outside of interrupt context). */
4146 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
4147 adapter->tx_timeout_count++;
4148 schedule_work(&adapter->reset_task);
4149 /* return immediately since reset is imminent */
4154 /* Force detection of hung controller every watchdog period */
4155 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
4158 /* Cause software interrupt to ensure rx ring is cleaned */
4159 if (adapter->msix_entries) {
4161 for (i = 0; i < adapter->num_q_vectors; i++)
4162 eics |= adapter->q_vector[i]->eims_value;
4163 E1000_WRITE_REG(hw, E1000_EICS, eics);
4165 E1000_WRITE_REG(hw, E1000_ICS, E1000_ICS_RXDMT0);
4168 igb_spoof_check(adapter);
4170 /* Reset the timer */
4171 if (!test_bit(__IGB_DOWN, &adapter->state))
4172 mod_timer(&adapter->watchdog_timer,
4173 round_jiffies(jiffies + 2 * HZ));
4176 static void igb_dma_err_task(struct work_struct *work)
4178 struct igb_adapter *adapter = container_of(work,
4182 struct e1000_hw *hw = &adapter->hw;
4183 struct net_device *netdev = adapter->netdev;
4187 hgptc = E1000_READ_REG(hw, E1000_HGPTC);
4188 if (hgptc) /* If incrementing then no need for the check below */
4189 goto dma_timer_reset;
4191 * Check to see if a bad DMA write target from an errant or
4192 * malicious VF has caused a PCIe error. If so then we can
4193 * issue a VFLR to the offending VF(s) and then resume without
4194 * requesting a full slot reset.
4197 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4198 ciaa = (vf << 16) | 0x80000000;
4199 /* 32 bit read so align, we really want status at offset 6 */
4200 ciaa |= PCI_COMMAND;
4201 E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4202 ciad = E1000_READ_REG(hw, E1000_CIAD);
4204 /* disable debug mode asap after reading data */
4205 E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4206 /* Get the upper 16 bits which will be the PCI status reg */
4208 if (ciad & (PCI_STATUS_REC_MASTER_ABORT |
4209 PCI_STATUS_REC_TARGET_ABORT |
4210 PCI_STATUS_SIG_SYSTEM_ERROR)) {
4211 netdev_err(netdev, "VF %d suffered error\n", vf);
4213 ciaa = (vf << 16) | 0x80000000;
4215 E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4216 ciad = 0x00008000; /* VFLR */
4217 E1000_WRITE_REG(hw, E1000_CIAD, ciad);
4219 E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4223 /* Reset the timer */
4224 if (!test_bit(__IGB_DOWN, &adapter->state))
4225 mod_timer(&adapter->dma_err_timer,
4226 round_jiffies(jiffies + HZ / 10));
4230 * igb_dma_err_timer - Timer Call-back
4231 * @data: pointer to adapter cast into an unsigned long
4233 static void igb_dma_err_timer(unsigned long data)
4235 struct igb_adapter *adapter = (struct igb_adapter *)data;
4236 /* Do the rest outside of interrupt context */
4237 schedule_work(&adapter->dma_err_task);
4240 enum latency_range {
4244 latency_invalid = 255
4248 * igb_update_ring_itr - update the dynamic ITR value based on packet size
4250 * Stores a new ITR value based on strictly on packet size. This
4251 * algorithm is less sophisticated than that used in igb_update_itr,
4252 * due to the difficulty of synchronizing statistics across multiple
4253 * receive rings. The divisors and thresholds used by this function
4254 * were determined based on theoretical maximum wire speed and testing
4255 * data, in order to minimize response time while increasing bulk
4257 * This functionality is controlled by the InterruptThrottleRate module
4258 * parameter (see igb_param.c)
4259 * NOTE: This function is called only when operating in a multiqueue
4260 * receive environment.
4261 * @q_vector: pointer to q_vector
4263 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
4265 int new_val = q_vector->itr_val;
4266 int avg_wire_size = 0;
4267 struct igb_adapter *adapter = q_vector->adapter;
4268 unsigned int packets;
4270 /* For non-gigabit speeds, just fix the interrupt rate at 4000
4271 * ints/sec - ITR timer value of 120 ticks.
4273 if (adapter->link_speed != SPEED_1000) {
4274 new_val = IGB_4K_ITR;
4278 packets = q_vector->rx.total_packets;
4280 avg_wire_size = q_vector->rx.total_bytes / packets;
4282 packets = q_vector->tx.total_packets;
4284 avg_wire_size = max_t(u32, avg_wire_size,
4285 q_vector->tx.total_bytes / packets);
4287 /* if avg_wire_size isn't set no work was done */
4291 /* Add 24 bytes to size to account for CRC, preamble, and gap */
4292 avg_wire_size += 24;
4294 /* Don't starve jumbo frames */
4295 avg_wire_size = min(avg_wire_size, 3000);
4297 /* Give a little boost to mid-size frames */
4298 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
4299 new_val = avg_wire_size / 3;
4301 new_val = avg_wire_size / 2;
4303 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4304 if (new_val < IGB_20K_ITR &&
4305 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4306 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4307 new_val = IGB_20K_ITR;
4310 if (new_val != q_vector->itr_val) {
4311 q_vector->itr_val = new_val;
4312 q_vector->set_itr = 1;
4315 q_vector->rx.total_bytes = 0;
4316 q_vector->rx.total_packets = 0;
4317 q_vector->tx.total_bytes = 0;
4318 q_vector->tx.total_packets = 0;
4322 * igb_update_itr - update the dynamic ITR value based on statistics
4323 * Stores a new ITR value based on packets and byte
4324 * counts during the last interrupt. The advantage of per interrupt
4325 * computation is faster updates and more accurate ITR for the current
4326 * traffic pattern. Constants in this function were computed
4327 * based on theoretical maximum wire speed and thresholds were set based
4328 * on testing data as well as attempting to minimize response time
4329 * while increasing bulk throughput.
4330 * this functionality is controlled by the InterruptThrottleRate module
4331 * parameter (see igb_param.c)
4332 * NOTE: These calculations are only valid when operating in a single-
4333 * queue environment.
4334 * @q_vector: pointer to q_vector
4335 * @ring_container: ring info to update the itr for
4337 static void igb_update_itr(struct igb_q_vector *q_vector,
4338 struct igb_ring_container *ring_container)
4340 unsigned int packets = ring_container->total_packets;
4341 unsigned int bytes = ring_container->total_bytes;
4342 u8 itrval = ring_container->itr;
4344 /* no packets, exit with status unchanged */
4349 case lowest_latency:
4350 /* handle TSO and jumbo frames */
4351 if (bytes/packets > 8000)
4352 itrval = bulk_latency;
4353 else if ((packets < 5) && (bytes > 512))
4354 itrval = low_latency;
4356 case low_latency: /* 50 usec aka 20000 ints/s */
4357 if (bytes > 10000) {
4358 /* this if handles the TSO accounting */
4359 if (bytes/packets > 8000) {
4360 itrval = bulk_latency;
4361 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
4362 itrval = bulk_latency;
4363 } else if ((packets > 35)) {
4364 itrval = lowest_latency;
4366 } else if (bytes/packets > 2000) {
4367 itrval = bulk_latency;
4368 } else if (packets <= 2 && bytes < 512) {
4369 itrval = lowest_latency;
4372 case bulk_latency: /* 250 usec aka 4000 ints/s */
4373 if (bytes > 25000) {
4375 itrval = low_latency;
4376 } else if (bytes < 1500) {
4377 itrval = low_latency;
4382 /* clear work counters since we have the values we need */
4383 ring_container->total_bytes = 0;
4384 ring_container->total_packets = 0;
4386 /* write updated itr to ring container */
4387 ring_container->itr = itrval;
4390 static void igb_set_itr(struct igb_q_vector *q_vector)
4392 struct igb_adapter *adapter = q_vector->adapter;
4393 u32 new_itr = q_vector->itr_val;
4396 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4397 if (adapter->link_speed != SPEED_1000) {
4399 new_itr = IGB_4K_ITR;
4403 igb_update_itr(q_vector, &q_vector->tx);
4404 igb_update_itr(q_vector, &q_vector->rx);
4406 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
4408 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4409 if (current_itr == lowest_latency &&
4410 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4411 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4412 current_itr = low_latency;
4414 switch (current_itr) {
4415 /* counts and packets in update_itr are dependent on these numbers */
4416 case lowest_latency:
4417 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
4420 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
4423 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */
4430 if (new_itr != q_vector->itr_val) {
4431 /* this attempts to bias the interrupt rate towards Bulk
4432 * by adding intermediate steps when interrupt rate is
4434 new_itr = new_itr > q_vector->itr_val ?
4435 max((new_itr * q_vector->itr_val) /
4436 (new_itr + (q_vector->itr_val >> 2)),
4439 /* Don't write the value here; it resets the adapter's
4440 * internal timer, and causes us to delay far longer than
4441 * we should between interrupts. Instead, we write the ITR
4442 * value at the beginning of the next interrupt so the timing
4443 * ends up being correct.
4445 q_vector->itr_val = new_itr;
4446 q_vector->set_itr = 1;
4450 void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
4451 u32 type_tucmd, u32 mss_l4len_idx)
4453 struct e1000_adv_tx_context_desc *context_desc;
4454 u16 i = tx_ring->next_to_use;
4456 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
4459 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
4461 /* set bits to identify this as an advanced context descriptor */
4462 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
4464 /* For 82575, context index must be unique per ring. */
4465 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4466 mss_l4len_idx |= tx_ring->reg_idx << 4;
4468 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
4469 context_desc->seqnum_seed = 0;
4470 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
4471 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
4474 static int igb_tso(struct igb_ring *tx_ring,
4475 struct igb_tx_buffer *first,
4479 struct sk_buff *skb = first->skb;
4480 u32 vlan_macip_lens, type_tucmd;
4481 u32 mss_l4len_idx, l4len;
4483 if (!skb_is_gso(skb))
4484 #endif /* NETIF_F_TSO */
4488 if (skb_header_cloned(skb)) {
4489 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
4494 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4495 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
4497 if (first->protocol == __constant_htons(ETH_P_IP)) {
4498 struct iphdr *iph = ip_hdr(skb);
4501 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4505 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4506 first->tx_flags |= IGB_TX_FLAGS_TSO |
4510 } else if (skb_is_gso_v6(skb)) {
4511 ipv6_hdr(skb)->payload_len = 0;
4512 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4513 &ipv6_hdr(skb)->daddr,
4515 first->tx_flags |= IGB_TX_FLAGS_TSO |
4520 /* compute header lengths */
4521 l4len = tcp_hdrlen(skb);
4522 *hdr_len = skb_transport_offset(skb) + l4len;
4524 /* update gso size and bytecount with header size */
4525 first->gso_segs = skb_shinfo(skb)->gso_segs;
4526 first->bytecount += (first->gso_segs - 1) * *hdr_len;
4529 mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
4530 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
4532 /* VLAN MACLEN IPLEN */
4533 vlan_macip_lens = skb_network_header_len(skb);
4534 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4535 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4537 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4540 #endif /* NETIF_F_TSO */
4543 static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
4545 struct sk_buff *skb = first->skb;
4546 u32 vlan_macip_lens = 0;
4547 u32 mss_l4len_idx = 0;
4550 if (skb->ip_summed != CHECKSUM_PARTIAL) {
4551 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
4555 switch (first->protocol) {
4556 case __constant_htons(ETH_P_IP):
4557 vlan_macip_lens |= skb_network_header_len(skb);
4558 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4559 l4_hdr = ip_hdr(skb)->protocol;
4561 #ifdef NETIF_F_IPV6_CSUM
4562 case __constant_htons(ETH_P_IPV6):
4563 vlan_macip_lens |= skb_network_header_len(skb);
4564 l4_hdr = ipv6_hdr(skb)->nexthdr;
4568 if (unlikely(net_ratelimit())) {
4569 dev_warn(tx_ring->dev,
4570 "partial checksum but proto=%x!\n",
4578 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4579 mss_l4len_idx = tcp_hdrlen(skb) <<
4580 E1000_ADVTXD_L4LEN_SHIFT;
4584 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4585 mss_l4len_idx = sizeof(struct sctphdr) <<
4586 E1000_ADVTXD_L4LEN_SHIFT;
4590 mss_l4len_idx = sizeof(struct udphdr) <<
4591 E1000_ADVTXD_L4LEN_SHIFT;
4594 if (unlikely(net_ratelimit())) {
4595 dev_warn(tx_ring->dev,
4596 "partial checksum but l4 proto=%x!\n",
4602 /* update TX checksum flag */
4603 first->tx_flags |= IGB_TX_FLAGS_CSUM;
4606 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4607 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4609 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4612 static __le32 igb_tx_cmd_type(u32 tx_flags)
4614 /* set type for advanced descriptor with frame checksum insertion */
4615 __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA |
4616 E1000_ADVTXD_DCMD_IFCS |
4617 E1000_ADVTXD_DCMD_DEXT);
4619 /* set HW vlan bit if vlan is present */
4620 if (tx_flags & IGB_TX_FLAGS_VLAN)
4621 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE);
4623 /* set timestamp bit if present */
4624 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4625 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP);
4627 /* set segmentation bits for TSO */
4628 if (tx_flags & IGB_TX_FLAGS_TSO)
4629 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE);
4634 static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
4635 union e1000_adv_tx_desc *tx_desc,
4636 u32 tx_flags, unsigned int paylen)
4638 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
4640 /* 82575 requires a unique index per ring if any offload is enabled */
4641 if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) &&
4642 test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4643 olinfo_status |= tx_ring->reg_idx << 4;
4645 /* insert L4 checksum */
4646 if (tx_flags & IGB_TX_FLAGS_CSUM) {
4647 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4649 /* insert IPv4 checksum */
4650 if (tx_flags & IGB_TX_FLAGS_IPV4)
4651 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4654 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4658 * The largest size we can write to the descriptor is 65535. In order to
4659 * maintain a power of two alignment we have to limit ourselves to 32K.
4661 #define IGB_MAX_TXD_PWR 15
4662 #define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
4664 static void igb_tx_map(struct igb_ring *tx_ring,
4665 struct igb_tx_buffer *first,
4668 struct sk_buff *skb = first->skb;
4669 struct igb_tx_buffer *tx_buffer;
4670 union e1000_adv_tx_desc *tx_desc;
4672 #ifdef MAX_SKB_FRAGS
4673 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
4674 unsigned int data_len = skb->data_len;
4676 unsigned int size = skb_headlen(skb);
4677 unsigned int paylen = skb->len - hdr_len;
4679 u32 tx_flags = first->tx_flags;
4680 u16 i = tx_ring->next_to_use;
4682 tx_desc = IGB_TX_DESC(tx_ring, i);
4684 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen);
4685 cmd_type = igb_tx_cmd_type(tx_flags);
4687 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4688 if (dma_mapping_error(tx_ring->dev, dma))
4691 /* record length, and DMA address */
4692 dma_unmap_len_set(first, len, size);
4693 dma_unmap_addr_set(first, dma, dma);
4694 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4696 #ifdef MAX_SKB_FRAGS
4699 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
4700 tx_desc->read.cmd_type_len =
4701 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD);
4705 if (i == tx_ring->count) {
4706 tx_desc = IGB_TX_DESC(tx_ring, 0);
4710 dma += IGB_MAX_DATA_PER_TXD;
4711 size -= IGB_MAX_DATA_PER_TXD;
4713 tx_desc->read.olinfo_status = 0;
4714 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4717 #ifdef MAX_SKB_FRAGS
4718 if (likely(!data_len))
4721 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
4725 if (i == tx_ring->count) {
4726 tx_desc = IGB_TX_DESC(tx_ring, 0);
4730 size = skb_frag_size(frag);
4733 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
4735 if (dma_mapping_error(tx_ring->dev, dma))
4738 tx_buffer = &tx_ring->tx_buffer_info[i];
4739 dma_unmap_len_set(tx_buffer, len, size);
4740 dma_unmap_addr_set(tx_buffer, dma, dma);
4742 tx_desc->read.olinfo_status = 0;
4743 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4748 #endif /* MAX_SKB_FRAGS */
4750 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
4751 #endif /* CONFIG_BQL */
4753 /* write last descriptor with RS and EOP bits */
4754 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD);
4755 tx_desc->read.cmd_type_len = cmd_type;
4757 /* set the timestamp */
4758 first->time_stamp = jiffies;
4761 * Force memory writes to complete before letting h/w know there
4762 * are new descriptors to fetch. (Only applicable for weak-ordered
4763 * memory model archs, such as IA-64).
4765 * We also need this memory barrier to make certain all of the
4766 * status bits have been updated before next_to_watch is written.
4770 /* set next_to_watch value indicating a packet is present */
4771 first->next_to_watch = tx_desc;
4774 if (i == tx_ring->count)
4777 tx_ring->next_to_use = i;
4779 writel(i, tx_ring->tail);
4781 /* we need this if more than one processor can write to our tail
4782 * at a time, it syncronizes IO on IA64/Altix systems */
4788 dev_err(tx_ring->dev, "TX DMA map failed\n");
4790 /* clear dma mappings for failed tx_buffer_info map */
4792 tx_buffer= &tx_ring->tx_buffer_info[i];
4793 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer);
4794 if (tx_buffer == first)
4801 tx_ring->next_to_use = i;
4804 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4806 struct net_device *netdev = netdev_ring(tx_ring);
4808 if (netif_is_multiqueue(netdev))
4809 netif_stop_subqueue(netdev, ring_queue_index(tx_ring));
4811 netif_stop_queue(netdev);
4813 /* Herbert's original patch had:
4814 * smp_mb__after_netif_stop_queue();
4815 * but since that doesn't exist yet, just open code it. */
4818 /* We need to check again in a case another CPU has just
4819 * made room available. */
4820 if (igb_desc_unused(tx_ring) < size)
4824 if (netif_is_multiqueue(netdev))
4825 netif_wake_subqueue(netdev, ring_queue_index(tx_ring));
4827 netif_wake_queue(netdev);
4829 tx_ring->tx_stats.restart_queue++;
4834 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4836 if (igb_desc_unused(tx_ring) >= size)
4838 return __igb_maybe_stop_tx(tx_ring, size);
4841 netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4842 struct igb_ring *tx_ring)
4844 struct igb_tx_buffer *first;
4847 __be16 protocol = vlan_get_protocol(skb);
4850 /* need: 1 descriptor per page,
4851 * + 2 desc gap to keep tail from touching head,
4852 * + 1 desc for skb->data,
4853 * + 1 desc for context descriptor,
4854 * otherwise try next time */
4855 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
4856 /* this is a hard error */
4857 return NETDEV_TX_BUSY;
4860 /* record the location of the first descriptor for this packet */
4861 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4863 first->bytecount = skb->len;
4864 first->gso_segs = 1;
4866 #ifdef HAVE_HW_TIME_STAMP
4867 #ifdef SKB_SHARED_TX_IS_UNION
4868 if (unlikely(skb_shinfo(skb)->tx_flags.flags & SKBTX_HW_TSTAMP)) {
4869 skb_shinfo(skb)->tx_flags.flags |= SKBTX_IN_PROGRESS;
4870 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4873 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4874 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4875 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4880 if (vlan_tx_tag_present(skb)) {
4881 tx_flags |= IGB_TX_FLAGS_VLAN;
4882 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4885 /* record initial flags and protocol */
4886 first->tx_flags = tx_flags;
4887 first->protocol = protocol;
4889 tso = igb_tso(tx_ring, first, &hdr_len);
4893 igb_tx_csum(tx_ring, first);
4895 igb_tx_map(tx_ring, first, hdr_len);
4897 #ifndef HAVE_TRANS_START_IN_QUEUE
4898 netdev_ring(tx_ring)->trans_start = jiffies;
4901 /* Make sure there is space in the ring for the next send. */
4902 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
4904 return NETDEV_TX_OK;
4907 igb_unmap_and_free_tx_resource(tx_ring, first);
4909 return NETDEV_TX_OK;
4913 static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4914 struct sk_buff *skb)
4916 unsigned int r_idx = skb->queue_mapping;
4918 if (r_idx >= adapter->num_tx_queues)
4919 r_idx = r_idx % adapter->num_tx_queues;
4921 return adapter->tx_ring[r_idx];
4924 #define igb_tx_queue_mapping(_adapter, _skb) (_adapter)->tx_ring[0]
4927 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4928 struct net_device *netdev)
4930 struct igb_adapter *adapter = netdev_priv(netdev);
4932 if (test_bit(__IGB_DOWN, &adapter->state)) {
4933 dev_kfree_skb_any(skb);
4934 return NETDEV_TX_OK;
4937 if (skb->len <= 0) {
4938 dev_kfree_skb_any(skb);
4939 return NETDEV_TX_OK;
4943 * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4944 * in order to meet this minimum size requirement.
4946 if (skb->len < 17) {
4947 if (skb_padto(skb, 17))
4948 return NETDEV_TX_OK;
4952 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
4956 * igb_tx_timeout - Respond to a Tx Hang
4957 * @netdev: network interface device structure
4959 static void igb_tx_timeout(struct net_device *netdev)
4961 struct igb_adapter *adapter = netdev_priv(netdev);
4962 struct e1000_hw *hw = &adapter->hw;
4964 /* Do the reset outside of interrupt context */
4965 adapter->tx_timeout_count++;
4967 if (hw->mac.type >= e1000_82580)
4968 hw->dev_spec._82575.global_device_reset = true;
4970 schedule_work(&adapter->reset_task);
4971 E1000_WRITE_REG(hw, E1000_EICS,
4972 (adapter->eims_enable_mask & ~adapter->eims_other));
4975 static void igb_reset_task(struct work_struct *work)
4977 struct igb_adapter *adapter;
4978 adapter = container_of(work, struct igb_adapter, reset_task);
4980 igb_reinit_locked(adapter);
4984 * igb_get_stats - Get System Network Statistics
4985 * @netdev: network interface device structure
4987 * Returns the address of the device statistics structure.
4988 * The statistics are updated here and also from the timer callback.
4990 static struct net_device_stats *igb_get_stats(struct net_device *netdev)
4992 struct igb_adapter *adapter = netdev_priv(netdev);
4994 if (!test_bit(__IGB_RESETTING, &adapter->state))
4995 igb_update_stats(adapter);
4997 #ifdef HAVE_NETDEV_STATS_IN_NETDEV
4998 /* only return the current stats */
4999 return &netdev->stats;
5001 /* only return the current stats */
5002 return &adapter->net_stats;
5003 #endif /* HAVE_NETDEV_STATS_IN_NETDEV */
5007 * igb_change_mtu - Change the Maximum Transfer Unit
5008 * @netdev: network interface device structure
5009 * @new_mtu: new value for maximum frame size
5011 * Returns 0 on success, negative on failure
5013 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
5015 struct igb_adapter *adapter = netdev_priv(netdev);
5016 struct pci_dev *pdev = adapter->pdev;
5017 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
5018 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
5019 u32 rx_buffer_len, i;
5022 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
5023 dev_err(pci_dev_to_dev(pdev), "Invalid MTU setting\n");
5027 #define MAX_STD_JUMBO_FRAME_SIZE 9238
5028 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
5029 dev_err(pci_dev_to_dev(pdev), "MTU > 9216 not supported.\n");
5033 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
5034 usleep_range(1000, 2000);
5036 /* igb_down has a dependency on max_frame_size */
5037 adapter->max_frame_size = max_frame;
5039 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
5040 #ifdef IGB_PER_PKT_TIMESTAMP
5041 if (adapter->hw.mac.type >= e1000_82580)
5042 max_frame += IGB_TS_HDR_LEN;
5046 * RLPML prevents us from receiving a frame larger than max_frame so
5047 * it is safe to just set the rx_buffer_len to max_frame without the
5048 * risk of an skb over panic.
5050 if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
5051 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
5053 rx_buffer_len = max_frame;
5056 if (netif_running(netdev))
5059 dev_info(pci_dev_to_dev(pdev), "changing MTU from %d to %d\n",
5060 netdev->mtu, new_mtu);
5061 netdev->mtu = new_mtu;
5063 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
5064 for (i = 0; i < adapter->num_rx_queues; i++)
5065 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
5068 if (netif_running(netdev))
5073 clear_bit(__IGB_RESETTING, &adapter->state);
5079 * igb_update_stats - Update the board statistics counters
5080 * @adapter: board private structure
5083 void igb_update_stats(struct igb_adapter *adapter)
5085 #ifdef HAVE_NETDEV_STATS_IN_NETDEV
5086 struct net_device_stats *net_stats = &adapter->netdev->stats;
5088 struct net_device_stats *net_stats = &adapter->net_stats;
5089 #endif /* HAVE_NETDEV_STATS_IN_NETDEV */
5090 struct e1000_hw *hw = &adapter->hw;
5092 struct pci_dev *pdev = adapter->pdev;
5099 u32 flushed = 0, coal = 0, recycled = 0;
5100 struct igb_q_vector *q_vector;
5103 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
5106 * Prevent stats update while adapter is being reset, or if the pci
5107 * connection is down.
5109 if (adapter->link_speed == 0)
5112 if (pci_channel_offline(pdev))
5117 for (i = 0; i < adapter->num_q_vectors; i++) {
5118 q_vector = adapter->q_vector[i];
5119 if (!q_vector || !q_vector->lrolist)
5121 flushed += q_vector->lrolist->stats.flushed;
5122 coal += q_vector->lrolist->stats.coal;
5123 recycled += q_vector->lrolist->stats.recycled;
5125 adapter->lro_stats.flushed = flushed;
5126 adapter->lro_stats.coal = coal;
5127 adapter->lro_stats.recycled = recycled;
5132 for (i = 0; i < adapter->num_rx_queues; i++) {
5133 u32 rqdpc_tmp = E1000_READ_REG(hw, E1000_RQDPC(i)) & 0x0FFF;
5134 struct igb_ring *ring = adapter->rx_ring[i];
5135 ring->rx_stats.drops += rqdpc_tmp;
5136 net_stats->rx_fifo_errors += rqdpc_tmp;
5137 #ifdef CONFIG_IGB_VMDQ_NETDEV
5138 if (!ring->vmdq_netdev) {
5139 bytes += ring->rx_stats.bytes;
5140 packets += ring->rx_stats.packets;
5143 bytes += ring->rx_stats.bytes;
5144 packets += ring->rx_stats.packets;
5148 net_stats->rx_bytes = bytes;
5149 net_stats->rx_packets = packets;
5153 for (i = 0; i < adapter->num_tx_queues; i++) {
5154 struct igb_ring *ring = adapter->tx_ring[i];
5155 #ifdef CONFIG_IGB_VMDQ_NETDEV
5156 if (!ring->vmdq_netdev) {
5157 bytes += ring->tx_stats.bytes;
5158 packets += ring->tx_stats.packets;
5161 bytes += ring->tx_stats.bytes;
5162 packets += ring->tx_stats.packets;
5165 net_stats->tx_bytes = bytes;
5166 net_stats->tx_packets = packets;
5168 /* read stats registers */
5169 adapter->stats.crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
5170 adapter->stats.gprc += E1000_READ_REG(hw, E1000_GPRC);
5171 adapter->stats.gorc += E1000_READ_REG(hw, E1000_GORCL);
5172 E1000_READ_REG(hw, E1000_GORCH); /* clear GORCL */
5173 adapter->stats.bprc += E1000_READ_REG(hw, E1000_BPRC);
5174 adapter->stats.mprc += E1000_READ_REG(hw, E1000_MPRC);
5175 adapter->stats.roc += E1000_READ_REG(hw, E1000_ROC);
5177 adapter->stats.prc64 += E1000_READ_REG(hw, E1000_PRC64);
5178 adapter->stats.prc127 += E1000_READ_REG(hw, E1000_PRC127);
5179 adapter->stats.prc255 += E1000_READ_REG(hw, E1000_PRC255);
5180 adapter->stats.prc511 += E1000_READ_REG(hw, E1000_PRC511);
5181 adapter->stats.prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
5182 adapter->stats.prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
5183 adapter->stats.symerrs += E1000_READ_REG(hw, E1000_SYMERRS);
5184 adapter->stats.sec += E1000_READ_REG(hw, E1000_SEC);
5186 mpc = E1000_READ_REG(hw, E1000_MPC);
5187 adapter->stats.mpc += mpc;
5188 net_stats->rx_fifo_errors += mpc;
5189 adapter->stats.scc += E1000_READ_REG(hw, E1000_SCC);
5190 adapter->stats.ecol += E1000_READ_REG(hw, E1000_ECOL);
5191 adapter->stats.mcc += E1000_READ_REG(hw, E1000_MCC);
5192 adapter->stats.latecol += E1000_READ_REG(hw, E1000_LATECOL);
5193 adapter->stats.dc += E1000_READ_REG(hw, E1000_DC);
5194 adapter->stats.rlec += E1000_READ_REG(hw, E1000_RLEC);
5195 adapter->stats.xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
5196 adapter->stats.xontxc += E1000_READ_REG(hw, E1000_XONTXC);
5197 adapter->stats.xoffrxc += E1000_READ_REG(hw, E1000_XOFFRXC);
5198 adapter->stats.xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
5199 adapter->stats.fcruc += E1000_READ_REG(hw, E1000_FCRUC);
5200 adapter->stats.gptc += E1000_READ_REG(hw, E1000_GPTC);
5201 adapter->stats.gotc += E1000_READ_REG(hw, E1000_GOTCL);
5202 E1000_READ_REG(hw, E1000_GOTCH); /* clear GOTCL */
5203 adapter->stats.rnbc += E1000_READ_REG(hw, E1000_RNBC);
5204 adapter->stats.ruc += E1000_READ_REG(hw, E1000_RUC);
5205 adapter->stats.rfc += E1000_READ_REG(hw, E1000_RFC);
5206 adapter->stats.rjc += E1000_READ_REG(hw, E1000_RJC);
5207 adapter->stats.tor += E1000_READ_REG(hw, E1000_TORH);
5208 adapter->stats.tot += E1000_READ_REG(hw, E1000_TOTH);
5209 adapter->stats.tpr += E1000_READ_REG(hw, E1000_TPR);
5211 adapter->stats.ptc64 += E1000_READ_REG(hw, E1000_PTC64);
5212 adapter->stats.ptc127 += E1000_READ_REG(hw, E1000_PTC127);
5213 adapter->stats.ptc255 += E1000_READ_REG(hw, E1000_PTC255);
5214 adapter->stats.ptc511 += E1000_READ_REG(hw, E1000_PTC511);
5215 adapter->stats.ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
5216 adapter->stats.ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
5218 adapter->stats.mptc += E1000_READ_REG(hw, E1000_MPTC);
5219 adapter->stats.bptc += E1000_READ_REG(hw, E1000_BPTC);
5221 adapter->stats.tpt += E1000_READ_REG(hw, E1000_TPT);
5222 adapter->stats.colc += E1000_READ_REG(hw, E1000_COLC);
5224 adapter->stats.algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
5225 /* read internal phy sepecific stats */
5226 reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
5227 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
5228 adapter->stats.rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
5229 adapter->stats.tncrs += E1000_READ_REG(hw, E1000_TNCRS);
5232 adapter->stats.tsctc += E1000_READ_REG(hw, E1000_TSCTC);
5233 adapter->stats.tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
5235 adapter->stats.iac += E1000_READ_REG(hw, E1000_IAC);
5236 adapter->stats.icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
5237 adapter->stats.icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
5238 adapter->stats.icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
5239 adapter->stats.ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
5240 adapter->stats.ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
5241 adapter->stats.ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
5242 adapter->stats.ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
5243 adapter->stats.icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
5245 /* Fill out the OS statistics structure */
5246 net_stats->multicast = adapter->stats.mprc;
5247 net_stats->collisions = adapter->stats.colc;
5251 /* RLEC on some newer hardware can be incorrect so build
5252 * our own version based on RUC and ROC */
5253 net_stats->rx_errors = adapter->stats.rxerrc +
5254 adapter->stats.crcerrs + adapter->stats.algnerrc +
5255 adapter->stats.ruc + adapter->stats.roc +
5256 adapter->stats.cexterr;
5257 net_stats->rx_length_errors = adapter->stats.ruc +
5259 net_stats->rx_crc_errors = adapter->stats.crcerrs;
5260 net_stats->rx_frame_errors = adapter->stats.algnerrc;
5261 net_stats->rx_missed_errors = adapter->stats.mpc;
5264 net_stats->tx_errors = adapter->stats.ecol +
5265 adapter->stats.latecol;
5266 net_stats->tx_aborted_errors = adapter->stats.ecol;
5267 net_stats->tx_window_errors = adapter->stats.latecol;
5268 net_stats->tx_carrier_errors = adapter->stats.tncrs;
5270 /* Tx Dropped needs to be maintained elsewhere */
5273 if (hw->phy.media_type == e1000_media_type_copper) {
5274 if ((adapter->link_speed == SPEED_1000) &&
5275 (!e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
5276 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
5277 adapter->phy_stats.idle_errors += phy_tmp;
5281 /* Management Stats */
5282 adapter->stats.mgptc += E1000_READ_REG(hw, E1000_MGTPTC);
5283 adapter->stats.mgprc += E1000_READ_REG(hw, E1000_MGTPRC);
5284 if (hw->mac.type > e1000_82580) {
5285 adapter->stats.o2bgptc += E1000_READ_REG(hw, E1000_O2BGPTC);
5286 adapter->stats.o2bspc += E1000_READ_REG(hw, E1000_O2BSPC);
5287 adapter->stats.b2ospc += E1000_READ_REG(hw, E1000_B2OSPC);
5288 adapter->stats.b2ogprc += E1000_READ_REG(hw, E1000_B2OGPRC);
5292 static irqreturn_t igb_msix_other(int irq, void *data)
5294 struct igb_adapter *adapter = data;
5295 struct e1000_hw *hw = &adapter->hw;
5296 u32 icr = E1000_READ_REG(hw, E1000_ICR);
5297 /* reading ICR causes bit 31 of EICR to be cleared */
5299 if (icr & E1000_ICR_DRSTA)
5300 schedule_work(&adapter->reset_task);
5302 if (icr & E1000_ICR_DOUTSYNC) {
5303 /* HW is reporting DMA is out of sync */
5304 adapter->stats.doosync++;
5305 /* The DMA Out of Sync is also indication of a spoof event
5306 * in IOV mode. Check the Wrong VM Behavior register to
5307 * see if it is really a spoof event. */
5308 igb_check_wvbr(adapter);
5311 /* Check for a mailbox event */
5312 if (icr & E1000_ICR_VMMB)
5313 igb_msg_task(adapter);
5315 if (icr & E1000_ICR_LSC) {
5316 hw->mac.get_link_status = 1;
5317 /* guard against interrupt when we're going down */
5318 if (!test_bit(__IGB_DOWN, &adapter->state))
5319 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5322 /* Check for MDD event */
5323 if (icr & E1000_ICR_MDDET)
5324 igb_process_mdd_event(adapter);
5326 E1000_WRITE_REG(hw, E1000_EIMS, adapter->eims_other);
5331 static void igb_write_itr(struct igb_q_vector *q_vector)
5333 struct igb_adapter *adapter = q_vector->adapter;
5334 u32 itr_val = q_vector->itr_val & 0x7FFC;
5336 if (!q_vector->set_itr)
5342 if (adapter->hw.mac.type == e1000_82575)
5343 itr_val |= itr_val << 16;
5345 itr_val |= E1000_EITR_CNT_IGNR;
5347 writel(itr_val, q_vector->itr_register);
5348 q_vector->set_itr = 0;
5351 static irqreturn_t igb_msix_ring(int irq, void *data)
5353 struct igb_q_vector *q_vector = data;
5355 /* Write the ITR value calculated from the previous interrupt. */
5356 igb_write_itr(q_vector);
5358 napi_schedule(&q_vector->napi);
5364 static void igb_update_dca(struct igb_q_vector *q_vector)
5366 struct igb_adapter *adapter = q_vector->adapter;
5367 struct e1000_hw *hw = &adapter->hw;
5368 int cpu = get_cpu();
5370 if (q_vector->cpu == cpu)
5373 if (q_vector->tx.ring) {
5374 int q = q_vector->tx.ring->reg_idx;
5375 u32 dca_txctrl = E1000_READ_REG(hw, E1000_DCA_TXCTRL(q));
5376 if (hw->mac.type == e1000_82575) {
5377 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
5378 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
5380 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
5381 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
5382 E1000_DCA_TXCTRL_CPUID_SHIFT_82576;
5384 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
5385 E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(q), dca_txctrl);
5387 if (q_vector->rx.ring) {
5388 int q = q_vector->rx.ring->reg_idx;
5389 u32 dca_rxctrl = E1000_READ_REG(hw, E1000_DCA_RXCTRL(q));
5390 if (hw->mac.type == e1000_82575) {
5391 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
5392 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
5394 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
5395 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
5396 E1000_DCA_RXCTRL_CPUID_SHIFT_82576;
5398 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
5399 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
5400 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
5401 E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(q), dca_rxctrl);
5403 q_vector->cpu = cpu;
5408 static void igb_setup_dca(struct igb_adapter *adapter)
5410 struct e1000_hw *hw = &adapter->hw;
5413 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
5416 /* Always use CB2 mode, difference is masked in the CB driver. */
5417 E1000_WRITE_REG(hw, E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
5419 for (i = 0; i < adapter->num_q_vectors; i++) {
5420 adapter->q_vector[i]->cpu = -1;
5421 igb_update_dca(adapter->q_vector[i]);
5425 static int __igb_notify_dca(struct device *dev, void *data)
5427 struct net_device *netdev = dev_get_drvdata(dev);
5428 struct igb_adapter *adapter = netdev_priv(netdev);
5429 struct pci_dev *pdev = adapter->pdev;
5430 struct e1000_hw *hw = &adapter->hw;
5431 unsigned long event = *(unsigned long *)data;
5434 case DCA_PROVIDER_ADD:
5435 /* if already enabled, don't do it again */
5436 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
5438 if (dca_add_requester(dev) == E1000_SUCCESS) {
5439 adapter->flags |= IGB_FLAG_DCA_ENABLED;
5440 dev_info(pci_dev_to_dev(pdev), "DCA enabled\n");
5441 igb_setup_dca(adapter);
5444 /* Fall Through since DCA is disabled. */
5445 case DCA_PROVIDER_REMOVE:
5446 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
5447 /* without this a class_device is left
5448 * hanging around in the sysfs model */
5449 dca_remove_requester(dev);
5450 dev_info(pci_dev_to_dev(pdev), "DCA disabled\n");
5451 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
5452 E1000_WRITE_REG(hw, E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_DISABLE);
5457 return E1000_SUCCESS;
5460 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
5465 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
5468 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
5470 #endif /* IGB_DCA */
5472 static int igb_vf_configure(struct igb_adapter *adapter, int vf)
5474 unsigned char mac_addr[ETH_ALEN];
5475 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
5476 struct pci_dev *pdev = adapter->pdev;
5477 struct e1000_hw *hw = &adapter->hw;
5478 struct pci_dev *pvfdev;
5479 unsigned int device_id;
5483 random_ether_addr(mac_addr);
5484 igb_set_vf_mac(adapter, vf, mac_addr);
5486 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
5487 switch (adapter->hw.mac.type) {
5489 device_id = IGB_82576_VF_DEV_ID;
5490 /* VF Stride for 82576 is 2 */
5491 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 1)) |
5495 device_id = IGB_I350_VF_DEV_ID;
5496 /* VF Stride for I350 is 4 */
5497 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 2)) |
5506 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5508 if (pvfdev->devfn == thisvf_devfn)
5510 pvfdev = pci_get_device(hw->vendor_id,
5515 adapter->vf_data[vf].vfdev = pvfdev;
5518 "Couldn't find pci dev ptr for VF %4.4x\n",
5520 return pvfdev != NULL;
5526 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
5527 static int igb_find_enabled_vfs(struct igb_adapter *adapter)
5529 struct e1000_hw *hw = &adapter->hw;
5530 struct pci_dev *pdev = adapter->pdev;
5531 struct pci_dev *pvfdev;
5534 unsigned int device_id;
5537 switch (adapter->hw.mac.type) {
5539 device_id = IGB_82576_VF_DEV_ID;
5540 /* VF Stride for 82576 is 2 */
5544 device_id = IGB_I350_VF_DEV_ID;
5545 /* VF Stride for I350 is 4 */
5554 vf_devfn = pdev->devfn + 0x80;
5555 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5557 if (pvfdev->devfn == vf_devfn)
5559 vf_devfn += vf_stride;
5560 pvfdev = pci_get_device(hw->vendor_id,
5568 static int igb_check_vf_assignment(struct igb_adapter *adapter)
5570 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
5572 for (i = 0; i < adapter->vfs_allocated_count; i++) {
5573 if (adapter->vf_data[i].vfdev) {
5574 if (adapter->vf_data[i].vfdev->dev_flags &
5575 PCI_DEV_FLAGS_ASSIGNED)
5583 static void igb_ping_all_vfs(struct igb_adapter *adapter)
5585 struct e1000_hw *hw = &adapter->hw;
5589 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
5590 ping = E1000_PF_CONTROL_MSG;
5591 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
5592 ping |= E1000_VT_MSGTYPE_CTS;
5593 e1000_write_mbx(hw, &ping, 1, i);
5598 * igb_mta_set_ - Set multicast filter table address
5599 * @adapter: pointer to the adapter structure
5600 * @hash_value: determines the MTA register and bit to set
5602 * The multicast table address is a register array of 32-bit registers.
5603 * The hash_value is used to determine what register the bit is in, the
5604 * current value is read, the new bit is OR'd in and the new value is
5605 * written back into the register.
5607 void igb_mta_set(struct igb_adapter *adapter, u32 hash_value)
5609 struct e1000_hw *hw = &adapter->hw;
5610 u32 hash_bit, hash_reg, mta;
5613 * The MTA is a register array of 32-bit registers. It is
5614 * treated like an array of (32*mta_reg_count) bits. We want to
5615 * set bit BitArray[hash_value]. So we figure out what register
5616 * the bit is in, read it, OR in the new bit, then write
5617 * back the new value. The (hw->mac.mta_reg_count - 1) serves as a
5618 * mask to bits 31:5 of the hash value which gives us the
5619 * register we're modifying. The hash bit within that register
5620 * is determined by the lower 5 bits of the hash value.
5622 hash_reg = (hash_value >> 5) & (hw->mac.mta_reg_count - 1);
5623 hash_bit = hash_value & 0x1F;
5625 mta = E1000_READ_REG_ARRAY(hw, E1000_MTA, hash_reg);
5627 mta |= (1 << hash_bit);
5629 E1000_WRITE_REG_ARRAY(hw, E1000_MTA, hash_reg, mta);
5630 E1000_WRITE_FLUSH(hw);
5633 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5636 struct e1000_hw *hw = &adapter->hw;
5637 u32 vmolr = E1000_READ_REG(hw, E1000_VMOLR(vf));
5638 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5640 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
5641 IGB_VF_FLAG_MULTI_PROMISC);
5642 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5644 #ifdef IGB_ENABLE_VF_PROMISC
5645 if (*msgbuf & E1000_VF_SET_PROMISC_UNICAST) {
5646 vmolr |= E1000_VMOLR_ROPE;
5647 vf_data->flags |= IGB_VF_FLAG_UNI_PROMISC;
5648 *msgbuf &= ~E1000_VF_SET_PROMISC_UNICAST;
5651 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
5652 vmolr |= E1000_VMOLR_MPME;
5653 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
5654 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
5657 * if we have hashes and we are clearing a multicast promisc
5658 * flag we need to write the hashes to the MTA as this step
5659 * was previously skipped
5661 if (vf_data->num_vf_mc_hashes > 30) {
5662 vmolr |= E1000_VMOLR_MPME;
5663 } else if (vf_data->num_vf_mc_hashes) {
5665 vmolr |= E1000_VMOLR_ROMPE;
5666 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5667 igb_mta_set(adapter, vf_data->vf_mc_hashes[j]);
5671 E1000_WRITE_REG(hw, E1000_VMOLR(vf), vmolr);
5673 /* there are flags left unprocessed, likely not supported */
5674 if (*msgbuf & E1000_VT_MSGINFO_MASK)
5681 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
5682 u32 *msgbuf, u32 vf)
5684 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5685 u16 *hash_list = (u16 *)&msgbuf[1];
5686 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5689 /* salt away the number of multicast addresses assigned
5690 * to this VF for later use to restore when the PF multi cast
5693 vf_data->num_vf_mc_hashes = n;
5695 /* only up to 30 hash values supported */
5699 /* store the hashes for later use */
5700 for (i = 0; i < n; i++)
5701 vf_data->vf_mc_hashes[i] = hash_list[i];
5703 /* Flush and reset the mta with the new values */
5704 igb_set_rx_mode(adapter->netdev);
5709 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
5711 struct e1000_hw *hw = &adapter->hw;
5712 struct vf_data_storage *vf_data;
5715 for (i = 0; i < adapter->vfs_allocated_count; i++) {
5716 u32 vmolr = E1000_READ_REG(hw, E1000_VMOLR(i));
5717 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5719 vf_data = &adapter->vf_data[i];
5721 if ((vf_data->num_vf_mc_hashes > 30) ||
5722 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5723 vmolr |= E1000_VMOLR_MPME;
5724 } else if (vf_data->num_vf_mc_hashes) {
5725 vmolr |= E1000_VMOLR_ROMPE;
5726 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5727 igb_mta_set(adapter, vf_data->vf_mc_hashes[j]);
5729 E1000_WRITE_REG(hw, E1000_VMOLR(i), vmolr);
5733 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5735 struct e1000_hw *hw = &adapter->hw;
5736 u32 pool_mask, reg, vid;
5740 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5742 /* Find the vlan filter for this id */
5743 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5744 reg = E1000_READ_REG(hw, E1000_VLVF(i));
5746 /* remove the vf from the pool */
5749 /* if pool is empty then remove entry from vfta */
5750 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5751 (reg & E1000_VLVF_VLANID_ENABLE)) {
5753 vid = reg & E1000_VLVF_VLANID_MASK;
5754 igb_vfta_set(adapter, vid, FALSE);
5757 E1000_WRITE_REG(hw, E1000_VLVF(i), reg);
5760 adapter->vf_data[vf].vlans_enabled = 0;
5762 vlan_default = adapter->vf_data[vf].default_vf_vlan_id;
5764 igb_vlvf_set(adapter, vlan_default, true, vf);
5767 s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5769 struct e1000_hw *hw = &adapter->hw;
5772 /* The vlvf table only exists on 82576 hardware and newer */
5773 if (hw->mac.type < e1000_82576)
5776 /* we only need to do this if VMDq is enabled */
5777 if (!adapter->vmdq_pools)
5780 /* Find the vlan filter for this id */
5781 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5782 reg = E1000_READ_REG(hw, E1000_VLVF(i));
5783 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5784 vid == (reg & E1000_VLVF_VLANID_MASK))
5789 if (i == E1000_VLVF_ARRAY_SIZE) {
5790 /* Did not find a matching VLAN ID entry that was
5791 * enabled. Search for a free filter entry, i.e.
5792 * one without the enable bit set
5794 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5795 reg = E1000_READ_REG(hw, E1000_VLVF(i));
5796 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5800 if (i < E1000_VLVF_ARRAY_SIZE) {
5801 /* Found an enabled/available entry */
5802 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5804 /* if !enabled we need to set this up in vfta */
5805 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
5806 /* add VID to filter table */
5807 igb_vfta_set(adapter, vid, TRUE);
5808 reg |= E1000_VLVF_VLANID_ENABLE;
5810 reg &= ~E1000_VLVF_VLANID_MASK;
5812 E1000_WRITE_REG(hw, E1000_VLVF(i), reg);
5814 /* do not modify RLPML for PF devices */
5815 if (vf >= adapter->vfs_allocated_count)
5816 return E1000_SUCCESS;
5818 if (!adapter->vf_data[vf].vlans_enabled) {
5820 reg = E1000_READ_REG(hw, E1000_VMOLR(vf));
5821 size = reg & E1000_VMOLR_RLPML_MASK;
5823 reg &= ~E1000_VMOLR_RLPML_MASK;
5825 E1000_WRITE_REG(hw, E1000_VMOLR(vf), reg);
5828 adapter->vf_data[vf].vlans_enabled++;
5831 if (i < E1000_VLVF_ARRAY_SIZE) {
5832 /* remove vf from the pool */
5833 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5834 /* if pool is empty then remove entry from vfta */
5835 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5837 igb_vfta_set(adapter, vid, FALSE);
5839 E1000_WRITE_REG(hw, E1000_VLVF(i), reg);
5841 /* do not modify RLPML for PF devices */
5842 if (vf >= adapter->vfs_allocated_count)
5843 return E1000_SUCCESS;
5845 adapter->vf_data[vf].vlans_enabled--;
5846 if (!adapter->vf_data[vf].vlans_enabled) {
5848 reg = E1000_READ_REG(hw, E1000_VMOLR(vf));
5849 size = reg & E1000_VMOLR_RLPML_MASK;
5851 reg &= ~E1000_VMOLR_RLPML_MASK;
5853 E1000_WRITE_REG(hw, E1000_VMOLR(vf), reg);
5857 return E1000_SUCCESS;
5861 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5863 struct e1000_hw *hw = &adapter->hw;
5866 E1000_WRITE_REG(hw, E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5868 E1000_WRITE_REG(hw, E1000_VMVIR(vf), 0);
5871 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5872 int vf, u16 vlan, u8 qos)
5875 struct igb_adapter *adapter = netdev_priv(netdev);
5877 /* VLAN IDs accepted range 0-4094 */
5878 if ((vf >= adapter->vfs_allocated_count) || (vlan > VLAN_VID_MASK-1) || (qos > 7))
5881 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5884 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5885 igb_set_vmolr(adapter, vf, !vlan);
5886 adapter->vf_data[vf].pf_vlan = vlan;
5887 adapter->vf_data[vf].pf_qos = qos;
5888 igb_set_vf_vlan_strip(adapter, vf, true);
5889 dev_info(&adapter->pdev->dev,
5890 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5891 if (test_bit(__IGB_DOWN, &adapter->state)) {
5892 dev_warn(&adapter->pdev->dev,
5893 "The VF VLAN has been set,"
5894 " but the PF device is not up.\n");
5895 dev_warn(&adapter->pdev->dev,
5896 "Bring the PF device up before"
5897 " attempting to use the VF device.\n");
5900 if (adapter->vf_data[vf].pf_vlan)
5901 dev_info(&adapter->pdev->dev,
5902 "Clearing VLAN on VF %d\n", vf);
5903 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5905 igb_set_vmvir(adapter, vlan, vf);
5906 igb_set_vmolr(adapter, vf, true);
5907 igb_set_vf_vlan_strip(adapter, vf, false);
5908 adapter->vf_data[vf].pf_vlan = 0;
5909 adapter->vf_data[vf].pf_qos = 0;
5916 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5918 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5919 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5922 igb_set_vf_vlan_strip(adapter, vf, true);
5924 igb_set_vf_vlan_strip(adapter, vf, false);
5926 return igb_vlvf_set(adapter, vid, add, vf);
5929 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
5931 struct e1000_hw *hw = &adapter->hw;
5933 /* clear flags except flag that the PF has set the MAC */
5934 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
5935 adapter->vf_data[vf].last_nack = jiffies;
5937 /* reset offloads to defaults */
5938 igb_set_vmolr(adapter, vf, true);
5940 /* reset vlans for device */
5941 igb_clear_vf_vfta(adapter, vf);
5943 if (adapter->vf_data[vf].pf_vlan)
5944 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5945 adapter->vf_data[vf].pf_vlan,
5946 adapter->vf_data[vf].pf_qos);
5948 igb_clear_vf_vfta(adapter, vf);
5951 /* reset multicast table array for vf */
5952 adapter->vf_data[vf].num_vf_mc_hashes = 0;
5954 /* Flush and reset the mta with the new values */
5955 igb_set_rx_mode(adapter->netdev);
5958 * Reset the VFs TDWBAL and TDWBAH registers which are not
5961 E1000_WRITE_REG(hw, E1000_TDWBAH(vf), 0);
5962 E1000_WRITE_REG(hw, E1000_TDWBAL(vf), 0);
5963 if (hw->mac.type == e1000_82576) {
5964 E1000_WRITE_REG(hw, E1000_TDWBAH(IGB_MAX_VF_FUNCTIONS + vf), 0);
5965 E1000_WRITE_REG(hw, E1000_TDWBAL(IGB_MAX_VF_FUNCTIONS + vf), 0);
5969 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5971 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5973 /* generate a new mac address as we were hotplug removed/added */
5974 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5975 random_ether_addr(vf_mac);
5977 /* process remaining reset events */
5978 igb_vf_reset(adapter, vf);
5981 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
5983 struct e1000_hw *hw = &adapter->hw;
5984 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5986 u8 *addr = (u8 *)(&msgbuf[1]);
5988 /* process all the same items cleared in a function level reset */
5989 igb_vf_reset(adapter, vf);
5991 /* set vf mac address */
5992 igb_del_mac_filter(adapter, vf_mac, vf);
5993 igb_add_mac_filter(adapter, vf_mac, vf);
5995 /* enable transmit and receive for vf */
5996 reg = E1000_READ_REG(hw, E1000_VFTE);
5997 E1000_WRITE_REG(hw, E1000_VFTE, reg | (1 << vf));
5998 reg = E1000_READ_REG(hw, E1000_VFRE);
5999 E1000_WRITE_REG(hw, E1000_VFRE, reg | (1 << vf));
6001 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
6003 /* reply to reset with ack and vf mac address */
6004 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
6005 memcpy(addr, vf_mac, 6);
6006 e1000_write_mbx(hw, msgbuf, 3, vf);
6009 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
6012 * The VF MAC Address is stored in a packed array of bytes
6013 * starting at the second 32 bit word of the msg array
6015 unsigned char *addr = (unsigned char *)&msg[1];
6018 if (is_valid_ether_addr(addr))
6019 err = igb_set_vf_mac(adapter, vf, addr);
6024 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
6026 struct e1000_hw *hw = &adapter->hw;
6027 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6028 u32 msg = E1000_VT_MSGTYPE_NACK;
6030 /* if device isn't clear to send it shouldn't be reading either */
6031 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
6032 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
6033 e1000_write_mbx(hw, &msg, 1, vf);
6034 vf_data->last_nack = jiffies;
6038 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
6040 struct pci_dev *pdev = adapter->pdev;
6041 u32 msgbuf[E1000_VFMAILBOX_SIZE];
6042 struct e1000_hw *hw = &adapter->hw;
6043 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6046 retval = e1000_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
6049 dev_err(pci_dev_to_dev(pdev), "Error receiving message from VF\n");
6053 /* this is a message we already processed, do nothing */
6054 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
6058 * until the vf completes a reset it should not be
6059 * allowed to start any configuration.
6062 if (msgbuf[0] == E1000_VF_RESET) {
6063 igb_vf_reset_msg(adapter, vf);
6067 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
6068 msgbuf[0] = E1000_VT_MSGTYPE_NACK;
6069 if (time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
6070 e1000_write_mbx(hw, msgbuf, 1, vf);
6071 vf_data->last_nack = jiffies;
6076 switch ((msgbuf[0] & 0xFFFF)) {
6077 case E1000_VF_SET_MAC_ADDR:
6079 #ifndef IGB_DISABLE_VF_MAC_SET
6080 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
6081 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
6084 "VF %d attempted to override administratively "
6085 "set MAC address\nReload the VF driver to "
6086 "resume operations\n", vf);
6089 case E1000_VF_SET_PROMISC:
6090 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
6092 case E1000_VF_SET_MULTICAST:
6093 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
6095 case E1000_VF_SET_LPE:
6096 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
6098 case E1000_VF_SET_VLAN:
6101 if (vf_data->pf_vlan)
6103 "VF %d attempted to override administratively "
6104 "set VLAN tag\nReload the VF driver to "
6105 "resume operations\n", vf);
6108 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
6111 dev_err(pci_dev_to_dev(pdev), "Unhandled Msg %08x\n", msgbuf[0]);
6112 retval = -E1000_ERR_MBX;
6116 /* notify the VF of the results of what it sent us */
6118 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
6120 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
6122 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
6124 e1000_write_mbx(hw, msgbuf, 1, vf);
6127 static void igb_msg_task(struct igb_adapter *adapter)
6129 struct e1000_hw *hw = &adapter->hw;
6132 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
6133 /* process any reset requests */
6134 if (!e1000_check_for_rst(hw, vf))
6135 igb_vf_reset_event(adapter, vf);
6137 /* process any messages pending */
6138 if (!e1000_check_for_msg(hw, vf))
6139 igb_rcv_msg_from_vf(adapter, vf);
6141 /* process any acks */
6142 if (!e1000_check_for_ack(hw, vf))
6143 igb_rcv_ack_from_vf(adapter, vf);
6148 * igb_set_uta - Set unicast filter table address
6149 * @adapter: board private structure
6151 * The unicast table address is a register array of 32-bit registers.
6152 * The table is meant to be used in a way similar to how the MTA is used
6153 * however due to certain limitations in the hardware it is necessary to
6154 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
6155 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
6157 static void igb_set_uta(struct igb_adapter *adapter)
6159 struct e1000_hw *hw = &adapter->hw;
6162 /* The UTA table only exists on 82576 hardware and newer */
6163 if (hw->mac.type < e1000_82576)
6166 /* we only need to do this if VMDq is enabled */
6167 if (!adapter->vmdq_pools)
6170 for (i = 0; i < hw->mac.uta_reg_count; i++)
6171 E1000_WRITE_REG_ARRAY(hw, E1000_UTA, i, ~0);
6175 * igb_intr_msi - Interrupt Handler
6176 * @irq: interrupt number
6177 * @data: pointer to a network interface device structure
6179 static irqreturn_t igb_intr_msi(int irq, void *data)
6181 struct igb_adapter *adapter = data;
6182 struct igb_q_vector *q_vector = adapter->q_vector[0];
6183 struct e1000_hw *hw = &adapter->hw;
6184 /* read ICR disables interrupts using IAM */
6185 u32 icr = E1000_READ_REG(hw, E1000_ICR);
6187 igb_write_itr(q_vector);
6189 if (icr & E1000_ICR_DRSTA)
6190 schedule_work(&adapter->reset_task);
6192 if (icr & E1000_ICR_DOUTSYNC) {
6193 /* HW is reporting DMA is out of sync */
6194 adapter->stats.doosync++;
6197 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
6198 hw->mac.get_link_status = 1;
6199 if (!test_bit(__IGB_DOWN, &adapter->state))
6200 mod_timer(&adapter->watchdog_timer, jiffies + 1);
6203 napi_schedule(&q_vector->napi);
6209 * igb_intr - Legacy Interrupt Handler
6210 * @irq: interrupt number
6211 * @data: pointer to a network interface device structure
6213 static irqreturn_t igb_intr(int irq, void *data)
6215 struct igb_adapter *adapter = data;
6216 struct igb_q_vector *q_vector = adapter->q_vector[0];
6217 struct e1000_hw *hw = &adapter->hw;
6218 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
6219 * need for the IMC write */
6220 u32 icr = E1000_READ_REG(hw, E1000_ICR);
6222 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
6223 * not set, then the adapter didn't send an interrupt */
6224 if (!(icr & E1000_ICR_INT_ASSERTED))
6227 igb_write_itr(q_vector);
6229 if (icr & E1000_ICR_DRSTA)
6230 schedule_work(&adapter->reset_task);
6232 if (icr & E1000_ICR_DOUTSYNC) {
6233 /* HW is reporting DMA is out of sync */
6234 adapter->stats.doosync++;
6237 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
6238 hw->mac.get_link_status = 1;
6239 /* guard against interrupt when we're going down */
6240 if (!test_bit(__IGB_DOWN, &adapter->state))
6241 mod_timer(&adapter->watchdog_timer, jiffies + 1);
6244 napi_schedule(&q_vector->napi);
6249 void igb_ring_irq_enable(struct igb_q_vector *q_vector)
6251 struct igb_adapter *adapter = q_vector->adapter;
6252 struct e1000_hw *hw = &adapter->hw;
6254 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
6255 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
6256 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
6257 igb_set_itr(q_vector);
6259 igb_update_ring_itr(q_vector);
6262 if (!test_bit(__IGB_DOWN, &adapter->state)) {
6263 if (adapter->msix_entries)
6264 E1000_WRITE_REG(hw, E1000_EIMS, q_vector->eims_value);
6266 igb_irq_enable(adapter);
6271 * igb_poll - NAPI Rx polling callback
6272 * @napi: napi polling structure
6273 * @budget: count of how many packets we should handle
6275 static int igb_poll(struct napi_struct *napi, int budget)
6277 struct igb_q_vector *q_vector = container_of(napi, struct igb_q_vector, napi);
6278 bool clean_complete = true;
6281 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
6282 igb_update_dca(q_vector);
6284 if (q_vector->tx.ring)
6285 clean_complete = igb_clean_tx_irq(q_vector);
6287 if (q_vector->rx.ring)
6288 clean_complete &= igb_clean_rx_irq(q_vector, budget);
6290 #ifndef HAVE_NETDEV_NAPI_LIST
6291 /* if netdev is disabled we need to stop polling */
6292 if (!netif_running(q_vector->adapter->netdev))
6293 clean_complete = true;
6296 /* If all work not completed, return budget and keep polling */
6297 if (!clean_complete)
6300 /* If not enough Rx work done, exit the polling mode */
6301 napi_complete(napi);
6302 igb_ring_irq_enable(q_vector);
6307 #ifdef HAVE_HW_TIME_STAMP
6309 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
6310 * @adapter: board private structure
6311 * @shhwtstamps: timestamp structure to update
6312 * @regval: unsigned 64bit system time value.
6314 * We need to convert the system time value stored in the RX/TXSTMP registers
6315 * into a hwtstamp which can be used by the upper level timestamping functions
6317 static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
6318 struct skb_shared_hwtstamps *shhwtstamps,
6324 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
6325 * 24 to match clock shift we setup earlier.
6327 if (adapter->hw.mac.type >= e1000_82580)
6328 regval <<= IGB_82580_TSYNC_SHIFT;
6330 ns = timecounter_cyc2time(&adapter->clock, regval);
6333 * force a timecompare_update here (even if less than a second
6334 * has passed) in order to prevent the case when ptpd or other
6335 * software jumps the clock offset. othwerise there is a small
6336 * window when the timestamp would be based on previous skew
6337 * and invalid results would be pushed to the network stack.
6339 timecompare_update(&adapter->compare, 0);
6340 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
6341 shhwtstamps->hwtstamp = ns_to_ktime(ns);
6342 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
6346 * igb_tx_hwtstamp - utility function which checks for TX time stamp
6347 * @q_vector: pointer to q_vector containing needed info
6348 * @buffer: pointer to igb_tx_buffer structure
6350 * If we were asked to do hardware stamping and such a time stamp is
6351 * available, then it must have been for this skb here because we only
6352 * allow only one such packet into the queue.
6354 static void igb_tx_hwtstamp(struct igb_q_vector *q_vector,
6355 struct igb_tx_buffer *buffer_info)
6357 struct igb_adapter *adapter = q_vector->adapter;
6358 struct e1000_hw *hw = &adapter->hw;
6359 struct skb_shared_hwtstamps shhwtstamps;
6362 /* if skb does not support hw timestamp or TX stamp not valid exit */
6363 if (likely(!(buffer_info->tx_flags & IGB_TX_FLAGS_TSTAMP)) ||
6364 !(E1000_READ_REG(hw, E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
6367 regval = E1000_READ_REG(hw, E1000_TXSTMPL);
6368 regval |= (u64)E1000_READ_REG(hw, E1000_TXSTMPH) << 32;
6370 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
6371 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
6376 * igb_clean_tx_irq - Reclaim resources after transmit completes
6377 * @q_vector: pointer to q_vector containing needed info
6378 * returns TRUE if ring is completely cleaned
6380 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
6382 struct igb_adapter *adapter = q_vector->adapter;
6383 struct igb_ring *tx_ring = q_vector->tx.ring;
6384 struct igb_tx_buffer *tx_buffer;
6385 union e1000_adv_tx_desc *tx_desc, *eop_desc;
6386 unsigned int total_bytes = 0, total_packets = 0;
6387 unsigned int budget = q_vector->tx.work_limit;
6388 unsigned int i = tx_ring->next_to_clean;
6390 if (test_bit(__IGB_DOWN, &adapter->state))
6393 tx_buffer = &tx_ring->tx_buffer_info[i];
6394 tx_desc = IGB_TX_DESC(tx_ring, i);
6395 i -= tx_ring->count;
6397 for (; budget; budget--) {
6398 eop_desc = tx_buffer->next_to_watch;
6400 /* prevent any other reads prior to eop_desc */
6403 /* if next_to_watch is not set then there is no work pending */
6407 /* if DD is not set pending work has not been completed */
6408 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
6411 /* clear next_to_watch to prevent false hangs */
6412 tx_buffer->next_to_watch = NULL;
6414 /* update the statistics for this packet */
6415 total_bytes += tx_buffer->bytecount;
6416 total_packets += tx_buffer->gso_segs;
6418 #ifdef HAVE_HW_TIME_STAMP
6419 /* retrieve hardware timestamp */
6420 igb_tx_hwtstamp(q_vector, tx_buffer);
6424 dev_kfree_skb_any(tx_buffer->skb);
6426 /* unmap skb header data */
6427 dma_unmap_single(tx_ring->dev,
6428 dma_unmap_addr(tx_buffer, dma),
6429 dma_unmap_len(tx_buffer, len),
6432 /* clear tx_buffer data */
6433 tx_buffer->skb = NULL;
6434 dma_unmap_len_set(tx_buffer, len, 0);
6436 /* clear last DMA location and unmap remaining buffers */
6437 while (tx_desc != eop_desc) {
6442 i -= tx_ring->count;
6443 tx_buffer = tx_ring->tx_buffer_info;
6444 tx_desc = IGB_TX_DESC(tx_ring, 0);
6447 /* unmap any remaining paged data */
6448 if (dma_unmap_len(tx_buffer, len)) {
6449 dma_unmap_page(tx_ring->dev,
6450 dma_unmap_addr(tx_buffer, dma),
6451 dma_unmap_len(tx_buffer, len),
6453 dma_unmap_len_set(tx_buffer, len, 0);
6457 /* move us one more past the eop_desc for start of next pkt */
6462 i -= tx_ring->count;
6463 tx_buffer = tx_ring->tx_buffer_info;
6464 tx_desc = IGB_TX_DESC(tx_ring, 0);
6469 netdev_tx_completed_queue(txring_txq(tx_ring),
6470 total_packets, total_bytes);
6471 #endif /* CONFIG_BQL */
6473 i += tx_ring->count;
6474 tx_ring->next_to_clean = i;
6475 tx_ring->tx_stats.bytes += total_bytes;
6476 tx_ring->tx_stats.packets += total_packets;
6477 q_vector->tx.total_bytes += total_bytes;
6478 q_vector->tx.total_packets += total_packets;
6480 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
6481 struct e1000_hw *hw = &adapter->hw;
6483 eop_desc = tx_buffer->next_to_watch;
6485 /* Detect a transmit hang in hardware, this serializes the
6486 * check with the clearing of time_stamp and movement of i */
6487 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
6489 time_after(jiffies, tx_buffer->time_stamp +
6490 (adapter->tx_timeout_factor * HZ))
6491 && !(E1000_READ_REG(hw, E1000_STATUS) &
6492 E1000_STATUS_TXOFF)) {
6494 /* detected Tx unit hang */
6495 dev_err(tx_ring->dev,
6496 "Detected Tx Unit Hang\n"
6500 " next_to_use <%x>\n"
6501 " next_to_clean <%x>\n"
6502 "buffer_info[next_to_clean]\n"
6503 " time_stamp <%lx>\n"
6504 " next_to_watch <%p>\n"
6506 " desc.status <%x>\n",
6507 tx_ring->queue_index,
6508 E1000_READ_REG(hw, E1000_TDH(tx_ring->reg_idx)),
6509 readl(tx_ring->tail),
6510 tx_ring->next_to_use,
6511 tx_ring->next_to_clean,
6512 tx_buffer->time_stamp,
6515 eop_desc->wb.status);
6516 if (netif_is_multiqueue(netdev_ring(tx_ring)))
6517 netif_stop_subqueue(netdev_ring(tx_ring),
6518 ring_queue_index(tx_ring));
6520 netif_stop_queue(netdev_ring(tx_ring));
6522 /* we are about to reset, no point in enabling stuff */
6527 if (unlikely(total_packets &&
6528 netif_carrier_ok(netdev_ring(tx_ring)) &&
6529 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
6530 /* Make sure that anybody stopping the queue after this
6531 * sees the new next_to_clean.
6534 if (netif_is_multiqueue(netdev_ring(tx_ring))) {
6535 if (__netif_subqueue_stopped(netdev_ring(tx_ring),
6536 ring_queue_index(tx_ring)) &&
6537 !(test_bit(__IGB_DOWN, &adapter->state))) {
6538 netif_wake_subqueue(netdev_ring(tx_ring),
6539 ring_queue_index(tx_ring));
6540 tx_ring->tx_stats.restart_queue++;
6543 if (netif_queue_stopped(netdev_ring(tx_ring)) &&
6544 !(test_bit(__IGB_DOWN, &adapter->state))) {
6545 netif_wake_queue(netdev_ring(tx_ring));
6546 tx_ring->tx_stats.restart_queue++;
6554 #ifdef HAVE_VLAN_RX_REGISTER
6556 * igb_receive_skb - helper function to handle rx indications
6557 * @q_vector: structure containing interrupt and ring information
6558 * @skb: packet to send up
6560 static void igb_receive_skb(struct igb_q_vector *q_vector,
6561 struct sk_buff *skb)
6563 struct vlan_group **vlgrp = netdev_priv(skb->dev);
6565 if (IGB_CB(skb)->vid) {
6567 vlan_gro_receive(&q_vector->napi, *vlgrp,
6568 IGB_CB(skb)->vid, skb);
6570 dev_kfree_skb_any(skb);
6573 napi_gro_receive(&q_vector->napi, skb);
6577 #endif /* HAVE_VLAN_RX_REGISTER */
6578 static inline void igb_rx_checksum(struct igb_ring *ring,
6579 union e1000_adv_rx_desc *rx_desc,
6580 struct sk_buff *skb)
6582 skb_checksum_none_assert(skb);
6584 /* Ignore Checksum bit is set */
6585 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
6588 /* Rx checksum disabled via ethtool */
6589 #ifdef HAVE_NDO_SET_FEATURES
6590 if (!(netdev_ring(ring)->features & NETIF_F_RXCSUM))
6592 if (!test_bit(IGB_RING_FLAG_RX_CSUM, &ring->flags))
6596 /* TCP/UDP checksum error bit is set */
6597 if (igb_test_staterr(rx_desc,
6598 E1000_RXDEXT_STATERR_TCPE |
6599 E1000_RXDEXT_STATERR_IPE)) {
6601 * work around errata with sctp packets where the TCPE aka
6602 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
6603 * packets, (aka let the stack check the crc32c)
6605 if (!((skb->len == 60) &&
6606 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags)))
6607 ring->rx_stats.csum_err++;
6609 /* let the stack verify checksum errors */
6612 /* It must be a TCP or UDP packet with a valid checksum */
6613 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
6614 E1000_RXD_STAT_UDPCS))
6615 skb->ip_summed = CHECKSUM_UNNECESSARY;
6618 #ifdef NETIF_F_RXHASH
6619 static inline void igb_rx_hash(struct igb_ring *ring,
6620 union e1000_adv_rx_desc *rx_desc,
6621 struct sk_buff *skb)
6623 if (netdev_ring(ring)->features & NETIF_F_RXHASH)
6624 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
6628 #ifdef HAVE_HW_TIME_STAMP
6629 static void igb_rx_hwtstamp(struct igb_q_vector *q_vector,
6630 union e1000_adv_rx_desc *rx_desc,
6631 struct sk_buff *skb)
6633 struct igb_adapter *adapter = q_vector->adapter;
6634 struct e1000_hw *hw = &adapter->hw;
6637 if (!igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP |
6638 E1000_RXDADV_STAT_TS))
6642 * If this bit is set, then the RX registers contain the time stamp. No
6643 * other packet will be time stamped until we read these registers, so
6644 * read the registers to make them available again. Because only one
6645 * packet can be time stamped at a time, we know that the register
6646 * values must belong to this one here and therefore we don't need to
6647 * compare any of the additional attributes stored for it.
6649 * If nothing went wrong, then it should have a skb_shared_tx that we
6650 * can turn into a skb_shared_hwtstamps.
6652 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
6653 u32 *stamp = (u32 *)skb->data;
6654 regval = le32_to_cpu(*(stamp + 2));
6655 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
6656 skb_pull(skb, IGB_TS_HDR_LEN);
6658 if(!(E1000_READ_REG(hw, E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
6661 regval = E1000_READ_REG(hw, E1000_RXSTMPL);
6662 regval |= (u64)E1000_READ_REG(hw, E1000_RXSTMPH) << 32;
6665 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
6668 static void igb_rx_vlan(struct igb_ring *ring,
6669 union e1000_adv_rx_desc *rx_desc,
6670 struct sk_buff *skb)
6672 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
6674 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
6675 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags))
6676 vid = be16_to_cpu(rx_desc->wb.upper.vlan);
6678 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
6679 #ifdef HAVE_VLAN_RX_REGISTER
6680 IGB_CB(skb)->vid = vid;
6682 IGB_CB(skb)->vid = 0;
6685 #ifdef HAVE_VLAN_PROTOCOL
6686 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
6688 __vlan_hwaccel_put_tag(skb, vid);
6696 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
6697 static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
6699 /* HW will not DMA in data larger than the given buffer, even if it
6700 * parses the (NFS, of course) header to be larger. In that case, it
6701 * fills the header buffer and spills the rest into the page.
6703 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info) &
6704 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
6705 if (hlen > IGB_RX_HDR_LEN)
6706 hlen = IGB_RX_HDR_LEN;
6713 * igb_merge_active_tail - merge active tail into lro skb
6714 * @tail: pointer to active tail in frag_list
6716 * This function merges the length and data of an active tail into the
6717 * skb containing the frag_list. It resets the tail's pointer to the head,
6718 * but it leaves the heads pointer to tail intact.
6720 static inline struct sk_buff *igb_merge_active_tail(struct sk_buff *tail)
6722 struct sk_buff *head = IGB_CB(tail)->head;
6727 head->len += tail->len;
6728 head->data_len += tail->len;
6729 head->truesize += tail->len;
6731 IGB_CB(tail)->head = NULL;
6737 * igb_add_active_tail - adds an active tail into the skb frag_list
6738 * @head: pointer to the start of the skb
6739 * @tail: pointer to active tail to add to frag_list
6741 * This function adds an active tail to the end of the frag list. This tail
6742 * will still be receiving data so we cannot yet ad it's stats to the main
6743 * skb. That is done via igb_merge_active_tail.
6745 static inline void igb_add_active_tail(struct sk_buff *head, struct sk_buff *tail)
6747 struct sk_buff *old_tail = IGB_CB(head)->tail;
6750 igb_merge_active_tail(old_tail);
6751 old_tail->next = tail;
6753 skb_shinfo(head)->frag_list = tail;
6756 IGB_CB(tail)->head = head;
6757 IGB_CB(head)->tail = tail;
6759 IGB_CB(head)->append_cnt++;
6763 * igb_close_active_frag_list - cleanup pointers on a frag_list skb
6764 * @head: pointer to head of an active frag list
6766 * This function will clear the frag_tail_tracker pointer on an active
6767 * frag_list and returns true if the pointer was actually set
6769 static inline bool igb_close_active_frag_list(struct sk_buff *head)
6771 struct sk_buff *tail = IGB_CB(head)->tail;
6776 igb_merge_active_tail(tail);
6778 IGB_CB(head)->tail = NULL;
6784 * igb_can_lro - returns true if packet is TCP/IPV4 and LRO is enabled
6785 * @adapter: board private structure
6786 * @rx_desc: pointer to the rx descriptor
6787 * @skb: pointer to the skb to be merged
6790 static inline bool igb_can_lro(struct igb_ring *rx_ring,
6791 union e1000_adv_rx_desc *rx_desc,
6792 struct sk_buff *skb)
6794 struct iphdr *iph = (struct iphdr *)skb->data;
6795 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
6797 /* verify LRO is enabled */
6798 if (!(netdev_ring(rx_ring)->features & NETIF_F_LRO))
6801 /* verify hardware indicates this is IPv4/TCP */
6802 if((!(pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_TCP)) ||
6803 !(pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_IPV4))))
6806 /* verify the header is large enough for us to read IP/TCP fields */
6807 if (!pskb_may_pull(skb, sizeof(struct igb_lrohdr)))
6810 /* verify there are no VLANs on packet */
6811 if (skb->protocol != __constant_htons(ETH_P_IP))
6814 /* ensure we are version 4 with no options */
6815 if (*(u8 *)iph != 0x45)
6818 /* .. and the packet is not fragmented */
6819 if (iph->frag_off & htons(IP_MF | IP_OFFSET))
6822 /* .. and that next header is TCP */
6823 if (iph->protocol != IPPROTO_TCP)
6829 static inline struct igb_lrohdr *igb_lro_hdr(struct sk_buff *skb)
6831 return (struct igb_lrohdr *)skb->data;
6835 * igb_lro_flush - Indicate packets to upper layer.
6837 * Update IP and TCP header part of head skb if more than one
6838 * skb's chained and indicate packets to upper layer.
6840 static void igb_lro_flush(struct igb_q_vector *q_vector,
6841 struct sk_buff *skb)
6843 struct igb_lro_list *lrolist = q_vector->lrolist;
6845 __skb_unlink(skb, &lrolist->active);
6847 if (IGB_CB(skb)->append_cnt) {
6848 struct igb_lrohdr *lroh = igb_lro_hdr(skb);
6850 /* close any active lro contexts */
6851 igb_close_active_frag_list(skb);
6853 /* incorporate ip header and re-calculate checksum */
6854 lroh->iph.tot_len = ntohs(skb->len);
6855 lroh->iph.check = 0;
6857 /* header length is 5 since we know no options exist */
6858 lroh->iph.check = ip_fast_csum((u8 *)lroh, 5);
6860 /* clear TCP checksum to indicate we are an LRO frame */
6863 /* incorporate latest timestamp into the tcp header */
6864 if (IGB_CB(skb)->tsecr) {
6865 lroh->ts[2] = IGB_CB(skb)->tsecr;
6866 lroh->ts[1] = htonl(IGB_CB(skb)->tsval);
6870 skb_shinfo(skb)->gso_size = IGB_CB(skb)->mss;
6874 #ifdef HAVE_VLAN_RX_REGISTER
6875 igb_receive_skb(q_vector, skb);
6877 napi_gro_receive(&q_vector->napi, skb);
6879 lrolist->stats.flushed++;
6882 static void igb_lro_flush_all(struct igb_q_vector *q_vector)
6884 struct igb_lro_list *lrolist = q_vector->lrolist;
6885 struct sk_buff *skb, *tmp;
6887 skb_queue_reverse_walk_safe(&lrolist->active, skb, tmp)
6888 igb_lro_flush(q_vector, skb);
6892 * igb_lro_header_ok - Main LRO function.
6894 static void igb_lro_header_ok(struct sk_buff *skb)
6896 struct igb_lrohdr *lroh = igb_lro_hdr(skb);
6897 u16 opt_bytes, data_len;
6899 IGB_CB(skb)->tail = NULL;
6900 IGB_CB(skb)->tsecr = 0;
6901 IGB_CB(skb)->append_cnt = 0;
6902 IGB_CB(skb)->mss = 0;
6904 /* ensure that the checksum is valid */
6905 if (skb->ip_summed != CHECKSUM_UNNECESSARY)
6908 /* If we see CE codepoint in IP header, packet is not mergeable */
6909 if (INET_ECN_is_ce(ipv4_get_dsfield(&lroh->iph)))
6912 /* ensure no bits set besides ack or psh */
6913 if (lroh->th.fin || lroh->th.syn || lroh->th.rst ||
6914 lroh->th.urg || lroh->th.ece || lroh->th.cwr ||
6918 /* store the total packet length */
6919 data_len = ntohs(lroh->iph.tot_len);
6921 /* remove any padding from the end of the skb */
6922 __pskb_trim(skb, data_len);
6924 /* remove header length from data length */
6925 data_len -= sizeof(struct igb_lrohdr);
6928 * check for timestamps. Since the only option we handle are timestamps,
6929 * we only have to handle the simple case of aligned timestamps
6931 opt_bytes = (lroh->th.doff << 2) - sizeof(struct tcphdr);
6932 if (opt_bytes != 0) {
6933 if ((opt_bytes != TCPOLEN_TSTAMP_ALIGNED) ||
6934 !pskb_may_pull(skb, sizeof(struct igb_lrohdr) +
6935 TCPOLEN_TSTAMP_ALIGNED) ||
6936 (lroh->ts[0] != htonl((TCPOPT_NOP << 24) |
6937 (TCPOPT_NOP << 16) |
6938 (TCPOPT_TIMESTAMP << 8) |
6939 TCPOLEN_TIMESTAMP)) ||
6940 (lroh->ts[2] == 0)) {
6944 IGB_CB(skb)->tsval = ntohl(lroh->ts[1]);
6945 IGB_CB(skb)->tsecr = lroh->ts[2];
6947 data_len -= TCPOLEN_TSTAMP_ALIGNED;
6950 /* record data_len as mss for the packet */
6951 IGB_CB(skb)->mss = data_len;
6952 IGB_CB(skb)->next_seq = ntohl(lroh->th.seq);
6955 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
6956 static bool igb_merge_frags(struct sk_buff *lro_skb, struct sk_buff *new_skb)
6958 struct sk_buff *tail;
6959 struct skb_shared_info *tail_info;
6960 struct skb_shared_info *new_skb_info;
6963 /* header must be empty to pull frags into current skb */
6964 if (skb_headlen(new_skb))
6967 if (IGB_CB(lro_skb)->tail)
6968 tail = IGB_CB(lro_skb)->tail;
6972 tail_info = skb_shinfo(tail);
6973 new_skb_info = skb_shinfo(new_skb);
6975 /* make sure we have room in frags list */
6976 if (new_skb_info->nr_frags >= (MAX_SKB_FRAGS - tail_info->nr_frags))
6979 /* bump append count */
6980 IGB_CB(lro_skb)->append_cnt++;
6982 /* copy frags into the last skb */
6983 memcpy(tail_info->frags + tail_info->nr_frags,
6984 new_skb_info->frags,
6985 new_skb_info->nr_frags * sizeof(skb_frag_t));
6987 /* copy size data over */
6988 tail_info->nr_frags += new_skb_info->nr_frags;
6989 data_len = IGB_CB(new_skb)->mss;
6990 tail->len += data_len;
6991 tail->data_len += data_len;
6992 tail->truesize += data_len;
6994 /* wipe record of data from new_skb */
6995 new_skb_info->nr_frags = 0;
6996 new_skb->len = new_skb->data_len = 0;
6997 new_skb->truesize -= data_len;
6998 new_skb->data = new_skb->head + NET_SKB_PAD + NET_IP_ALIGN;
6999 skb_reset_tail_pointer(new_skb);
7000 new_skb->protocol = 0;
7001 new_skb->ip_summed = CHECKSUM_NONE;
7002 #ifdef HAVE_VLAN_RX_REGISTER
7003 IGB_CB(new_skb)->vid = 0;
7005 new_skb->vlan_tci = 0;
7011 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7013 * igb_lro_queue - if able, queue skb into lro chain
7014 * @q_vector: structure containing interrupt and ring information
7015 * @new_skb: pointer to current skb being checked
7017 * Checks whether the skb given is eligible for LRO and if that's
7018 * fine chains it to the existing lro_skb based on flowid. If an LRO for
7019 * the flow doesn't exist create one.
7021 static struct sk_buff *igb_lro_queue(struct igb_q_vector *q_vector,
7022 struct sk_buff *new_skb)
7024 struct sk_buff *lro_skb;
7025 struct igb_lro_list *lrolist = q_vector->lrolist;
7026 struct igb_lrohdr *lroh = igb_lro_hdr(new_skb);
7027 __be32 saddr = lroh->iph.saddr;
7028 __be32 daddr = lroh->iph.daddr;
7029 __be32 tcp_ports = *(__be32 *)&lroh->th;
7031 #ifdef HAVE_VLAN_RX_REGISTER
7032 u16 vid = IGB_CB(new_skb)->vid;
7034 u16 vid = new_skb->vlan_tci;
7037 igb_lro_header_ok(new_skb);
7040 * we have a packet that might be eligible for LRO,
7041 * so see if it matches anything we might expect
7043 skb_queue_walk(&lrolist->active, lro_skb) {
7044 if (*(__be32 *)&igb_lro_hdr(lro_skb)->th != tcp_ports ||
7045 igb_lro_hdr(lro_skb)->iph.saddr != saddr ||
7046 igb_lro_hdr(lro_skb)->iph.daddr != daddr)
7049 #ifdef HAVE_VLAN_RX_REGISTER
7050 if (IGB_CB(lro_skb)->vid != vid)
7052 if (lro_skb->vlan_tci != vid)
7056 /* out of order packet */
7057 if (IGB_CB(lro_skb)->next_seq != IGB_CB(new_skb)->next_seq) {
7058 igb_lro_flush(q_vector, lro_skb);
7059 IGB_CB(new_skb)->mss = 0;
7063 /* TCP timestamp options have changed */
7064 if (!IGB_CB(lro_skb)->tsecr != !IGB_CB(new_skb)->tsecr) {
7065 igb_lro_flush(q_vector, lro_skb);
7069 /* make sure timestamp values are increasing */
7070 if (IGB_CB(lro_skb)->tsecr &&
7071 IGB_CB(lro_skb)->tsval > IGB_CB(new_skb)->tsval) {
7072 igb_lro_flush(q_vector, lro_skb);
7073 IGB_CB(new_skb)->mss = 0;
7077 data_len = IGB_CB(new_skb)->mss;
7080 * malformed header, no tcp data, resultant packet would
7081 * be too large, or new skb is larger than our current mss.
7083 if (data_len == 0 ||
7084 data_len > IGB_CB(lro_skb)->mss ||
7085 data_len > IGB_CB(lro_skb)->free) {
7086 igb_lro_flush(q_vector, lro_skb);
7090 /* ack sequence numbers or window size has changed */
7091 if (igb_lro_hdr(lro_skb)->th.ack_seq != lroh->th.ack_seq ||
7092 igb_lro_hdr(lro_skb)->th.window != lroh->th.window) {
7093 igb_lro_flush(q_vector, lro_skb);
7097 /* Remove IP and TCP header*/
7098 skb_pull(new_skb, new_skb->len - data_len);
7100 /* update timestamp and timestamp echo response */
7101 IGB_CB(lro_skb)->tsval = IGB_CB(new_skb)->tsval;
7102 IGB_CB(lro_skb)->tsecr = IGB_CB(new_skb)->tsecr;
7104 /* update sequence and free space */
7105 IGB_CB(lro_skb)->next_seq += data_len;
7106 IGB_CB(lro_skb)->free -= data_len;
7108 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7109 /* if header is empty pull pages into current skb */
7110 if (igb_merge_frags(lro_skb, new_skb)) {
7111 lrolist->stats.recycled++;
7114 /* chain this new skb in frag_list */
7115 igb_add_active_tail(lro_skb, new_skb);
7117 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7121 if ((data_len < IGB_CB(lro_skb)->mss) || lroh->th.psh) {
7122 igb_lro_hdr(lro_skb)->th.psh |= lroh->th.psh;
7123 igb_lro_flush(q_vector, lro_skb);
7126 lrolist->stats.coal++;
7130 if (IGB_CB(new_skb)->mss && !lroh->th.psh) {
7131 /* if we are at capacity flush the tail */
7132 if (skb_queue_len(&lrolist->active) >= IGB_LRO_MAX) {
7133 lro_skb = skb_peek_tail(&lrolist->active);
7135 igb_lro_flush(q_vector, lro_skb);
7138 /* update sequence and free space */
7139 IGB_CB(new_skb)->next_seq += IGB_CB(new_skb)->mss;
7140 IGB_CB(new_skb)->free = 65521 - new_skb->len;
7142 /* .. and insert at the front of the active list */
7143 __skb_queue_head(&lrolist->active, new_skb);
7145 lrolist->stats.coal++;
7149 /* packet not handled by any of the above, pass it to the stack */
7150 #ifdef HAVE_VLAN_RX_REGISTER
7151 igb_receive_skb(q_vector, new_skb);
7153 napi_gro_receive(&q_vector->napi, new_skb);
7158 #endif /* IGB_NO_LRO */
7159 static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
7161 struct igb_ring *rx_ring = q_vector->rx.ring;
7162 union e1000_adv_rx_desc *rx_desc;
7163 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7164 const int current_node = numa_node_id();
7166 unsigned int total_bytes = 0, total_packets = 0;
7167 u16 cleaned_count = igb_desc_unused(rx_ring);
7168 u16 i = rx_ring->next_to_clean;
7170 rx_desc = IGB_RX_DESC(rx_ring, i);
7172 while (igb_test_staterr(rx_desc, E1000_RXD_STAT_DD)) {
7173 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
7174 struct sk_buff *skb = buffer_info->skb;
7175 union e1000_adv_rx_desc *next_rxd;
7177 buffer_info->skb = NULL;
7178 prefetch(skb->data);
7181 if (i == rx_ring->count)
7184 next_rxd = IGB_RX_DESC(rx_ring, i);
7188 * This memory barrier is needed to keep us from reading
7189 * any other fields out of the rx_desc until we know the
7190 * RXD_STAT_DD bit is set
7194 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7195 __skb_put(skb, le16_to_cpu(rx_desc->wb.upper.length));
7196 dma_unmap_single(rx_ring->dev, buffer_info->dma,
7197 rx_ring->rx_buffer_len,
7199 buffer_info->dma = 0;
7202 if (!skb_is_nonlinear(skb)) {
7203 __skb_put(skb, igb_get_hlen(rx_desc));
7204 dma_unmap_single(rx_ring->dev, buffer_info->dma,
7207 buffer_info->dma = 0;
7210 if (rx_desc->wb.upper.length) {
7211 u16 length = le16_to_cpu(rx_desc->wb.upper.length);
7213 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
7215 buffer_info->page_offset,
7219 skb->data_len += length;
7220 skb->truesize += length;
7222 if ((page_count(buffer_info->page) != 1) ||
7223 (page_to_nid(buffer_info->page) != current_node))
7224 buffer_info->page = NULL;
7226 get_page(buffer_info->page);
7228 dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
7229 PAGE_SIZE / 2, DMA_FROM_DEVICE);
7230 buffer_info->page_dma = 0;
7233 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)) {
7234 struct igb_rx_buffer *next_buffer;
7235 next_buffer = &rx_ring->rx_buffer_info[i];
7236 buffer_info->skb = next_buffer->skb;
7237 buffer_info->dma = next_buffer->dma;
7238 next_buffer->skb = skb;
7239 next_buffer->dma = 0;
7243 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7244 if (igb_test_staterr(rx_desc,
7245 E1000_RXDEXT_ERR_FRAME_ERR_MASK)) {
7246 dev_kfree_skb_any(skb);
7250 #ifdef HAVE_HW_TIME_STAMP
7251 igb_rx_hwtstamp(q_vector, rx_desc, skb);
7253 #ifdef NETIF_F_RXHASH
7254 igb_rx_hash(rx_ring, rx_desc, skb);
7256 igb_rx_checksum(rx_ring, rx_desc, skb);
7257 igb_rx_vlan(rx_ring, rx_desc, skb);
7259 total_bytes += skb->len;
7262 skb->protocol = eth_type_trans(skb, netdev_ring(rx_ring));
7265 if (igb_can_lro(rx_ring, rx_desc, skb))
7266 buffer_info->skb = igb_lro_queue(q_vector, skb);
7269 #ifdef HAVE_VLAN_RX_REGISTER
7270 igb_receive_skb(q_vector, skb);
7272 napi_gro_receive(&q_vector->napi, skb);
7276 netdev_ring(rx_ring)->last_rx = jiffies;
7286 /* return some buffers to hardware, one at a time is too slow */
7287 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
7288 igb_alloc_rx_buffers(rx_ring, cleaned_count);
7292 /* use prefetched values */
7296 rx_ring->next_to_clean = i;
7297 rx_ring->rx_stats.packets += total_packets;
7298 rx_ring->rx_stats.bytes += total_bytes;
7299 q_vector->rx.total_packets += total_packets;
7300 q_vector->rx.total_bytes += total_bytes;
7303 igb_alloc_rx_buffers(rx_ring, cleaned_count);
7306 if (netdev_ring(rx_ring)->features & NETIF_F_LRO)
7307 igb_lro_flush_all(q_vector);
7309 #endif /* IGB_NO_LRO */
7313 static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
7314 struct igb_rx_buffer *bi)
7316 struct sk_buff *skb = bi->skb;
7317 dma_addr_t dma = bi->dma;
7323 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7324 skb = netdev_alloc_skb_ip_align(netdev_ring(rx_ring),
7325 rx_ring->rx_buffer_len);
7327 skb = netdev_alloc_skb_ip_align(netdev_ring(rx_ring),
7332 rx_ring->rx_stats.alloc_failed++;
7336 /* initialize skb for ring */
7337 skb_record_rx_queue(skb, ring_queue_index(rx_ring));
7340 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7341 dma = dma_map_single(rx_ring->dev, skb->data,
7342 rx_ring->rx_buffer_len, DMA_FROM_DEVICE);
7344 dma = dma_map_single(rx_ring->dev, skb->data,
7345 IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
7348 if (dma_mapping_error(rx_ring->dev, dma)) {
7349 rx_ring->rx_stats.alloc_failed++;
7357 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7358 static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
7359 struct igb_rx_buffer *bi)
7361 struct page *page = bi->page;
7362 dma_addr_t page_dma = bi->page_dma;
7363 unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
7369 page = alloc_page(GFP_ATOMIC | __GFP_COLD);
7371 if (unlikely(!page)) {
7372 rx_ring->rx_stats.alloc_failed++;
7377 page_dma = dma_map_page(rx_ring->dev, page,
7378 page_offset, PAGE_SIZE / 2,
7381 if (dma_mapping_error(rx_ring->dev, page_dma)) {
7382 rx_ring->rx_stats.alloc_failed++;
7386 bi->page_dma = page_dma;
7387 bi->page_offset = page_offset;
7391 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7393 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
7394 * @adapter: address of board private structure
7396 void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
7398 union e1000_adv_rx_desc *rx_desc;
7399 struct igb_rx_buffer *bi;
7400 u16 i = rx_ring->next_to_use;
7402 rx_desc = IGB_RX_DESC(rx_ring, i);
7403 bi = &rx_ring->rx_buffer_info[i];
7404 i -= rx_ring->count;
7406 while (cleaned_count--) {
7407 if (!igb_alloc_mapped_skb(rx_ring, bi))
7410 /* Refresh the desc even if buffer_addrs didn't change
7411 * because each write-back erases this info. */
7412 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7413 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
7415 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
7417 if (!igb_alloc_mapped_page(rx_ring, bi))
7420 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
7422 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7427 rx_desc = IGB_RX_DESC(rx_ring, 0);
7428 bi = rx_ring->rx_buffer_info;
7429 i -= rx_ring->count;
7432 /* clear the hdr_addr for the next_to_use descriptor */
7433 rx_desc->read.hdr_addr = 0;
7436 i += rx_ring->count;
7438 if (rx_ring->next_to_use != i) {
7439 rx_ring->next_to_use = i;
7441 /* Force memory writes to complete before letting h/w
7442 * know there are new descriptors to fetch. (Only
7443 * applicable for weak-ordered memory model archs,
7444 * such as IA-64). */
7446 writel(i, rx_ring->tail);
7457 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
7459 struct igb_adapter *adapter = netdev_priv(netdev);
7460 struct mii_ioctl_data *data = if_mii(ifr);
7462 if (adapter->hw.phy.media_type != e1000_media_type_copper)
7467 data->phy_id = adapter->hw.phy.addr;
7470 if (!capable(CAP_NET_ADMIN))
7472 if (e1000_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
7480 return E1000_SUCCESS;
7484 #ifdef HAVE_HW_TIME_STAMP
7486 * igb_hwtstamp_ioctl - control hardware time stamping
7491 * Outgoing time stamping can be enabled and disabled. Play nice and
7492 * disable it when requested, although it shouldn't case any overhead
7493 * when no packet needs it. At most one packet in the queue may be
7494 * marked for time stamping, otherwise it would be impossible to tell
7495 * for sure to which packet the hardware time stamp belongs.
7497 * Incoming time stamping has to be configured via the hardware
7498 * filters. Not all combinations are supported, in particular event
7499 * type has to be specified. Matching the kind of event packet is
7500 * not supported, with the exception of "all V2 events regardless of
7504 static int igb_hwtstamp_ioctl(struct net_device *netdev,
7505 struct ifreq *ifr, int cmd)
7507 struct igb_adapter *adapter = netdev_priv(netdev);
7508 struct e1000_hw *hw = &adapter->hw;
7509 struct hwtstamp_config config;
7510 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
7511 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
7512 u32 tsync_rx_cfg = 0;
7517 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
7520 /* reserved for future extensions */
7524 switch (config.tx_type) {
7525 case HWTSTAMP_TX_OFF:
7527 case HWTSTAMP_TX_ON:
7533 switch (config.rx_filter) {
7534 case HWTSTAMP_FILTER_NONE:
7537 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
7538 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
7539 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
7540 case HWTSTAMP_FILTER_ALL:
7542 * register TSYNCRXCFG must be set, therefore it is not
7543 * possible to time stamp both Sync and Delay_Req messages
7544 * => fall back to time stamping all packets
7546 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
7547 config.rx_filter = HWTSTAMP_FILTER_ALL;
7549 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
7550 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
7551 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
7554 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
7555 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
7556 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
7559 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
7560 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
7561 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
7562 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
7565 config.rx_filter = HWTSTAMP_FILTER_SOME;
7567 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
7568 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
7569 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
7570 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
7573 config.rx_filter = HWTSTAMP_FILTER_SOME;
7575 case HWTSTAMP_FILTER_PTP_V2_EVENT:
7576 case HWTSTAMP_FILTER_PTP_V2_SYNC:
7577 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
7578 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
7579 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
7587 if (hw->mac.type == e1000_82575) {
7588 if (tsync_rx_ctl | tsync_tx_ctl)
7593 #ifdef IGB_PER_PKT_TIMESTAMP
7595 * Per-packet timestamping only works if all packets are
7596 * timestamped, so enable timestamping in all packets as
7597 * long as one rx filter was configured.
7599 if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) {
7600 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
7601 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
7605 /* enable/disable TX */
7606 regval = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
7607 regval &= ~E1000_TSYNCTXCTL_ENABLED;
7608 regval |= tsync_tx_ctl;
7609 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, regval);
7611 /* enable/disable RX */
7612 regval = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
7613 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
7614 regval |= tsync_rx_ctl;
7615 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, regval);
7617 /* define which PTP packets are time stamped */
7618 E1000_WRITE_REG(hw, E1000_TSYNCRXCFG, tsync_rx_cfg);
7620 /* define ethertype filter for timestamped packets */
7622 E1000_WRITE_REG(hw, E1000_ETQF(3),
7623 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
7624 E1000_ETQF_1588 | /* enable timestamping */
7625 ETH_P_1588)); /* 1588 eth protocol type */
7627 E1000_WRITE_REG(hw, E1000_ETQF(3), 0);
7629 #define PTP_PORT 319
7630 /* L4 Queue Filter[3]: filter by destination port and protocol */
7632 u32 ftqf = (IPPROTO_UDP /* UDP */
7633 | E1000_FTQF_VF_BP /* VF not compared */
7634 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
7635 | E1000_FTQF_MASK); /* mask all inputs */
7636 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
7638 E1000_WRITE_REG(hw, E1000_IMIR(3), htons(PTP_PORT));
7639 E1000_WRITE_REG(hw, E1000_IMIREXT(3),
7640 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
7641 if (hw->mac.type == e1000_82576) {
7642 /* enable source port check */
7643 E1000_WRITE_REG(hw, E1000_SPQF(3), htons(PTP_PORT));
7644 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
7646 E1000_WRITE_REG(hw, E1000_FTQF(3), ftqf);
7648 E1000_WRITE_REG(hw, E1000_FTQF(3), E1000_FTQF_MASK);
7650 E1000_WRITE_FLUSH(hw);
7652 adapter->hwtstamp_config = config;
7654 /* clear TX/RX time stamp registers, just to be sure */
7655 regval = E1000_READ_REG(hw, E1000_TXSTMPH);
7656 regval = E1000_READ_REG(hw, E1000_RXSTMPH);
7658 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
7669 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
7676 return igb_mii_ioctl(netdev, ifr, cmd);
7678 #ifdef HAVE_HW_TIME_STAMP
7680 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
7682 #ifdef ETHTOOL_OPS_COMPAT
7684 return ethtool_ioctl(ifr);
7691 s32 e1000_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
7693 struct igb_adapter *adapter = hw->back;
7696 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
7698 return -E1000_ERR_CONFIG;
7700 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
7702 return E1000_SUCCESS;
7705 s32 e1000_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
7707 struct igb_adapter *adapter = hw->back;
7710 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
7712 return -E1000_ERR_CONFIG;
7714 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
7716 return E1000_SUCCESS;
7719 #ifdef HAVE_VLAN_RX_REGISTER
7720 static void igb_vlan_mode(struct net_device *netdev, struct vlan_group *vlgrp)
7722 void igb_vlan_mode(struct net_device *netdev, u32 features)
7725 struct igb_adapter *adapter = netdev_priv(netdev);
7726 struct e1000_hw *hw = &adapter->hw;
7729 #ifdef HAVE_VLAN_RX_REGISTER
7730 bool enable = !!vlgrp;
7732 igb_irq_disable(adapter);
7734 adapter->vlgrp = vlgrp;
7736 if (!test_bit(__IGB_DOWN, &adapter->state))
7737 igb_irq_enable(adapter);
7739 bool enable = !!(features & NETIF_F_HW_VLAN_RX);
7743 /* enable VLAN tag insert/strip */
7744 ctrl = E1000_READ_REG(hw, E1000_CTRL);
7745 ctrl |= E1000_CTRL_VME;
7746 E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
7748 /* Disable CFI check */
7749 rctl = E1000_READ_REG(hw, E1000_RCTL);
7750 rctl &= ~E1000_RCTL_CFIEN;
7751 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
7753 /* disable VLAN tag insert/strip */
7754 ctrl = E1000_READ_REG(hw, E1000_CTRL);
7755 ctrl &= ~E1000_CTRL_VME;
7756 E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
7759 #ifndef CONFIG_IGB_VMDQ_NETDEV
7760 for (i = 0; i < adapter->vmdq_pools; i++) {
7761 igb_set_vf_vlan_strip(adapter,
7762 adapter->vfs_allocated_count + i,
7767 igb_set_vf_vlan_strip(adapter,
7768 adapter->vfs_allocated_count,
7771 for (i = 1; i < adapter->vmdq_pools; i++) {
7772 #ifdef HAVE_VLAN_RX_REGISTER
7773 struct igb_vmdq_adapter *vadapter;
7774 vadapter = netdev_priv(adapter->vmdq_netdev[i-1]);
7775 enable = !!vadapter->vlgrp;
7777 struct net_device *vnetdev;
7778 vnetdev = adapter->vmdq_netdev[i-1];
7779 enable = !!(vnetdev->features & NETIF_F_HW_VLAN_RX);
7781 igb_set_vf_vlan_strip(adapter,
7782 adapter->vfs_allocated_count + i,
7787 igb_rlpml_set(adapter);
7790 #ifdef HAVE_VLAN_PROTOCOL
7791 static int igb_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
7792 #elif defined HAVE_INT_NDO_VLAN_RX_ADD_VID
7793 static int igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
7795 static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
7798 struct igb_adapter *adapter = netdev_priv(netdev);
7799 int pf_id = adapter->vfs_allocated_count;
7801 /* attempt to add filter to vlvf array */
7802 igb_vlvf_set(adapter, vid, TRUE, pf_id);
7804 /* add the filter since PF can receive vlans w/o entry in vlvf */
7805 igb_vfta_set(adapter, vid, TRUE);
7806 #ifndef HAVE_NETDEV_VLAN_FEATURES
7808 /* Copy feature flags from netdev to the vlan netdev for this vid.
7809 * This allows things like TSO to bubble down to our vlan device.
7810 * There is no need to update netdev for vlan 0 (DCB), since it
7811 * wouldn't has v_netdev.
7813 if (adapter->vlgrp) {
7814 struct vlan_group *vlgrp = adapter->vlgrp;
7815 struct net_device *v_netdev = vlan_group_get_device(vlgrp, vid);
7817 v_netdev->features |= netdev->features;
7818 vlan_group_set_device(vlgrp, vid, v_netdev);
7822 #ifndef HAVE_VLAN_RX_REGISTER
7824 set_bit(vid, adapter->active_vlans);
7826 #ifdef HAVE_INT_NDO_VLAN_RX_ADD_VID
7831 #ifdef HAVE_VLAN_PROTOCOL
7832 static int igb_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
7833 #elif defined HAVE_INT_NDO_VLAN_RX_ADD_VID
7834 static int igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
7836 static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
7839 struct igb_adapter *adapter = netdev_priv(netdev);
7840 int pf_id = adapter->vfs_allocated_count;
7843 #ifdef HAVE_VLAN_RX_REGISTER
7844 igb_irq_disable(adapter);
7846 vlan_group_set_device(adapter->vlgrp, vid, NULL);
7848 if (!test_bit(__IGB_DOWN, &adapter->state))
7849 igb_irq_enable(adapter);
7851 #endif /* HAVE_VLAN_RX_REGISTER */
7852 /* remove vlan from VLVF table array */
7853 err = igb_vlvf_set(adapter, vid, FALSE, pf_id);
7855 /* if vid was not present in VLVF just remove it from table */
7857 igb_vfta_set(adapter, vid, FALSE);
7858 #ifndef HAVE_VLAN_RX_REGISTER
7860 clear_bit(vid, adapter->active_vlans);
7862 #ifdef HAVE_INT_NDO_VLAN_RX_ADD_VID
7867 static void igb_restore_vlan(struct igb_adapter *adapter)
7869 #ifdef HAVE_VLAN_RX_REGISTER
7870 igb_vlan_mode(adapter->netdev, adapter->vlgrp);
7872 if (adapter->vlgrp) {
7874 for (vid = 0; vid < VLAN_N_VID; vid++) {
7875 if (!vlan_group_get_device(adapter->vlgrp, vid))
7877 igb_vlan_rx_add_vid(adapter->netdev, vid);
7883 igb_vlan_mode(adapter->netdev, adapter->netdev->features);
7885 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
7886 #ifdef HAVE_VLAN_PROTOCOL
7887 igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
7889 igb_vlan_rx_add_vid(adapter->netdev, vid);
7895 int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
7897 struct pci_dev *pdev = adapter->pdev;
7898 struct e1000_mac_info *mac = &adapter->hw.mac;
7902 /* Fiber NIC's only allow 1000 gbps Full duplex */
7903 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes ) &&
7904 spddplx != (SPEED_1000 + DUPLEX_FULL)) {
7905 dev_err(pci_dev_to_dev(pdev),
7906 "Unsupported Speed/Duplex configuration\n");
7911 case SPEED_10 + DUPLEX_HALF:
7912 mac->forced_speed_duplex = ADVERTISE_10_HALF;
7914 case SPEED_10 + DUPLEX_FULL:
7915 mac->forced_speed_duplex = ADVERTISE_10_FULL;
7917 case SPEED_100 + DUPLEX_HALF:
7918 mac->forced_speed_duplex = ADVERTISE_100_HALF;
7920 case SPEED_100 + DUPLEX_FULL:
7921 mac->forced_speed_duplex = ADVERTISE_100_FULL;
7923 case SPEED_1000 + DUPLEX_FULL:
7925 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
7927 case SPEED_1000 + DUPLEX_HALF: /* not supported */
7929 dev_err(pci_dev_to_dev(pdev), "Unsupported Speed/Duplex configuration\n");
7935 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
7938 struct net_device *netdev = pci_get_drvdata(pdev);
7939 struct igb_adapter *adapter = netdev_priv(netdev);
7940 struct e1000_hw *hw = &adapter->hw;
7941 u32 ctrl, rctl, status;
7942 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
7947 netif_device_detach(netdev);
7949 if (netif_running(netdev))
7950 __igb_close(netdev, true);
7952 igb_clear_interrupt_scheme(adapter);
7955 retval = pci_save_state(pdev);
7960 status = E1000_READ_REG(hw, E1000_STATUS);
7961 if (status & E1000_STATUS_LU)
7962 wufc &= ~E1000_WUFC_LNKC;
7965 igb_setup_rctl(adapter);
7966 igb_set_rx_mode(netdev);
7968 /* turn on all-multi mode if wake on multicast is enabled */
7969 if (wufc & E1000_WUFC_MC) {
7970 rctl = E1000_READ_REG(hw, E1000_RCTL);
7971 rctl |= E1000_RCTL_MPE;
7972 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
7975 ctrl = E1000_READ_REG(hw, E1000_CTRL);
7976 /* phy power management enable */
7977 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
7978 ctrl |= E1000_CTRL_ADVD3WUC;
7979 E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
7981 /* Allow time for pending master requests to run */
7982 e1000_disable_pcie_master(hw);
7984 E1000_WRITE_REG(hw, E1000_WUC, E1000_WUC_PME_EN);
7985 E1000_WRITE_REG(hw, E1000_WUFC, wufc);
7987 E1000_WRITE_REG(hw, E1000_WUC, 0);
7988 E1000_WRITE_REG(hw, E1000_WUFC, 0);
7991 *enable_wake = wufc || adapter->en_mng_pt;
7993 igb_power_down_link(adapter);
7995 igb_power_up_link(adapter);
7997 /* Release control of h/w to f/w. If f/w is AMT enabled, this
7998 * would have already happened in close and is redundant. */
7999 igb_release_hw_control(adapter);
8001 pci_disable_device(pdev);
8007 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
8008 static int igb_suspend(struct device *dev)
8012 struct pci_dev *pdev = to_pci_dev(dev);
8014 retval = __igb_shutdown(pdev, &wake, 0);
8019 pci_prepare_to_sleep(pdev);
8021 pci_wake_from_d3(pdev, false);
8022 pci_set_power_state(pdev, PCI_D3hot);
8028 static int igb_resume(struct device *dev)
8030 struct pci_dev *pdev = to_pci_dev(dev);
8031 struct net_device *netdev = pci_get_drvdata(pdev);
8032 struct igb_adapter *adapter = netdev_priv(netdev);
8033 struct e1000_hw *hw = &adapter->hw;
8036 pci_set_power_state(pdev, PCI_D0);
8037 pci_restore_state(pdev);
8038 pci_save_state(pdev);
8040 err = pci_enable_device_mem(pdev);
8042 dev_err(pci_dev_to_dev(pdev),
8043 "igb: Cannot enable PCI device from suspend\n");
8046 pci_set_master(pdev);
8048 pci_enable_wake(pdev, PCI_D3hot, 0);
8049 pci_enable_wake(pdev, PCI_D3cold, 0);
8051 #ifdef CONFIG_PM_RUNTIME
8052 if (!rtnl_is_locked()) {
8054 * shut up ASSERT_RTNL() warning in
8055 * netif_set_real_num_tx/rx_queues.
8058 err = igb_init_interrupt_scheme(adapter);
8061 err = igb_init_interrupt_scheme(adapter);
8065 if (igb_init_interrupt_scheme(adapter)) {
8066 #endif /* CONFIG_PM_RUNTIME */
8067 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for queues\n");
8073 /* let the f/w know that the h/w is now under the control of the
8075 igb_get_hw_control(adapter);
8077 E1000_WRITE_REG(hw, E1000_WUS, ~0);
8079 if (netdev->flags & IFF_UP) {
8080 err = __igb_open(netdev, true);
8085 netif_device_attach(netdev);
8090 #ifdef CONFIG_PM_RUNTIME
8091 static int igb_runtime_idle(struct device *dev)
8093 struct pci_dev *pdev = to_pci_dev(dev);
8094 struct net_device *netdev = pci_get_drvdata(pdev);
8095 struct igb_adapter *adapter = netdev_priv(netdev);
8097 if (!igb_has_link(adapter))
8098 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
8103 static int igb_runtime_suspend(struct device *dev)
8105 struct pci_dev *pdev = to_pci_dev(dev);
8109 retval = __igb_shutdown(pdev, &wake, 1);
8114 pci_prepare_to_sleep(pdev);
8116 pci_wake_from_d3(pdev, false);
8117 pci_set_power_state(pdev, PCI_D3hot);
8123 static int igb_runtime_resume(struct device *dev)
8125 return igb_resume(dev);
8127 #endif /* CONFIG_PM_RUNTIME */
8128 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
8129 #endif /* CONFIG_PM */
8131 #ifdef USE_REBOOT_NOTIFIER
8132 /* only want to do this for 2.4 kernels? */
8133 static int igb_notify_reboot(struct notifier_block *nb, unsigned long event,
8136 struct pci_dev *pdev = NULL;
8143 while ((pdev = pci_find_device(PCI_ANY_ID, PCI_ANY_ID, pdev))) {
8144 if (pci_dev_driver(pdev) == &igb_driver) {
8145 __igb_shutdown(pdev, &wake, 0);
8146 if (event == SYS_POWER_OFF) {
8147 pci_wake_from_d3(pdev, wake);
8148 pci_set_power_state(pdev, PCI_D3hot);
8156 static void igb_shutdown(struct pci_dev *pdev)
8160 __igb_shutdown(pdev, &wake, 0);
8162 if (system_state == SYSTEM_POWER_OFF) {
8163 pci_wake_from_d3(pdev, wake);
8164 pci_set_power_state(pdev, PCI_D3hot);
8167 #endif /* USE_REBOOT_NOTIFIER */
8169 #ifdef CONFIG_NET_POLL_CONTROLLER
8171 * Polling 'interrupt' - used by things like netconsole to send skbs
8172 * without having to re-enable interrupts. It's not called while
8173 * the interrupt routine is executing.
8175 static void igb_netpoll(struct net_device *netdev)
8177 struct igb_adapter *adapter = netdev_priv(netdev);
8178 struct e1000_hw *hw = &adapter->hw;
8179 struct igb_q_vector *q_vector;
8182 for (i = 0; i < adapter->num_q_vectors; i++) {
8183 q_vector = adapter->q_vector[i];
8184 if (adapter->msix_entries)
8185 E1000_WRITE_REG(hw, E1000_EIMC, q_vector->eims_value);
8187 igb_irq_disable(adapter);
8188 napi_schedule(&q_vector->napi);
8191 #endif /* CONFIG_NET_POLL_CONTROLLER */
8194 #define E1000_DEV_ID_82576_VF 0x10CA
8196 * igb_io_error_detected - called when PCI error is detected
8197 * @pdev: Pointer to PCI device
8198 * @state: The current pci connection state
8200 * This function is called after a PCI bus error affecting
8201 * this device has been detected.
8203 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
8204 pci_channel_state_t state)
8206 struct net_device *netdev = pci_get_drvdata(pdev);
8207 struct igb_adapter *adapter = netdev_priv(netdev);
8209 #ifdef CONFIG_PCI_IOV__UNUSED
8210 struct pci_dev *bdev, *vfdev;
8211 u32 dw0, dw1, dw2, dw3;
8213 u16 req_id, pf_func;
8215 if (!(adapter->flags & IGB_FLAG_DETECT_BAD_DMA))
8216 goto skip_bad_vf_detection;
8218 bdev = pdev->bus->self;
8219 while (bdev && (bdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT))
8220 bdev = bdev->bus->self;
8223 goto skip_bad_vf_detection;
8225 pos = pci_find_ext_capability(bdev, PCI_EXT_CAP_ID_ERR);
8227 goto skip_bad_vf_detection;
8229 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG, &dw0);
8230 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 4, &dw1);
8231 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 8, &dw2);
8232 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 12, &dw3);
8235 /* On the 82576 if bit 7 of the requestor ID is set then it's a VF */
8236 if (!(req_id & 0x0080))
8237 goto skip_bad_vf_detection;
8239 pf_func = req_id & 0x01;
8240 if ((pf_func & 1) == (pdev->devfn & 1)) {
8242 vf = (req_id & 0x7F) >> 1;
8243 dev_err(pci_dev_to_dev(pdev),
8244 "VF %d has caused a PCIe error\n", vf);
8245 dev_err(pci_dev_to_dev(pdev),
8246 "TLP: dw0: %8.8x\tdw1: %8.8x\tdw2: "
8247 "%8.8x\tdw3: %8.8x\n",
8248 dw0, dw1, dw2, dw3);
8250 /* Find the pci device of the offending VF */
8251 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL,
8252 E1000_DEV_ID_82576_VF, NULL);
8254 if (vfdev->devfn == (req_id & 0xFF))
8256 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL,
8257 E1000_DEV_ID_82576_VF, vfdev);
8260 * There's a slim chance the VF could have been hot plugged,
8261 * so if it is no longer present we don't need to issue the
8262 * VFLR. Just clean up the AER in that case.
8265 dev_err(pci_dev_to_dev(pdev),
8266 "Issuing VFLR to VF %d\n", vf);
8267 pci_write_config_dword(vfdev, 0xA8, 0x00008000);
8270 pci_cleanup_aer_uncorrect_error_status(pdev);
8274 * Even though the error may have occurred on the other port
8275 * we still need to increment the vf error reference count for
8276 * both ports because the I/O resume function will be called
8279 adapter->vferr_refcount++;
8281 return PCI_ERS_RESULT_RECOVERED;
8283 skip_bad_vf_detection:
8284 #endif /* CONFIG_PCI_IOV */
8286 netif_device_detach(netdev);
8288 if (state == pci_channel_io_perm_failure)
8289 return PCI_ERS_RESULT_DISCONNECT;
8291 if (netif_running(netdev))
8293 pci_disable_device(pdev);
8295 /* Request a slot slot reset. */
8296 return PCI_ERS_RESULT_NEED_RESET;
8300 * igb_io_slot_reset - called after the pci bus has been reset.
8301 * @pdev: Pointer to PCI device
8303 * Restart the card from scratch, as if from a cold-boot. Implementation
8304 * resembles the first-half of the igb_resume routine.
8306 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
8308 struct net_device *netdev = pci_get_drvdata(pdev);
8309 struct igb_adapter *adapter = netdev_priv(netdev);
8310 struct e1000_hw *hw = &adapter->hw;
8311 pci_ers_result_t result;
8313 if (pci_enable_device_mem(pdev)) {
8314 dev_err(pci_dev_to_dev(pdev),
8315 "Cannot re-enable PCI device after reset.\n");
8316 result = PCI_ERS_RESULT_DISCONNECT;
8318 pci_set_master(pdev);
8319 pci_restore_state(pdev);
8320 pci_save_state(pdev);
8322 pci_enable_wake(pdev, PCI_D3hot, 0);
8323 pci_enable_wake(pdev, PCI_D3cold, 0);
8325 schedule_work(&adapter->reset_task);
8326 E1000_WRITE_REG(hw, E1000_WUS, ~0);
8327 result = PCI_ERS_RESULT_RECOVERED;
8330 pci_cleanup_aer_uncorrect_error_status(pdev);
8336 * igb_io_resume - called when traffic can start flowing again.
8337 * @pdev: Pointer to PCI device
8339 * This callback is called when the error recovery driver tells us that
8340 * its OK to resume normal operation. Implementation resembles the
8341 * second-half of the igb_resume routine.
8343 static void igb_io_resume(struct pci_dev *pdev)
8345 struct net_device *netdev = pci_get_drvdata(pdev);
8346 struct igb_adapter *adapter = netdev_priv(netdev);
8348 if (adapter->vferr_refcount) {
8349 dev_info(pci_dev_to_dev(pdev), "Resuming after VF err\n");
8350 adapter->vferr_refcount--;
8354 if (netif_running(netdev)) {
8355 if (igb_up(adapter)) {
8356 dev_err(pci_dev_to_dev(pdev), "igb_up failed after reset\n");
8361 netif_device_attach(netdev);
8363 /* let the f/w know that the h/w is now under the control of the
8365 igb_get_hw_control(adapter);
8368 #endif /* HAVE_PCI_ERS */
8370 int igb_add_mac_filter(struct igb_adapter *adapter, u8 *addr, u16 queue)
8372 struct e1000_hw *hw = &adapter->hw;
8375 if (is_zero_ether_addr(addr))
8378 for (i = 0; i < hw->mac.rar_entry_count; i++) {
8379 if (adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE)
8381 adapter->mac_table[i].state = (IGB_MAC_STATE_MODIFIED |
8382 IGB_MAC_STATE_IN_USE);
8383 memcpy(adapter->mac_table[i].addr, addr, ETH_ALEN);
8384 adapter->mac_table[i].queue = queue;
8385 igb_sync_mac_table(adapter);
8390 int igb_del_mac_filter(struct igb_adapter *adapter, u8* addr, u16 queue)
8392 /* search table for addr, if found, set to 0 and sync */
8394 struct e1000_hw *hw = &adapter->hw;
8396 if (is_zero_ether_addr(addr))
8398 for (i = 0; i < hw->mac.rar_entry_count; i++) {
8399 if (!compare_ether_addr(addr, adapter->mac_table[i].addr) &&
8400 adapter->mac_table[i].queue == queue) {
8401 adapter->mac_table[i].state = IGB_MAC_STATE_MODIFIED;
8402 memset(adapter->mac_table[i].addr, 0, ETH_ALEN);
8403 adapter->mac_table[i].queue = 0;
8404 igb_sync_mac_table(adapter);
8410 static int igb_set_vf_mac(struct igb_adapter *adapter,
8411 int vf, unsigned char *mac_addr)
8413 igb_del_mac_filter(adapter, adapter->vf_data[vf].vf_mac_addresses, vf);
8414 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
8416 igb_add_mac_filter(adapter, mac_addr, vf);
8422 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
8424 struct igb_adapter *adapter = netdev_priv(netdev);
8425 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
8427 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
8428 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
8429 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
8430 " change effective.\n");
8431 if (test_bit(__IGB_DOWN, &adapter->state)) {
8432 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
8433 " but the PF device is not up.\n");
8434 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
8435 " attempting to use the VF device.\n");
8437 return igb_set_vf_mac(adapter, vf, mac);
8440 static int igb_link_mbps(int internal_link_speed)
8442 switch (internal_link_speed) {
8452 static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
8459 /* Calculate the rate factor values to set */
8460 rf_int = link_speed / tx_rate;
8461 rf_dec = (link_speed - (rf_int * tx_rate));
8462 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
8464 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
8465 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
8466 E1000_RTTBCNRC_RF_INT_MASK);
8467 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
8472 E1000_WRITE_REG(hw, E1000_RTTDQSEL, vf); /* vf X uses queue X */
8474 * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
8475 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
8477 E1000_WRITE_REG(hw, E1000_RTTBCNRM(0), 0x14);
8478 E1000_WRITE_REG(hw, E1000_RTTBCNRC, bcnrc_val);
8481 static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
8483 int actual_link_speed, i;
8484 bool reset_rate = false;
8486 /* VF TX rate limit was not set */
8487 if ((adapter->vf_rate_link_speed == 0) ||
8488 (adapter->hw.mac.type != e1000_82576))
8491 actual_link_speed = igb_link_mbps(adapter->link_speed);
8492 if (actual_link_speed != adapter->vf_rate_link_speed) {
8494 adapter->vf_rate_link_speed = 0;
8495 dev_info(&adapter->pdev->dev,
8496 "Link speed has been changed. VF Transmit rate is disabled\n");
8499 for (i = 0; i < adapter->vfs_allocated_count; i++) {
8501 adapter->vf_data[i].tx_rate = 0;
8503 igb_set_vf_rate_limit(&adapter->hw, i,
8504 adapter->vf_data[i].tx_rate, actual_link_speed);
8508 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
8510 struct igb_adapter *adapter = netdev_priv(netdev);
8511 struct e1000_hw *hw = &adapter->hw;
8512 int actual_link_speed;
8514 if (hw->mac.type != e1000_82576)
8517 actual_link_speed = igb_link_mbps(adapter->link_speed);
8518 if ((vf >= adapter->vfs_allocated_count) ||
8519 (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) ||
8520 (tx_rate < 0) || (tx_rate > actual_link_speed))
8523 adapter->vf_rate_link_speed = actual_link_speed;
8524 adapter->vf_data[vf].tx_rate = (u16)tx_rate;
8525 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
8530 static int igb_ndo_get_vf_config(struct net_device *netdev,
8531 int vf, struct ifla_vf_info *ivi)
8533 struct igb_adapter *adapter = netdev_priv(netdev);
8534 if (vf >= adapter->vfs_allocated_count)
8537 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
8538 ivi->tx_rate = adapter->vf_data[vf].tx_rate;
8539 ivi->vlan = adapter->vf_data[vf].pf_vlan;
8540 ivi->qos = adapter->vf_data[vf].pf_qos;
8544 static void igb_vmm_control(struct igb_adapter *adapter)
8546 struct e1000_hw *hw = &adapter->hw;
8549 switch (hw->mac.type) {
8552 /* replication is not supported for 82575 */
8555 /* notify HW that the MAC is adding vlan tags */
8556 reg = E1000_READ_REG(hw, E1000_DTXCTL);
8557 reg |= (E1000_DTXCTL_VLAN_ADDED |
8558 E1000_DTXCTL_SPOOF_INT);
8559 E1000_WRITE_REG(hw, E1000_DTXCTL, reg);
8561 /* enable replication vlan tag stripping */
8562 reg = E1000_READ_REG(hw, E1000_RPLOLR);
8563 reg |= E1000_RPLOLR_STRVLAN;
8564 E1000_WRITE_REG(hw, E1000_RPLOLR, reg);
8566 /* none of the above registers are supported by i350 */
8570 /* Enable Malicious Driver Detection */
8571 if ((hw->mac.type == e1000_i350) && (adapter->vfs_allocated_count) &&
8573 igb_enable_mdd(adapter);
8575 /* enable replication and loopback support */
8576 e1000_vmdq_set_loopback_pf(hw, adapter->vfs_allocated_count ||
8577 adapter->vmdq_pools);
8579 e1000_vmdq_set_anti_spoofing_pf(hw, adapter->vfs_allocated_count ||
8580 adapter->vmdq_pools,
8581 adapter->vfs_allocated_count);
8582 e1000_vmdq_set_replication_pf(hw, adapter->vfs_allocated_count ||
8583 adapter->vmdq_pools);
8586 static void igb_init_fw(struct igb_adapter *adapter)
8588 struct e1000_fw_drv_info fw_cmd;
8589 struct e1000_hw *hw = &adapter->hw;
8593 mask = E1000_SWFW_PHY0_SM;
8595 if (!hw->mac.ops.acquire_swfw_sync(hw, mask)) {
8596 for (i = 0; i <= FW_MAX_RETRIES; i++) {
8597 E1000_WRITE_REG(hw, E1000_FWSTS, E1000_FWSTS_FWRI);
8598 fw_cmd.hdr.cmd = FW_CMD_DRV_INFO;
8599 fw_cmd.hdr.buf_len = FW_CMD_DRV_INFO_LEN;
8600 fw_cmd.hdr.cmd_or_resp.cmd_resv = FW_CMD_RESERVED;
8601 fw_cmd.port_num = hw->bus.func;
8602 fw_cmd.drv_version = FW_FAMILY_DRV_VER;
8603 fw_cmd.hdr.checksum = 0;
8604 fw_cmd.hdr.checksum = e1000_calculate_checksum((u8 *)&fw_cmd,
8606 fw_cmd.hdr.buf_len));
8607 e1000_host_interface_command(hw, (u8*)&fw_cmd,
8609 if (fw_cmd.hdr.cmd_or_resp.ret_status == FW_STATUS_SUCCESS)
8613 dev_warn(pci_dev_to_dev(adapter->pdev),
8614 "Unable to get semaphore, firmware init failed.\n");
8615 hw->mac.ops.release_swfw_sync(hw, mask);
8618 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
8620 struct e1000_hw *hw = &adapter->hw;
8624 if (hw->mac.type > e1000_82580) {
8625 if (adapter->dmac != IGB_DMAC_DISABLE) {
8628 /* force threshold to 0. */
8629 E1000_WRITE_REG(hw, E1000_DMCTXTH, 0);
8632 * DMA Coalescing high water mark needs to be greater
8633 * than the Rx threshold. Set hwm to PBA - max frame
8634 * size in 16B units, capping it at PBA - 6KB.
8636 hwm = 64 * pba - adapter->max_frame_size / 16;
8637 if (hwm < 64 * (pba - 6))
8638 hwm = 64 * (pba - 6);
8639 reg = E1000_READ_REG(hw, E1000_FCRTC);
8640 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
8641 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
8642 & E1000_FCRTC_RTH_COAL_MASK);
8643 E1000_WRITE_REG(hw, E1000_FCRTC, reg);
8646 * Set the DMA Coalescing Rx threshold to PBA - 2 * max
8647 * frame size, capping it at PBA - 10KB.
8649 dmac_thr = pba - adapter->max_frame_size / 512;
8650 if (dmac_thr < pba - 10)
8651 dmac_thr = pba - 10;
8652 reg = E1000_READ_REG(hw, E1000_DMACR);
8653 reg &= ~E1000_DMACR_DMACTHR_MASK;
8654 reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
8655 & E1000_DMACR_DMACTHR_MASK);
8657 /* transition to L0x or L1 if available..*/
8658 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
8660 /* watchdog timer= msec values in 32usec intervals */
8661 reg |= ((adapter->dmac) >> 5);
8662 E1000_WRITE_REG(hw, E1000_DMACR, reg);
8664 /* no lower threshold to disable coalescing(smart fifb)-UTRESH=0*/
8665 E1000_WRITE_REG(hw, E1000_DMCRTRH, 0);
8668 * This sets the time to wait before requesting transition to
8669 * low power state to number of usecs needed to receive 1 512
8670 * byte frame at gigabit line rate
8672 reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
8674 E1000_WRITE_REG(hw, E1000_DMCTLX, reg);
8676 /* free space in tx packet buffer to wake from DMA coal */
8677 E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
8678 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
8680 /* make low power state decision controlled by DMA coal */
8681 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
8682 reg &= ~E1000_PCIEMISC_LX_DECISION;
8683 E1000_WRITE_REG(hw, E1000_PCIEMISC, reg);
8684 } /* endif adapter->dmac is not disabled */
8685 } else if (hw->mac.type == e1000_82580) {
8686 u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
8687 E1000_WRITE_REG(hw, E1000_PCIEMISC,
8688 reg & ~E1000_PCIEMISC_LX_DECISION);
8689 E1000_WRITE_REG(hw, E1000_DMACR, 0);
8697 * igb_probe - Device Initialization Routine
8698 * @pdev: PCI device information struct
8699 * @ent: entry in igb_pci_tbl
8701 * Returns 0 on success, negative on failure
8703 * igb_probe initializes an adapter identified by a pci_dev structure.
8704 * The OS initialization, configuring of the adapter private structure,
8705 * and a hardware reset occur.
8707 int igb_kni_probe(struct pci_dev *pdev,
8708 struct net_device **lad_dev)
8710 struct net_device *netdev;
8711 struct igb_adapter *adapter;
8712 struct e1000_hw *hw;
8713 u16 eeprom_data = 0;
8714 u8 pba_str[E1000_PBANUM_LENGTH];
8716 static int global_quad_port_a; /* global quad port a indication */
8717 int i, err, pci_using_dac = 0;
8718 static int cards_found;
8720 err = pci_enable_device_mem(pdev);
8726 err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
8728 err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
8732 err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
8734 err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
8736 IGB_ERR("No usable DMA configuration, "
8743 #ifndef HAVE_ASPM_QUIRKS
8744 /* 82575 requires that the pci-e link partner disable the L0s state */
8745 switch (pdev->device) {
8746 case E1000_DEV_ID_82575EB_COPPER:
8747 case E1000_DEV_ID_82575EB_FIBER_SERDES:
8748 case E1000_DEV_ID_82575GB_QUAD_COPPER:
8749 pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S);
8754 #endif /* HAVE_ASPM_QUIRKS */
8755 err = pci_request_selected_regions(pdev,
8756 pci_select_bars(pdev,
8762 pci_enable_pcie_error_reporting(pdev);
8764 pci_set_master(pdev);
8768 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
8771 netdev = alloc_etherdev(sizeof(struct igb_adapter));
8772 #endif /* HAVE_TX_MQ */
8774 goto err_alloc_etherdev;
8777 SET_MODULE_OWNER(netdev);
8778 SET_NETDEV_DEV(netdev, &pdev->dev);
8780 //pci_set_drvdata(pdev, netdev);
8782 adapter = netdev_priv(netdev);
8783 adapter->netdev = netdev;
8784 adapter->pdev = pdev;
8787 adapter->port_num = hw->bus.func;
8788 adapter->msg_enable = (1 << debug) - 1;
8791 err = pci_save_state(pdev);
8796 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
8797 pci_resource_len(pdev, 0));
8801 #ifdef HAVE_NET_DEVICE_OPS
8802 netdev->netdev_ops = &igb_netdev_ops;
8803 #else /* HAVE_NET_DEVICE_OPS */
8804 netdev->open = &igb_open;
8805 netdev->stop = &igb_close;
8806 netdev->get_stats = &igb_get_stats;
8807 #ifdef HAVE_SET_RX_MODE
8808 netdev->set_rx_mode = &igb_set_rx_mode;
8810 netdev->set_multicast_list = &igb_set_rx_mode;
8811 netdev->set_mac_address = &igb_set_mac;
8812 netdev->change_mtu = &igb_change_mtu;
8813 netdev->do_ioctl = &igb_ioctl;
8814 #ifdef HAVE_TX_TIMEOUT
8815 netdev->tx_timeout = &igb_tx_timeout;
8817 netdev->vlan_rx_register = igb_vlan_mode;
8818 netdev->vlan_rx_add_vid = igb_vlan_rx_add_vid;
8819 netdev->vlan_rx_kill_vid = igb_vlan_rx_kill_vid;
8820 #ifdef CONFIG_NET_POLL_CONTROLLER
8821 netdev->poll_controller = igb_netpoll;
8823 netdev->hard_start_xmit = &igb_xmit_frame;
8824 #endif /* HAVE_NET_DEVICE_OPS */
8825 igb_set_ethtool_ops(netdev);
8826 #ifdef HAVE_TX_TIMEOUT
8827 netdev->watchdog_timeo = 5 * HZ;
8830 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
8832 adapter->bd_number = cards_found;
8834 /* setup the private structure */
8835 err = igb_sw_init(adapter);
8839 e1000_get_bus_info(hw);
8841 hw->phy.autoneg_wait_to_complete = FALSE;
8842 hw->mac.adaptive_ifs = FALSE;
8844 /* Copper options */
8845 if (hw->phy.media_type == e1000_media_type_copper) {
8847 hw->phy.mdix = ETH_TP_MDI_INVALID;
8849 hw->phy.mdix = AUTO_ALL_MODES;
8850 #endif /* ETH_TP_MDI_X */
8851 hw->phy.disable_polarity_correction = FALSE;
8852 hw->phy.ms_type = e1000_ms_hw_default;
8855 if (e1000_check_reset_block(hw))
8856 dev_info(pci_dev_to_dev(pdev),
8857 "PHY reset is blocked due to SOL/IDER session.\n");
8860 * features is initialized to 0 in allocation, it might have bits
8861 * set by igb_sw_init so we should use an or instead of an
8864 netdev->features |= NETIF_F_SG |
8866 #ifdef NETIF_F_IPV6_CSUM
8874 #endif /* NETIF_F_TSO */
8875 #ifdef NETIF_F_RXHASH
8878 #ifdef HAVE_NDO_SET_FEATURES
8881 NETIF_F_HW_VLAN_RX |
8884 #ifdef HAVE_NDO_SET_FEATURES
8885 /* copy netdev features into list of user selectable features */
8886 netdev->hw_features |= netdev->features;
8889 /* give us the option of enabling LRO later */
8890 netdev->hw_features |= NETIF_F_LRO;
8895 /* this is only needed on kernels prior to 2.6.39 */
8896 netdev->features |= NETIF_F_GRO;
8900 /* set this bit last since it cannot be part of hw_features */
8901 netdev->features |= NETIF_F_HW_VLAN_FILTER;
8903 #ifdef HAVE_NETDEV_VLAN_FEATURES
8904 netdev->vlan_features |= NETIF_F_TSO |
8912 netdev->features |= NETIF_F_HIGHDMA;
8914 if (hw->mac.type >= e1000_82576)
8915 netdev->features |= NETIF_F_SCTP_CSUM;
8918 adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
8920 /* before reading the NVM, reset the controller to put the device in a
8921 * known good starting state */
8925 /* make sure the NVM is good */
8926 if (e1000_validate_nvm_checksum(hw) < 0) {
8927 dev_err(pci_dev_to_dev(pdev), "The NVM Checksum Is Not"
8933 /* copy the MAC address out of the NVM */
8934 if (e1000_read_mac_addr(hw))
8935 dev_err(pci_dev_to_dev(pdev), "NVM Read Error\n");
8936 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
8937 #ifdef ETHTOOL_GPERMADDR
8938 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
8940 if (!is_valid_ether_addr(netdev->perm_addr)) {
8942 if (!is_valid_ether_addr(netdev->dev_addr)) {
8944 dev_err(pci_dev_to_dev(pdev), "Invalid MAC Address\n");
8949 memcpy(&adapter->mac_table[0].addr, hw->mac.addr, netdev->addr_len);
8950 adapter->mac_table[0].queue = adapter->vfs_allocated_count;
8951 adapter->mac_table[0].state = (IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE);
8952 igb_rar_set(adapter, 0);
8954 /* get firmware version for ethtool -i */
8955 e1000_read_nvm(&adapter->hw, 5, 1, &adapter->fw_version);
8957 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
8958 (unsigned long) adapter);
8959 if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
8960 setup_timer(&adapter->dma_err_timer, &igb_dma_err_timer,
8961 (unsigned long) adapter);
8962 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
8963 (unsigned long) adapter);
8965 INIT_WORK(&adapter->reset_task, igb_reset_task);
8966 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
8967 if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
8968 INIT_WORK(&adapter->dma_err_task, igb_dma_err_task);
8971 /* Initialize link properties that are user-changeable */
8972 adapter->fc_autoneg = true;
8973 hw->mac.autoneg = true;
8974 hw->phy.autoneg_advertised = 0x2f;
8976 hw->fc.requested_mode = e1000_fc_default;
8977 hw->fc.current_mode = e1000_fc_default;
8979 e1000_validate_mdi_setting(hw);
8981 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
8982 * enable the ACPI Magic Packet filter
8985 if (hw->bus.func == 0)
8986 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
8987 else if (hw->mac.type >= e1000_82580)
8988 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
8989 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
8991 else if (hw->bus.func == 1)
8992 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
8994 if (eeprom_data & IGB_EEPROM_APME)
8995 adapter->eeprom_wol |= E1000_WUFC_MAG;
8997 /* now that we have the eeprom settings, apply the special cases where
8998 * the eeprom may be wrong or the board simply won't support wake on
8999 * lan on a particular port */
9000 switch (pdev->device) {
9001 case E1000_DEV_ID_82575GB_QUAD_COPPER:
9002 adapter->eeprom_wol = 0;
9004 case E1000_DEV_ID_82575EB_FIBER_SERDES:
9005 case E1000_DEV_ID_82576_FIBER:
9006 case E1000_DEV_ID_82576_SERDES:
9007 /* Wake events only supported on port A for dual fiber
9008 * regardless of eeprom setting */
9009 if (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_FUNC_1)
9010 adapter->eeprom_wol = 0;
9012 case E1000_DEV_ID_82576_QUAD_COPPER:
9013 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
9014 /* if quad port adapter, disable WoL on all but port A */
9015 if (global_quad_port_a != 0)
9016 adapter->eeprom_wol = 0;
9018 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
9019 /* Reset for multiple quad port adapters */
9020 if (++global_quad_port_a == 4)
9021 global_quad_port_a = 0;
9025 /* initialize the wol settings based on the eeprom settings */
9026 adapter->wol = adapter->eeprom_wol;
9028 device_set_wakeup_enable(pci_dev_to_dev(adapter->pdev), adapter->wol);
9030 /* reset the hardware with the new settings */
9033 /* let the f/w know that the h/w is now under the control of the
9035 igb_get_hw_control(adapter);
9037 strncpy(netdev->name, "eth%d", IFNAMSIZ);
9038 err = register_netdev(netdev);
9042 #ifdef CONFIG_IGB_VMDQ_NETDEV
9043 err = igb_init_vmdq_netdevs(adapter);
9047 /* carrier off reporting is important to ethtool even BEFORE open */
9048 netif_carrier_off(netdev);
9051 if (dca_add_requester(&pdev->dev) == E1000_SUCCESS) {
9052 adapter->flags |= IGB_FLAG_DCA_ENABLED;
9053 dev_info(pci_dev_to_dev(pdev), "DCA enabled\n");
9054 igb_setup_dca(adapter);
9058 #ifdef HAVE_HW_TIME_STAMP
9059 /* do hw tstamp init after resetting */
9060 igb_init_hw_timer(adapter);
9065 dev_info(pci_dev_to_dev(pdev), "Intel(R) Gigabit Ethernet Network Connection\n");
9066 /* print bus type/speed/width info */
9067 dev_info(pci_dev_to_dev(pdev), "%s: (PCIe:%s:%s) ",
9069 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5GT/s" :
9070 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0GT/s" :
9072 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4\n" :
9073 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2\n" :
9074 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1\n" :
9076 dev_info(pci_dev_to_dev(pdev), "%s: MAC: ", netdev->name);
9077 for (i = 0; i < 6; i++)
9078 printk("%2.2x%c", netdev->dev_addr[i], i == 5 ? '\n' : ':');
9080 ret_val = e1000_read_pba_string(hw, pba_str, E1000_PBANUM_LENGTH);
9082 strncpy(pba_str, "Unknown", sizeof(pba_str) - 1);
9083 dev_info(pci_dev_to_dev(pdev), "%s: PBA No: %s\n", netdev->name,
9086 /* Initialize the thermal sensor on i350 devices. */
9087 if (hw->mac.type == e1000_i350 && hw->bus.func == 0) {
9091 * Read the NVM to determine if this i350 device supports an
9092 * external thermal sensor.
9094 e1000_read_nvm(hw, NVM_ETS_CFG, 1, &ets_word);
9095 if (ets_word != 0x0000 && ets_word != 0xFFFF)
9096 adapter->ets = true;
9098 adapter->ets = false;
9101 igb_sysfs_init(adapter);
9104 igb_procfs_init(adapter);
9105 #endif /* IGB_PROCFS */
9106 #endif /* IGB_SYSFS */
9109 adapter->ets = false;
9112 switch (hw->mac.type) {
9114 /* Enable EEE for internal copper PHY devices */
9115 if (hw->phy.media_type == e1000_media_type_copper)
9116 e1000_set_eee_i350(hw);
9118 /* send driver version info to firmware */
9119 igb_init_fw(adapter);
9125 if (netdev->features & NETIF_F_LRO)
9126 dev_info(pci_dev_to_dev(pdev), "Internal LRO is enabled \n");
9128 dev_info(pci_dev_to_dev(pdev), "LRO is disabled \n");
9130 dev_info(pci_dev_to_dev(pdev),
9131 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
9132 adapter->msix_entries ? "MSI-X" :
9133 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9134 adapter->num_rx_queues, adapter->num_tx_queues);
9139 pm_runtime_put_noidle(&pdev->dev);
9143 //igb_release_hw_control(adapter);
9145 //if (!e1000_check_reset_block(hw))
9146 // e1000_phy_hw_reset(hw);
9148 if (hw->flash_address)
9149 iounmap(hw->flash_address);
9151 //igb_clear_interrupt_scheme(adapter);
9152 //igb_reset_sriov_capability(adapter);
9153 iounmap(hw->hw_addr);
9155 free_netdev(netdev);
9157 //pci_release_selected_regions(pdev,
9158 // pci_select_bars(pdev, IORESOURCE_MEM));
9161 pci_disable_device(pdev);
9166 void igb_kni_remove(struct pci_dev *pdev)
9168 pci_disable_device(pdev);