1 /*******************************************************************************
3 Intel(R) Gigabit Ethernet Linux driver
4 Copyright(c) 2007-2012 Intel Corporation.
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
26 *******************************************************************************/
28 #include <linux/module.h>
29 #include <linux/types.h>
30 #include <linux/init.h>
31 #include <linux/vmalloc.h>
32 #include <linux/pagemap.h>
33 #include <linux/netdevice.h>
34 #include <linux/tcp.h>
36 #include <net/checksum.h>
38 #include <linux/ipv6.h>
39 #include <net/ip6_checksum.h>
43 #include <linux/mii.h>
46 #include <linux/ethtool.h>
48 #include <linux/if_vlan.h>
49 #ifdef CONFIG_PM_RUNTIME
50 #include <linux/pm_runtime.h>
51 #endif /* CONFIG_PM_RUNTIME */
56 #include <linux/uio_driver.h>
60 #define VERSION_SUFFIX
65 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." __stringify(BUILD) VERSION_SUFFIX DRV_DEBUG DRV_HW_PERF
67 char igb_driver_name[] = "igb";
68 char igb_driver_version[] = DRV_VERSION;
69 static const char igb_driver_string[] =
70 "Intel(R) Gigabit Ethernet Network Driver";
71 static const char igb_copyright[] = "Copyright (c) 2007-2011 Intel Corporation.";
73 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER) },
75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER) },
76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES) },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII) },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER) },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER) },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER) },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES) },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII) },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL) },
87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII) },
88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES) },
89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE) },
90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP) },
91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576) },
92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS) },
93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES) },
94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER) },
95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES) },
96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD) },
97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2) },
98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER) },
99 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER) },
100 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES) },
101 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER) },
102 /* required last entry */
106 //MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
107 static void igb_set_sriov_capability(struct igb_adapter *adapter) __attribute__((__unused__));
108 void igb_reset(struct igb_adapter *);
109 static int igb_setup_all_tx_resources(struct igb_adapter *);
110 static int igb_setup_all_rx_resources(struct igb_adapter *);
111 static void igb_free_all_tx_resources(struct igb_adapter *);
112 static void igb_free_all_rx_resources(struct igb_adapter *);
113 static void igb_setup_mrqc(struct igb_adapter *);
114 void igb_update_stats(struct igb_adapter *);
115 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
116 static void __devexit igb_remove(struct pci_dev *pdev);
117 #ifdef HAVE_HW_TIME_STAMP
118 static void igb_init_hw_timer(struct igb_adapter *adapter);
120 static int igb_sw_init(struct igb_adapter *);
121 static int igb_open(struct net_device *);
122 static int igb_close(struct net_device *);
123 static void igb_configure_tx(struct igb_adapter *);
124 static void igb_configure_rx(struct igb_adapter *);
125 static void igb_clean_all_tx_rings(struct igb_adapter *);
126 static void igb_clean_all_rx_rings(struct igb_adapter *);
127 static void igb_clean_tx_ring(struct igb_ring *);
128 static void igb_set_rx_mode(struct net_device *);
129 static void igb_update_phy_info(unsigned long);
130 static void igb_watchdog(unsigned long);
131 static void igb_watchdog_task(struct work_struct *);
132 static void igb_dma_err_task(struct work_struct *);
133 static void igb_dma_err_timer(unsigned long data);
134 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
135 static struct net_device_stats *igb_get_stats(struct net_device *);
136 static int igb_change_mtu(struct net_device *, int);
137 void igb_full_sync_mac_table(struct igb_adapter *adapter);
138 static int igb_set_mac(struct net_device *, void *);
139 static void igb_set_uta(struct igb_adapter *adapter);
140 static irqreturn_t igb_intr(int irq, void *);
141 static irqreturn_t igb_intr_msi(int irq, void *);
142 static irqreturn_t igb_msix_other(int irq, void *);
143 static irqreturn_t igb_msix_ring(int irq, void *);
145 static void igb_update_dca(struct igb_q_vector *);
146 static void igb_setup_dca(struct igb_adapter *);
148 static int igb_poll(struct napi_struct *, int);
149 static bool igb_clean_tx_irq(struct igb_q_vector *);
150 static bool igb_clean_rx_irq(struct igb_q_vector *, int);
151 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
152 static void igb_tx_timeout(struct net_device *);
153 static void igb_reset_task(struct work_struct *);
154 #ifdef HAVE_VLAN_RX_REGISTER
155 static void igb_vlan_mode(struct net_device *, struct vlan_group *);
157 #ifdef HAVE_VLAN_PROTOCOL
158 static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16);
159 static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16);
160 #elif defined HAVE_INT_NDO_VLAN_RX_ADD_VID
161 static int igb_vlan_rx_add_vid(struct net_device *, u16);
162 static int igb_vlan_rx_kill_vid(struct net_device *, u16);
164 static void igb_vlan_rx_add_vid(struct net_device *, u16);
165 static void igb_vlan_rx_kill_vid(struct net_device *, u16);
167 static void igb_restore_vlan(struct igb_adapter *);
168 void igb_rar_set(struct igb_adapter *adapter, u32 index);
169 static void igb_ping_all_vfs(struct igb_adapter *);
170 static void igb_msg_task(struct igb_adapter *);
171 static void igb_vmm_control(struct igb_adapter *);
172 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
173 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
174 static void igb_process_mdd_event(struct igb_adapter *);
176 static int igb_ndo_set_vf_mac( struct net_device *netdev, int vf, u8 *mac);
177 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
178 int vf, u16 vlan, u8 qos);
179 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
180 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
181 struct ifla_vf_info *ivi);
182 static void igb_check_vf_rate_limit(struct igb_adapter *);
184 static int igb_vf_configure(struct igb_adapter *adapter, int vf);
185 static int igb_check_vf_assignment(struct igb_adapter *adapter);
186 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
187 static int igb_find_enabled_vfs(struct igb_adapter *adapter);
190 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
191 static int igb_suspend(struct device *dev);
192 static int igb_resume(struct device *dev);
193 #ifdef CONFIG_PM_RUNTIME
194 static int igb_runtime_suspend(struct device *dev);
195 static int igb_runtime_resume(struct device *dev);
196 static int igb_runtime_idle(struct device *dev);
197 #endif /* CONFIG_PM_RUNTIME */
198 static const struct dev_pm_ops igb_pm_ops = {
199 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,34)
200 .suspend = igb_suspend,
201 .resume = igb_resume,
202 .freeze = igb_suspend,
204 .poweroff = igb_suspend,
205 .restore = igb_resume,
206 #ifdef CONFIG_PM_RUNTIME
207 .runtime_suspend = igb_runtime_suspend,
208 .runtime_resume = igb_runtime_resume,
209 .runtime_idle = igb_runtime_idle,
211 #else /* Linux >= 2.6.34 */
212 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
213 #ifdef CONFIG_PM_RUNTIME
214 SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
216 #endif /* CONFIG_PM_RUNTIME */
217 #endif /* Linux version */
219 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
220 #endif /* CONFIG_PM */
221 #ifndef USE_REBOOT_NOTIFIER
222 static void igb_shutdown(struct pci_dev *);
224 static int igb_notify_reboot(struct notifier_block *, unsigned long, void *);
225 static struct notifier_block igb_notifier_reboot = {
226 .notifier_call = igb_notify_reboot,
232 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
233 static struct notifier_block dca_notifier = {
234 .notifier_call = igb_notify_dca,
239 #ifdef CONFIG_NET_POLL_CONTROLLER
240 /* for netdump / net console */
241 static void igb_netpoll(struct net_device *);
245 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
246 pci_channel_state_t);
247 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
248 static void igb_io_resume(struct pci_dev *);
250 static struct pci_error_handlers igb_err_handler = {
251 .error_detected = igb_io_error_detected,
252 .slot_reset = igb_io_slot_reset,
253 .resume = igb_io_resume,
257 static void igb_init_fw(struct igb_adapter *adapter);
258 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
260 static struct pci_driver igb_driver = {
261 .name = igb_driver_name,
262 .id_table = igb_pci_tbl,
264 .remove = __devexit_p(igb_remove),
266 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
267 .driver.pm = &igb_pm_ops,
268 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
269 #endif /* CONFIG_PM */
270 #ifndef USE_REBOOT_NOTIFIER
271 .shutdown = igb_shutdown,
274 .err_handler = &igb_err_handler
278 //MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
279 //MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
280 //MODULE_LICENSE("GPL");
281 //MODULE_VERSION(DRV_VERSION);
283 static void igb_vfta_set(struct igb_adapter *adapter, u32 vid, bool add)
285 struct e1000_hw *hw = &adapter->hw;
286 struct e1000_host_mng_dhcp_cookie *mng_cookie = &hw->mng_cookie;
287 u32 index = (vid >> E1000_VFTA_ENTRY_SHIFT) & E1000_VFTA_ENTRY_MASK;
288 u32 mask = 1 << (vid & E1000_VFTA_ENTRY_BIT_SHIFT_MASK);
292 * if this is the management vlan the only option is to add it in so
293 * that the management pass through will continue to work
295 if ((mng_cookie->status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
296 (vid == mng_cookie->vlan_id))
299 vfta = adapter->shadow_vfta[index];
306 e1000_write_vfta(hw, index, vfta);
307 adapter->shadow_vfta[index] = vfta;
310 #ifdef HAVE_HW_TIME_STAMP
312 * igb_read_clock - read raw cycle counter (to be used by time counter)
314 static cycle_t igb_read_clock(const struct cyclecounter *tc)
316 struct igb_adapter *adapter =
317 container_of(tc, struct igb_adapter, cycles);
318 struct e1000_hw *hw = &adapter->hw;
323 * The timestamp latches on lowest register read. For the 82580
324 * the lowest register is SYSTIMR instead of SYSTIML. However we never
325 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
327 if (hw->mac.type >= e1000_82580) {
328 stamp = E1000_READ_REG(hw, E1000_SYSTIMR) >> 8;
329 shift = IGB_82580_TSYNC_SHIFT;
332 stamp |= (u64)E1000_READ_REG(hw, E1000_SYSTIML) << shift;
333 stamp |= (u64)E1000_READ_REG(hw, E1000_SYSTIMH) << (shift + 32);
337 #endif /* SIOCSHWTSTAMP */
338 static int debug = NETIF_MSG_DRV | NETIF_MSG_PROBE;
339 //module_param(debug, int, 0);
340 //MODULE_PARM_DESC(debug, "Debug level (0=none, ..., 16=all)");
343 * igb_init_module - Driver Registration Routine
345 * igb_init_module is the first routine called when the driver is
346 * loaded. All it does is register with the PCI subsystem.
348 static int __init igb_init_module(void)
352 printk(KERN_INFO "%s - version %s\n",
353 igb_driver_string, igb_driver_version);
355 printk(KERN_INFO "%s\n", igb_copyright);
357 /* only use IGB_PROCFS if IGB_SYSFS is not defined */
360 if (igb_procfs_topdir_init())
361 printk(KERN_INFO "Procfs failed to initialize topdir\n");
362 #endif /* IGB_PROCFS */
363 #endif /* IGB_SYSFS */
366 dca_register_notify(&dca_notifier);
368 ret = pci_register_driver(&igb_driver);
369 #ifdef USE_REBOOT_NOTIFIER
371 register_reboot_notifier(&igb_notifier_reboot);
378 #define module_init(x) static int x(void) __attribute__((__unused__));
379 module_init(igb_init_module);
382 * igb_exit_module - Driver Exit Cleanup Routine
384 * igb_exit_module is called just before the driver is removed
387 static void __exit igb_exit_module(void)
390 dca_unregister_notify(&dca_notifier);
392 #ifdef USE_REBOOT_NOTIFIER
393 unregister_reboot_notifier(&igb_notifier_reboot);
395 pci_unregister_driver(&igb_driver);
398 /* only compile IGB_PROCFS if IGB_SYSFS is not defined */
401 igb_procfs_topdir_exit();
402 #endif /* IGB_PROCFS */
403 #endif /* IGB_SYSFS */
407 #define module_exit(x) static void x(void) __attribute__((__unused__));
408 module_exit(igb_exit_module);
410 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
412 * igb_cache_ring_register - Descriptor ring to register mapping
413 * @adapter: board private structure to initialize
415 * Once we know the feature-set enabled for the device, we'll cache
416 * the register offset the descriptor ring is assigned to.
418 static void igb_cache_ring_register(struct igb_adapter *adapter)
421 u32 rbase_offset = adapter->vfs_allocated_count;
423 switch (adapter->hw.mac.type) {
425 /* The queues are allocated for virtualization such that VF 0
426 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
427 * In order to avoid collision we start at the first free queue
428 * and continue consuming queues in the same sequence
430 if ((adapter->rss_queues > 1) && adapter->vmdq_pools) {
431 for (; i < adapter->rss_queues; i++)
432 adapter->rx_ring[i]->reg_idx = rbase_offset +
439 for (; i < adapter->num_rx_queues; i++)
440 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
441 for (; j < adapter->num_tx_queues; j++)
442 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
447 static void igb_free_queues(struct igb_adapter *adapter)
451 for (i = 0; i < adapter->num_tx_queues; i++) {
452 kfree(adapter->tx_ring[i]);
453 adapter->tx_ring[i] = NULL;
455 for (i = 0; i < adapter->num_rx_queues; i++) {
456 kfree(adapter->rx_ring[i]);
457 adapter->rx_ring[i] = NULL;
459 adapter->num_rx_queues = 0;
460 adapter->num_tx_queues = 0;
464 * igb_alloc_queues - Allocate memory for all rings
465 * @adapter: board private structure to initialize
467 * We allocate one ring per queue at run-time since we don't know the
468 * number of queues at compile-time.
470 static int igb_alloc_queues(struct igb_adapter *adapter)
472 struct igb_ring *ring;
474 #ifdef HAVE_DEVICE_NUMA_NODE
475 int orig_node = adapter->node;
476 #endif /* HAVE_DEVICE_NUMA_NODE */
478 for (i = 0; i < adapter->num_tx_queues; i++) {
479 #ifdef HAVE_DEVICE_NUMA_NODE
480 if (orig_node == -1) {
481 int cur_node = next_online_node(adapter->node);
482 if (cur_node == MAX_NUMNODES)
483 cur_node = first_online_node;
484 adapter->node = cur_node;
486 #endif /* HAVE_DEVICE_NUMA_NODE */
487 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
490 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
493 ring->count = adapter->tx_ring_count;
494 ring->queue_index = i;
495 ring->dev = pci_dev_to_dev(adapter->pdev);
496 ring->netdev = adapter->netdev;
497 ring->numa_node = adapter->node;
498 /* For 82575, context index must be unique per ring. */
499 if (adapter->hw.mac.type == e1000_82575)
500 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
501 adapter->tx_ring[i] = ring;
503 #ifdef HAVE_DEVICE_NUMA_NODE
504 /* Restore the adapter's original node */
505 adapter->node = orig_node;
506 #endif /* HAVE_DEVICE_NUMA_NODE */
508 for (i = 0; i < adapter->num_rx_queues; i++) {
509 #ifdef HAVE_DEVICE_NUMA_NODE
510 if (orig_node == -1) {
511 int cur_node = next_online_node(adapter->node);
512 if (cur_node == MAX_NUMNODES)
513 cur_node = first_online_node;
514 adapter->node = cur_node;
516 #endif /* HAVE_DEVICE_NUMA_NODE */
517 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
520 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
523 ring->count = adapter->rx_ring_count;
524 ring->queue_index = i;
525 ring->dev = pci_dev_to_dev(adapter->pdev);
526 ring->netdev = adapter->netdev;
527 ring->numa_node = adapter->node;
528 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
529 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
531 #ifndef HAVE_NDO_SET_FEATURES
532 /* enable rx checksum */
533 set_bit(IGB_RING_FLAG_RX_CSUM, &ring->flags);
536 /* set flag indicating ring supports SCTP checksum offload */
537 if (adapter->hw.mac.type >= e1000_82576)
538 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
540 /* On i350, loopback VLAN packets have the tag byte-swapped. */
541 if (adapter->hw.mac.type == e1000_i350)
542 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
544 adapter->rx_ring[i] = ring;
546 #ifdef HAVE_DEVICE_NUMA_NODE
547 /* Restore the adapter's original node */
548 adapter->node = orig_node;
549 #endif /* HAVE_DEVICE_NUMA_NODE */
551 igb_cache_ring_register(adapter);
553 return E1000_SUCCESS;
556 #ifdef HAVE_DEVICE_NUMA_NODE
557 /* Restore the adapter's original node */
558 adapter->node = orig_node;
559 #endif /* HAVE_DEVICE_NUMA_NODE */
560 igb_free_queues(adapter);
565 static void igb_configure_lli(struct igb_adapter *adapter)
567 struct e1000_hw *hw = &adapter->hw;
570 /* LLI should only be enabled for MSI-X or MSI interrupts */
571 if (!adapter->msix_entries && !(adapter->flags & IGB_FLAG_HAS_MSI))
574 if (adapter->lli_port) {
575 /* use filter 0 for port */
576 port = htons((u16)adapter->lli_port);
577 E1000_WRITE_REG(hw, E1000_IMIR(0),
578 (port | E1000_IMIR_PORT_IM_EN));
579 E1000_WRITE_REG(hw, E1000_IMIREXT(0),
580 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
583 if (adapter->flags & IGB_FLAG_LLI_PUSH) {
584 /* use filter 1 for push flag */
585 E1000_WRITE_REG(hw, E1000_IMIR(1),
586 (E1000_IMIR_PORT_BP | E1000_IMIR_PORT_IM_EN));
587 E1000_WRITE_REG(hw, E1000_IMIREXT(1),
588 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_PSH));
591 if (adapter->lli_size) {
592 /* use filter 2 for size */
593 E1000_WRITE_REG(hw, E1000_IMIR(2),
594 (E1000_IMIR_PORT_BP | E1000_IMIR_PORT_IM_EN));
595 E1000_WRITE_REG(hw, E1000_IMIREXT(2),
596 (adapter->lli_size | E1000_IMIREXT_CTRL_BP));
602 * igb_write_ivar - configure ivar for given MSI-X vector
603 * @hw: pointer to the HW structure
604 * @msix_vector: vector number we are allocating to a given ring
605 * @index: row index of IVAR register to write within IVAR table
606 * @offset: column offset of in IVAR, should be multiple of 8
608 * This function is intended to handle the writing of the IVAR register
609 * for adapters 82576 and newer. The IVAR table consists of 2 columns,
610 * each containing an cause allocation for an Rx and Tx ring, and a
611 * variable number of rows depending on the number of queues supported.
613 static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
614 int index, int offset)
616 u32 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
618 /* clear any bits that are currently set */
619 ivar &= ~((u32)0xFF << offset);
621 /* write vector and valid bit */
622 ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
624 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
627 #define IGB_N0_QUEUE -1
628 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
630 struct igb_adapter *adapter = q_vector->adapter;
631 struct e1000_hw *hw = &adapter->hw;
632 int rx_queue = IGB_N0_QUEUE;
633 int tx_queue = IGB_N0_QUEUE;
636 if (q_vector->rx.ring)
637 rx_queue = q_vector->rx.ring->reg_idx;
638 if (q_vector->tx.ring)
639 tx_queue = q_vector->tx.ring->reg_idx;
641 switch (hw->mac.type) {
643 /* The 82575 assigns vectors using a bitmask, which matches the
644 bitmask for the EICR/EIMS/EIMC registers. To assign one
645 or more queues to a vector, we write the appropriate bits
646 into the MSIXBM register for that vector. */
647 if (rx_queue > IGB_N0_QUEUE)
648 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
649 if (tx_queue > IGB_N0_QUEUE)
650 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
651 if (!adapter->msix_entries && msix_vector == 0)
652 msixbm |= E1000_EIMS_OTHER;
653 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), msix_vector, msixbm);
654 q_vector->eims_value = msixbm;
658 * 82576 uses a table that essentially consists of 2 columns
659 * with 8 rows. The ordering is column-major so we use the
660 * lower 3 bits as the row index, and the 4th bit as the
663 if (rx_queue > IGB_N0_QUEUE)
664 igb_write_ivar(hw, msix_vector,
666 (rx_queue & 0x8) << 1);
667 if (tx_queue > IGB_N0_QUEUE)
668 igb_write_ivar(hw, msix_vector,
670 ((tx_queue & 0x8) << 1) + 8);
671 q_vector->eims_value = 1 << msix_vector;
676 * On 82580 and newer adapters the scheme is similar to 82576
677 * however instead of ordering column-major we have things
678 * ordered row-major. So we traverse the table by using
679 * bit 0 as the column offset, and the remaining bits as the
682 if (rx_queue > IGB_N0_QUEUE)
683 igb_write_ivar(hw, msix_vector,
685 (rx_queue & 0x1) << 4);
686 if (tx_queue > IGB_N0_QUEUE)
687 igb_write_ivar(hw, msix_vector,
689 ((tx_queue & 0x1) << 4) + 8);
690 q_vector->eims_value = 1 << msix_vector;
697 /* add q_vector eims value to global eims_enable_mask */
698 adapter->eims_enable_mask |= q_vector->eims_value;
700 /* configure q_vector to set itr on first interrupt */
701 q_vector->set_itr = 1;
705 * igb_configure_msix - Configure MSI-X hardware
707 * igb_configure_msix sets up the hardware to properly
708 * generate MSI-X interrupts.
710 static void igb_configure_msix(struct igb_adapter *adapter)
714 struct e1000_hw *hw = &adapter->hw;
716 adapter->eims_enable_mask = 0;
718 /* set vector for other causes, i.e. link changes */
719 switch (hw->mac.type) {
721 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
722 /* enable MSI-X PBA support*/
723 tmp |= E1000_CTRL_EXT_PBA_CLR;
725 /* Auto-Mask interrupts upon ICR read. */
726 tmp |= E1000_CTRL_EXT_EIAME;
727 tmp |= E1000_CTRL_EXT_IRCA;
729 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
731 /* enable msix_other interrupt */
732 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), vector++,
734 adapter->eims_other = E1000_EIMS_OTHER;
741 /* Turn on MSI-X capability first, or our settings
742 * won't stick. And it will take days to debug. */
743 E1000_WRITE_REG(hw, E1000_GPIE, E1000_GPIE_MSIX_MODE |
744 E1000_GPIE_PBA | E1000_GPIE_EIAME |
747 /* enable msix_other interrupt */
748 adapter->eims_other = 1 << vector;
749 tmp = (vector++ | E1000_IVAR_VALID) << 8;
751 E1000_WRITE_REG(hw, E1000_IVAR_MISC, tmp);
754 /* do nothing, since nothing else supports MSI-X */
756 } /* switch (hw->mac.type) */
758 adapter->eims_enable_mask |= adapter->eims_other;
760 for (i = 0; i < adapter->num_q_vectors; i++)
761 igb_assign_vector(adapter->q_vector[i], vector++);
763 E1000_WRITE_FLUSH(hw);
767 * igb_request_msix - Initialize MSI-X interrupts
769 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
772 static int igb_request_msix(struct igb_adapter *adapter)
774 struct net_device *netdev = adapter->netdev;
775 struct e1000_hw *hw = &adapter->hw;
776 int i, err = 0, vector = 0;
778 err = request_irq(adapter->msix_entries[vector].vector,
779 &igb_msix_other, 0, netdev->name, adapter);
784 for (i = 0; i < adapter->num_q_vectors; i++) {
785 struct igb_q_vector *q_vector = adapter->q_vector[i];
787 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
789 if (q_vector->rx.ring && q_vector->tx.ring)
790 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
791 q_vector->rx.ring->queue_index);
792 else if (q_vector->tx.ring)
793 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
794 q_vector->tx.ring->queue_index);
795 else if (q_vector->rx.ring)
796 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
797 q_vector->rx.ring->queue_index);
799 sprintf(q_vector->name, "%s-unused", netdev->name);
801 err = request_irq(adapter->msix_entries[vector].vector,
802 igb_msix_ring, 0, q_vector->name,
809 igb_configure_msix(adapter);
815 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
817 if (adapter->msix_entries) {
818 pci_disable_msix(adapter->pdev);
819 kfree(adapter->msix_entries);
820 adapter->msix_entries = NULL;
821 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
822 pci_disable_msi(adapter->pdev);
827 * igb_free_q_vectors - Free memory allocated for interrupt vectors
828 * @adapter: board private structure to initialize
830 * This function frees the memory allocated to the q_vectors. In addition if
831 * NAPI is enabled it will delete any references to the NAPI struct prior
832 * to freeing the q_vector.
834 static void igb_free_q_vectors(struct igb_adapter *adapter)
838 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
839 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
840 adapter->q_vector[v_idx] = NULL;
843 netif_napi_del(&q_vector->napi);
845 if (q_vector->lrolist) {
846 __skb_queue_purge(&q_vector->lrolist->active);
847 vfree(q_vector->lrolist);
848 q_vector->lrolist = NULL;
853 adapter->num_q_vectors = 0;
857 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
859 * This function resets the device so that it has 0 rx queues, tx queues, and
860 * MSI-X interrupts allocated.
862 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
864 igb_free_queues(adapter);
865 igb_free_q_vectors(adapter);
866 igb_reset_interrupt_capability(adapter);
870 * igb_process_mdd_event
871 * @adapter - board private structure
873 * Identify a malicious VF, disable the VF TX/RX queues and log a message.
875 static void igb_process_mdd_event(struct igb_adapter *adapter)
877 struct e1000_hw *hw = &adapter->hw;
878 u32 lvmmc, vfte, vfre, mdfb;
881 lvmmc = E1000_READ_REG(hw, E1000_LVMMC);
882 vf_queue = lvmmc >> 29;
884 /* VF index cannot be bigger or equal to VFs allocated */
885 if (vf_queue >= adapter->vfs_allocated_count)
888 netdev_info(adapter->netdev,
889 "VF %d misbehaved. VF queues are disabled. "
890 "VM misbehavior code is 0x%x\n", vf_queue, lvmmc);
892 /* Disable VFTE and VFRE related bits */
893 vfte = E1000_READ_REG(hw, E1000_VFTE);
894 vfte &= ~(1 << vf_queue);
895 E1000_WRITE_REG(hw, E1000_VFTE, vfte);
897 vfre = E1000_READ_REG(hw, E1000_VFRE);
898 vfre &= ~(1 << vf_queue);
899 E1000_WRITE_REG(hw, E1000_VFRE, vfre);
901 /* Disable MDFB related bit */
902 mdfb = E1000_READ_REG(hw, E1000_MDFB);
903 mdfb &= ~(1 << vf_queue);
904 E1000_WRITE_REG(hw, E1000_MDFB, mdfb);
906 /* Reset the specific VF */
907 E1000_WRITE_REG(hw, E1000_VTCTRL(vf_queue), E1000_VTCTRL_RST);
912 * @adapter - board private structure
914 * Disable MDD behavior in the HW
916 static void igb_disable_mdd(struct igb_adapter *adapter)
918 struct e1000_hw *hw = &adapter->hw;
921 if (hw->mac.type != e1000_i350)
924 reg = E1000_READ_REG(hw, E1000_DTXCTL);
925 reg &= (~E1000_DTXCTL_MDP_EN);
926 E1000_WRITE_REG(hw, E1000_DTXCTL, reg);
931 * @adapter - board private structure
933 * Enable the HW to detect malicious driver and sends an interrupt to
936 * Only available on i350 device
938 static void igb_enable_mdd(struct igb_adapter *adapter)
940 struct e1000_hw *hw = &adapter->hw;
943 if (hw->mac.type != e1000_i350)
946 reg = E1000_READ_REG(hw, E1000_DTXCTL);
947 reg |= E1000_DTXCTL_MDP_EN;
948 E1000_WRITE_REG(hw, E1000_DTXCTL, reg);
952 * igb_reset_sriov_capability - disable SR-IOV if enabled
954 * Attempt to disable single root IO virtualization capabilites present in the
957 static void igb_reset_sriov_capability(struct igb_adapter *adapter)
959 struct pci_dev *pdev = adapter->pdev;
960 struct e1000_hw *hw = &adapter->hw;
962 /* reclaim resources allocated to VFs */
963 if (adapter->vf_data) {
964 if (!igb_check_vf_assignment(adapter)) {
966 * disable iov and allow time for transactions to
969 pci_disable_sriov(pdev);
972 dev_info(pci_dev_to_dev(pdev), "IOV Disabled\n");
974 dev_info(pci_dev_to_dev(pdev), "IOV Not Disabled\n "
975 "VF(s) are assigned to guests!\n");
977 /* Disable Malicious Driver Detection */
978 igb_disable_mdd(adapter);
980 /* free vf data storage */
981 kfree(adapter->vf_data);
982 adapter->vf_data = NULL;
984 /* switch rings back to PF ownership */
985 E1000_WRITE_REG(hw, E1000_IOVCTL,
986 E1000_IOVCTL_REUSE_VFQ);
987 E1000_WRITE_FLUSH(hw);
991 adapter->vfs_allocated_count = 0;
995 * igb_set_sriov_capability - setup SR-IOV if supported
997 * Attempt to enable single root IO virtualization capabilites present in the
1000 static void igb_set_sriov_capability(struct igb_adapter *adapter)
1002 struct pci_dev *pdev = adapter->pdev;
1006 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
1007 old_vfs = igb_find_enabled_vfs(adapter);
1010 dev_info(pci_dev_to_dev(pdev),
1011 "%d pre-allocated VFs found - override "
1012 "max_vfs setting of %d\n", old_vfs,
1013 adapter->vfs_allocated_count);
1014 adapter->vfs_allocated_count = old_vfs;
1016 /* no VFs requested, do nothing */
1017 if (!adapter->vfs_allocated_count)
1020 /* allocate vf data storage */
1021 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
1022 sizeof(struct vf_data_storage),
1025 if (adapter->vf_data) {
1027 if (pci_enable_sriov(pdev,
1028 adapter->vfs_allocated_count))
1031 for (i = 0; i < adapter->vfs_allocated_count; i++)
1032 igb_vf_configure(adapter, i);
1034 /* DMA Coalescing is not supported in IOV mode. */
1035 if (adapter->hw.mac.type >= e1000_i350)
1036 adapter->dmac = IGB_DMAC_DISABLE;
1037 if (adapter->hw.mac.type < e1000_i350)
1038 adapter->flags |= IGB_FLAG_DETECT_BAD_DMA;
1044 kfree(adapter->vf_data);
1045 adapter->vf_data = NULL;
1046 adapter->vfs_allocated_count = 0;
1047 dev_warn(pci_dev_to_dev(pdev),
1048 "Failed to initialize SR-IOV virtualization\n");
1052 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1054 * Attempt to configure interrupts using the best available
1055 * capabilities of the hardware and kernel.
1057 static void igb_set_interrupt_capability(struct igb_adapter *adapter)
1059 struct pci_dev *pdev = adapter->pdev;
1063 /* Number of supported queues. */
1064 adapter->num_rx_queues = adapter->rss_queues;
1066 if (adapter->vmdq_pools > 1)
1067 adapter->num_rx_queues += adapter->vmdq_pools - 1;
1070 if (adapter->vmdq_pools)
1071 adapter->num_tx_queues = adapter->vmdq_pools;
1073 adapter->num_tx_queues = adapter->num_rx_queues;
1075 adapter->num_tx_queues = max_t(u32, 1, adapter->vmdq_pools);
1078 switch (adapter->int_mode) {
1079 case IGB_INT_MODE_MSIX:
1080 /* start with one vector for every rx queue */
1081 numvecs = adapter->num_rx_queues;
1083 /* if tx handler is seperate add 1 for every tx queue */
1084 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1085 numvecs += adapter->num_tx_queues;
1087 /* store the number of vectors reserved for queues */
1088 adapter->num_q_vectors = numvecs;
1090 /* add 1 vector for link status interrupts */
1092 adapter->msix_entries = kcalloc(numvecs,
1093 sizeof(struct msix_entry),
1095 if (adapter->msix_entries) {
1096 for (i = 0; i < numvecs; i++)
1097 adapter->msix_entries[i].entry = i;
1099 err = pci_enable_msix(pdev,
1100 adapter->msix_entries, numvecs);
1104 /* MSI-X failed, so fall through and try MSI */
1105 dev_warn(pci_dev_to_dev(pdev), "Failed to initialize MSI-X interrupts. "
1106 "Falling back to MSI interrupts.\n");
1107 igb_reset_interrupt_capability(adapter);
1108 case IGB_INT_MODE_MSI:
1109 if (!pci_enable_msi(pdev))
1110 adapter->flags |= IGB_FLAG_HAS_MSI;
1112 dev_warn(pci_dev_to_dev(pdev), "Failed to initialize MSI "
1113 "interrupts. Falling back to legacy "
1116 case IGB_INT_MODE_LEGACY:
1117 /* disable advanced features and set number of queues to 1 */
1118 igb_reset_sriov_capability(adapter);
1119 adapter->vmdq_pools = 0;
1120 adapter->rss_queues = 1;
1121 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1122 adapter->num_rx_queues = 1;
1123 adapter->num_tx_queues = 1;
1124 adapter->num_q_vectors = 1;
1125 /* Don't do anything; this is system default */
1130 /* Notify the stack of the (possibly) reduced Tx Queue count. */
1131 #ifdef CONFIG_NETDEVICES_MULTIQUEUE
1132 adapter->netdev->egress_subqueue_count = adapter->num_tx_queues;
1134 adapter->netdev->real_num_tx_queues =
1135 (adapter->vmdq_pools ? 1 : adapter->num_tx_queues);
1136 #endif /* CONFIG_NETDEVICES_MULTIQUEUE */
1137 #endif /* HAVE_TX_MQ */
1141 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1142 * @adapter: board private structure to initialize
1144 * We allocate one q_vector per queue interrupt. If allocation fails we
1147 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1149 struct igb_q_vector *q_vector;
1150 struct e1000_hw *hw = &adapter->hw;
1152 #ifdef HAVE_DEVICE_NUMA_NODE
1153 int orig_node = adapter->node;
1154 #endif /* HAVE_DEVICE_NUMA_NODE */
1156 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1157 #ifdef HAVE_DEVICE_NUMA_NODE
1158 if ((adapter->num_q_vectors == (adapter->num_rx_queues +
1159 adapter->num_tx_queues)) &&
1160 (adapter->num_rx_queues == v_idx))
1161 adapter->node = orig_node;
1162 if (orig_node == -1) {
1163 int cur_node = next_online_node(adapter->node);
1164 if (cur_node == MAX_NUMNODES)
1165 cur_node = first_online_node;
1166 adapter->node = cur_node;
1168 #endif /* HAVE_DEVICE_NUMA_NODE */
1169 q_vector = kzalloc_node(sizeof(struct igb_q_vector), GFP_KERNEL,
1172 q_vector = kzalloc(sizeof(struct igb_q_vector),
1176 q_vector->adapter = adapter;
1177 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1178 q_vector->itr_val = IGB_START_ITR;
1179 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1180 adapter->q_vector[v_idx] = q_vector;
1182 if (v_idx < adapter->num_rx_queues) {
1183 int size = sizeof(struct igb_lro_list);
1184 q_vector->lrolist = vzalloc_node(size, q_vector->numa_node);
1185 if (!q_vector->lrolist)
1186 q_vector->lrolist = vzalloc(size);
1187 if (!q_vector->lrolist)
1189 __skb_queue_head_init(&q_vector->lrolist->active);
1191 #endif /* IGB_NO_LRO */
1193 #ifdef HAVE_DEVICE_NUMA_NODE
1194 /* Restore the adapter's original node */
1195 adapter->node = orig_node;
1196 #endif /* HAVE_DEVICE_NUMA_NODE */
1201 #ifdef HAVE_DEVICE_NUMA_NODE
1202 /* Restore the adapter's original node */
1203 adapter->node = orig_node;
1204 #endif /* HAVE_DEVICE_NUMA_NODE */
1205 igb_free_q_vectors(adapter);
1209 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1210 int ring_idx, int v_idx)
1212 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1214 q_vector->rx.ring = adapter->rx_ring[ring_idx];
1215 q_vector->rx.ring->q_vector = q_vector;
1216 q_vector->rx.count++;
1217 q_vector->itr_val = adapter->rx_itr_setting;
1218 if (q_vector->itr_val && q_vector->itr_val <= 3)
1219 q_vector->itr_val = IGB_START_ITR;
1222 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1223 int ring_idx, int v_idx)
1225 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1227 q_vector->tx.ring = adapter->tx_ring[ring_idx];
1228 q_vector->tx.ring->q_vector = q_vector;
1229 q_vector->tx.count++;
1230 q_vector->itr_val = adapter->tx_itr_setting;
1231 q_vector->tx.work_limit = adapter->tx_work_limit;
1232 if (q_vector->itr_val && q_vector->itr_val <= 3)
1233 q_vector->itr_val = IGB_START_ITR;
1237 * igb_map_ring_to_vector - maps allocated queues to vectors
1239 * This function maps the recently allocated queues to vectors.
1241 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1246 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1247 (adapter->num_q_vectors < adapter->num_tx_queues))
1250 if (adapter->num_q_vectors >=
1251 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1252 for (i = 0; i < adapter->num_rx_queues; i++)
1253 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1254 for (i = 0; i < adapter->num_tx_queues; i++)
1255 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1257 for (i = 0; i < adapter->num_rx_queues; i++) {
1258 if (i < adapter->num_tx_queues)
1259 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1260 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1262 for (; i < adapter->num_tx_queues; i++)
1263 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1269 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1271 * This function initializes the interrupts and allocates all of the queues.
1273 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1275 struct pci_dev *pdev = adapter->pdev;
1278 igb_set_interrupt_capability(adapter);
1280 err = igb_alloc_q_vectors(adapter);
1282 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for vectors\n");
1283 goto err_alloc_q_vectors;
1286 err = igb_alloc_queues(adapter);
1288 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for queues\n");
1289 goto err_alloc_queues;
1292 err = igb_map_ring_to_vector(adapter);
1294 dev_err(pci_dev_to_dev(pdev), "Invalid q_vector to ring mapping\n");
1295 goto err_map_queues;
1301 igb_free_queues(adapter);
1303 igb_free_q_vectors(adapter);
1304 err_alloc_q_vectors:
1305 igb_reset_interrupt_capability(adapter);
1310 * igb_request_irq - initialize interrupts
1312 * Attempts to configure interrupts using the best available
1313 * capabilities of the hardware and kernel.
1315 static int igb_request_irq(struct igb_adapter *adapter)
1317 struct net_device *netdev = adapter->netdev;
1318 struct pci_dev *pdev = adapter->pdev;
1321 if (adapter->msix_entries) {
1322 err = igb_request_msix(adapter);
1325 /* fall back to MSI */
1326 igb_clear_interrupt_scheme(adapter);
1327 igb_reset_sriov_capability(adapter);
1328 if (!pci_enable_msi(pdev))
1329 adapter->flags |= IGB_FLAG_HAS_MSI;
1330 igb_free_all_tx_resources(adapter);
1331 igb_free_all_rx_resources(adapter);
1332 adapter->num_tx_queues = 1;
1333 adapter->num_rx_queues = 1;
1334 adapter->num_q_vectors = 1;
1335 err = igb_alloc_q_vectors(adapter);
1337 dev_err(pci_dev_to_dev(pdev),
1338 "Unable to allocate memory for vectors\n");
1341 err = igb_alloc_queues(adapter);
1343 dev_err(pci_dev_to_dev(pdev),
1344 "Unable to allocate memory for queues\n");
1345 igb_free_q_vectors(adapter);
1348 igb_setup_all_tx_resources(adapter);
1349 igb_setup_all_rx_resources(adapter);
1352 igb_assign_vector(adapter->q_vector[0], 0);
1354 if (adapter->flags & IGB_FLAG_HAS_MSI) {
1355 err = request_irq(pdev->irq, &igb_intr_msi, 0,
1356 netdev->name, adapter);
1360 /* fall back to legacy interrupts */
1361 igb_reset_interrupt_capability(adapter);
1362 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1365 err = request_irq(pdev->irq, &igb_intr, IRQF_SHARED,
1366 netdev->name, adapter);
1369 dev_err(pci_dev_to_dev(pdev), "Error %d getting interrupt\n",
1376 static void igb_free_irq(struct igb_adapter *adapter)
1378 if (adapter->msix_entries) {
1381 free_irq(adapter->msix_entries[vector++].vector, adapter);
1383 for (i = 0; i < adapter->num_q_vectors; i++)
1384 free_irq(adapter->msix_entries[vector++].vector,
1385 adapter->q_vector[i]);
1387 free_irq(adapter->pdev->irq, adapter);
1392 * igb_irq_disable - Mask off interrupt generation on the NIC
1393 * @adapter: board private structure
1395 static void igb_irq_disable(struct igb_adapter *adapter)
1397 struct e1000_hw *hw = &adapter->hw;
1400 * we need to be careful when disabling interrupts. The VFs are also
1401 * mapped into these registers and so clearing the bits can cause
1402 * issues on the VF drivers so we only need to clear what we set
1404 if (adapter->msix_entries) {
1405 u32 regval = E1000_READ_REG(hw, E1000_EIAM);
1406 E1000_WRITE_REG(hw, E1000_EIAM, regval & ~adapter->eims_enable_mask);
1407 E1000_WRITE_REG(hw, E1000_EIMC, adapter->eims_enable_mask);
1408 regval = E1000_READ_REG(hw, E1000_EIAC);
1409 E1000_WRITE_REG(hw, E1000_EIAC, regval & ~adapter->eims_enable_mask);
1412 E1000_WRITE_REG(hw, E1000_IAM, 0);
1413 E1000_WRITE_REG(hw, E1000_IMC, ~0);
1414 E1000_WRITE_FLUSH(hw);
1416 if (adapter->msix_entries) {
1419 synchronize_irq(adapter->msix_entries[vector++].vector);
1421 for (i = 0; i < adapter->num_q_vectors; i++)
1422 synchronize_irq(adapter->msix_entries[vector++].vector);
1424 synchronize_irq(adapter->pdev->irq);
1429 * igb_irq_enable - Enable default interrupt generation settings
1430 * @adapter: board private structure
1432 static void igb_irq_enable(struct igb_adapter *adapter)
1434 struct e1000_hw *hw = &adapter->hw;
1436 if (adapter->msix_entries) {
1437 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1438 u32 regval = E1000_READ_REG(hw, E1000_EIAC);
1439 E1000_WRITE_REG(hw, E1000_EIAC, regval | adapter->eims_enable_mask);
1440 regval = E1000_READ_REG(hw, E1000_EIAM);
1441 E1000_WRITE_REG(hw, E1000_EIAM, regval | adapter->eims_enable_mask);
1442 E1000_WRITE_REG(hw, E1000_EIMS, adapter->eims_enable_mask);
1443 if (adapter->vfs_allocated_count) {
1444 E1000_WRITE_REG(hw, E1000_MBVFIMR, 0xFF);
1445 ims |= E1000_IMS_VMMB;
1446 /* For I350 device only enable MDD interrupts*/
1447 if ((adapter->mdd) &&
1448 (adapter->hw.mac.type == e1000_i350))
1449 ims |= E1000_IMS_MDDET;
1451 E1000_WRITE_REG(hw, E1000_IMS, ims);
1453 E1000_WRITE_REG(hw, E1000_IMS, IMS_ENABLE_MASK |
1455 E1000_WRITE_REG(hw, E1000_IAM, IMS_ENABLE_MASK |
1460 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1462 struct e1000_hw *hw = &adapter->hw;
1463 u16 vid = adapter->hw.mng_cookie.vlan_id;
1464 u16 old_vid = adapter->mng_vlan_id;
1466 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1467 /* add VID to filter table */
1468 igb_vfta_set(adapter, vid, TRUE);
1469 adapter->mng_vlan_id = vid;
1471 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1474 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1476 #ifdef HAVE_VLAN_RX_REGISTER
1477 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1479 !test_bit(old_vid, adapter->active_vlans)) {
1481 /* remove VID from filter table */
1482 igb_vfta_set(adapter, old_vid, FALSE);
1487 * igb_release_hw_control - release control of the h/w to f/w
1488 * @adapter: address of board private structure
1490 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1491 * For ASF and Pass Through versions of f/w this means that the
1492 * driver is no longer loaded.
1495 static void igb_release_hw_control(struct igb_adapter *adapter)
1497 struct e1000_hw *hw = &adapter->hw;
1500 /* Let firmware take over control of h/w */
1501 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1502 E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1503 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1507 * igb_get_hw_control - get control of the h/w from f/w
1508 * @adapter: address of board private structure
1510 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1511 * For ASF and Pass Through versions of f/w this means that
1512 * the driver is loaded.
1515 static void igb_get_hw_control(struct igb_adapter *adapter)
1517 struct e1000_hw *hw = &adapter->hw;
1520 /* Let firmware know the driver has taken over */
1521 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
1522 E1000_WRITE_REG(hw, E1000_CTRL_EXT,
1523 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1527 * igb_configure - configure the hardware for RX and TX
1528 * @adapter: private board structure
1530 static void igb_configure(struct igb_adapter *adapter)
1532 struct net_device *netdev = adapter->netdev;
1535 igb_get_hw_control(adapter);
1536 igb_set_rx_mode(netdev);
1538 igb_restore_vlan(adapter);
1540 igb_setup_tctl(adapter);
1541 igb_setup_mrqc(adapter);
1542 igb_setup_rctl(adapter);
1544 igb_configure_tx(adapter);
1545 igb_configure_rx(adapter);
1547 e1000_rx_fifo_flush_82575(&adapter->hw);
1548 #ifdef CONFIG_NETDEVICES_MULTIQUEUE
1550 if (adapter->num_tx_queues > 1)
1551 netdev->features |= NETIF_F_MULTI_QUEUE;
1553 netdev->features &= ~NETIF_F_MULTI_QUEUE;
1556 /* call igb_desc_unused which always leaves
1557 * at least 1 descriptor unused to make sure
1558 * next_to_use != next_to_clean */
1559 for (i = 0; i < adapter->num_rx_queues; i++) {
1560 struct igb_ring *ring = adapter->rx_ring[i];
1561 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
1566 * igb_power_up_link - Power up the phy/serdes link
1567 * @adapter: address of board private structure
1569 void igb_power_up_link(struct igb_adapter *adapter)
1571 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1572 e1000_power_up_phy(&adapter->hw);
1574 e1000_power_up_fiber_serdes_link(&adapter->hw);
1576 e1000_phy_hw_reset(&adapter->hw);
1580 * igb_power_down_link - Power down the phy/serdes link
1581 * @adapter: address of board private structure
1583 static void igb_power_down_link(struct igb_adapter *adapter)
1585 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1586 e1000_power_down_phy(&adapter->hw);
1588 e1000_shutdown_fiber_serdes_link(&adapter->hw);
1592 * igb_up - Open the interface and prepare it to handle traffic
1593 * @adapter: board private structure
1595 int igb_up(struct igb_adapter *adapter)
1597 struct e1000_hw *hw = &adapter->hw;
1600 /* hardware has been reset, we need to reload some things */
1601 igb_configure(adapter);
1603 clear_bit(__IGB_DOWN, &adapter->state);
1605 for (i = 0; i < adapter->num_q_vectors; i++)
1606 napi_enable(&(adapter->q_vector[i]->napi));
1608 if (adapter->msix_entries)
1609 igb_configure_msix(adapter);
1611 igb_assign_vector(adapter->q_vector[0], 0);
1613 igb_configure_lli(adapter);
1615 /* Clear any pending interrupts. */
1616 E1000_READ_REG(hw, E1000_ICR);
1617 igb_irq_enable(adapter);
1619 /* notify VFs that reset has been completed */
1620 if (adapter->vfs_allocated_count) {
1621 u32 reg_data = E1000_READ_REG(hw, E1000_CTRL_EXT);
1622 reg_data |= E1000_CTRL_EXT_PFRSTD;
1623 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg_data);
1626 netif_tx_start_all_queues(adapter->netdev);
1628 if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
1629 schedule_work(&adapter->dma_err_task);
1630 /* start the watchdog. */
1631 hw->mac.get_link_status = 1;
1632 schedule_work(&adapter->watchdog_task);
1637 void igb_down(struct igb_adapter *adapter)
1639 struct net_device *netdev = adapter->netdev;
1640 struct e1000_hw *hw = &adapter->hw;
1644 /* signal that we're down so the interrupt handler does not
1645 * reschedule our watchdog timer */
1646 set_bit(__IGB_DOWN, &adapter->state);
1648 /* disable receives in the hardware */
1649 rctl = E1000_READ_REG(hw, E1000_RCTL);
1650 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
1651 /* flush and sleep below */
1653 netif_tx_stop_all_queues(netdev);
1655 /* disable transmits in the hardware */
1656 tctl = E1000_READ_REG(hw, E1000_TCTL);
1657 tctl &= ~E1000_TCTL_EN;
1658 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
1659 /* flush both disables and wait for them to finish */
1660 E1000_WRITE_FLUSH(hw);
1661 usleep_range(10000, 20000);
1663 for (i = 0; i < adapter->num_q_vectors; i++)
1664 napi_disable(&(adapter->q_vector[i]->napi));
1666 igb_irq_disable(adapter);
1668 del_timer_sync(&adapter->watchdog_timer);
1669 if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
1670 del_timer_sync(&adapter->dma_err_timer);
1671 del_timer_sync(&adapter->phy_info_timer);
1673 netif_carrier_off(netdev);
1675 /* record the stats before reset*/
1676 igb_update_stats(adapter);
1678 adapter->link_speed = 0;
1679 adapter->link_duplex = 0;
1682 if (!pci_channel_offline(adapter->pdev))
1687 igb_clean_all_tx_rings(adapter);
1688 igb_clean_all_rx_rings(adapter);
1691 /* since we reset the hardware DCA settings were cleared */
1692 igb_setup_dca(adapter);
1696 void igb_reinit_locked(struct igb_adapter *adapter)
1698 WARN_ON(in_interrupt());
1699 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1700 usleep_range(1000, 2000);
1703 clear_bit(__IGB_RESETTING, &adapter->state);
1706 void igb_reset(struct igb_adapter *adapter)
1708 struct pci_dev *pdev = adapter->pdev;
1709 struct e1000_hw *hw = &adapter->hw;
1710 struct e1000_mac_info *mac = &hw->mac;
1711 struct e1000_fc_info *fc = &hw->fc;
1712 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1715 /* Repartition Pba for greater than 9k mtu
1716 * To take effect CTRL.RST is required.
1718 switch (mac->type) {
1721 pba = E1000_READ_REG(hw, E1000_RXPBS);
1722 pba = e1000_rxpbs_adjust_82580(pba);
1725 pba = E1000_READ_REG(hw, E1000_RXPBS);
1726 pba &= E1000_RXPBS_SIZE_MASK_82576;
1730 pba = E1000_PBA_34K;
1734 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1735 (mac->type < e1000_82576)) {
1736 /* adjust PBA for jumbo frames */
1737 E1000_WRITE_REG(hw, E1000_PBA, pba);
1739 /* To maintain wire speed transmits, the Tx FIFO should be
1740 * large enough to accommodate two full transmit packets,
1741 * rounded up to the next 1KB and expressed in KB. Likewise,
1742 * the Rx FIFO should be large enough to accommodate at least
1743 * one full receive packet and is similarly rounded up and
1744 * expressed in KB. */
1745 pba = E1000_READ_REG(hw, E1000_PBA);
1746 /* upper 16 bits has Tx packet buffer allocation size in KB */
1747 tx_space = pba >> 16;
1748 /* lower 16 bits has Rx packet buffer allocation size in KB */
1750 /* the tx fifo also stores 16 bytes of information about the tx
1751 * but don't include ethernet FCS because hardware appends it */
1752 min_tx_space = (adapter->max_frame_size +
1753 sizeof(union e1000_adv_tx_desc) -
1755 min_tx_space = ALIGN(min_tx_space, 1024);
1756 min_tx_space >>= 10;
1757 /* software strips receive CRC, so leave room for it */
1758 min_rx_space = adapter->max_frame_size;
1759 min_rx_space = ALIGN(min_rx_space, 1024);
1760 min_rx_space >>= 10;
1762 /* If current Tx allocation is less than the min Tx FIFO size,
1763 * and the min Tx FIFO size is less than the current Rx FIFO
1764 * allocation, take space away from current Rx allocation */
1765 if (tx_space < min_tx_space &&
1766 ((min_tx_space - tx_space) < pba)) {
1767 pba = pba - (min_tx_space - tx_space);
1769 /* if short on rx space, rx wins and must trump tx
1771 if (pba < min_rx_space)
1774 E1000_WRITE_REG(hw, E1000_PBA, pba);
1777 /* flow control settings */
1778 /* The high water mark must be low enough to fit one full frame
1779 * (or the size used for early receive) above it in the Rx FIFO.
1780 * Set it to the lower of:
1781 * - 90% of the Rx FIFO size, or
1782 * - the full Rx FIFO size minus one full frame */
1783 hwm = min(((pba << 10) * 9 / 10),
1784 ((pba << 10) - 2 * adapter->max_frame_size));
1786 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1787 fc->low_water = fc->high_water - 16;
1788 fc->pause_time = 0xFFFF;
1790 fc->current_mode = fc->requested_mode;
1792 /* disable receive for all VFs and wait one second */
1793 if (adapter->vfs_allocated_count) {
1796 * Clear all flags except indication that the PF has set
1797 * the VF MAC addresses administratively
1799 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1800 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
1802 /* ping all the active vfs to let them know we are going down */
1803 igb_ping_all_vfs(adapter);
1805 /* disable transmits and receives */
1806 E1000_WRITE_REG(hw, E1000_VFRE, 0);
1807 E1000_WRITE_REG(hw, E1000_VFTE, 0);
1810 /* Allow time for pending master requests to run */
1812 E1000_WRITE_REG(hw, E1000_WUC, 0);
1814 if (e1000_init_hw(hw))
1815 dev_err(pci_dev_to_dev(pdev), "Hardware Error\n");
1817 igb_init_dmac(adapter, pba);
1818 /* Re-initialize the thermal sensor on i350 devices. */
1819 if (mac->type == e1000_i350 && hw->bus.func == 0) {
1821 * If present, re-initialize the external thermal sensor
1825 e1000_set_i2c_bb(hw);
1826 e1000_init_thermal_sensor_thresh(hw);
1828 if (!netif_running(adapter->netdev))
1829 igb_power_down_link(adapter);
1831 igb_update_mng_vlan(adapter);
1833 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1834 E1000_WRITE_REG(hw, E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1836 e1000_get_phy_info(hw);
1839 #ifdef HAVE_NDO_SET_FEATURES
1840 static netdev_features_t igb_fix_features(struct net_device *netdev,
1841 netdev_features_t features)
1844 * Since there is no support for separate tx vlan accel
1845 * enabled make sure tx flag is cleared if rx is.
1847 if (!(features & NETIF_F_HW_VLAN_RX))
1848 features &= ~NETIF_F_HW_VLAN_TX;
1850 /* If Rx checksum is disabled, then LRO should also be disabled */
1851 if (!(features & NETIF_F_RXCSUM))
1852 features &= ~NETIF_F_LRO;
1857 static int igb_set_features(struct net_device *netdev,
1858 netdev_features_t features)
1860 u32 changed = netdev->features ^ features;
1862 if (changed & NETIF_F_HW_VLAN_RX)
1863 igb_vlan_mode(netdev, features);
1868 #endif /* HAVE_NDO_SET_FEATURES */
1869 #ifdef HAVE_NET_DEVICE_OPS
1870 static const struct net_device_ops igb_netdev_ops = {
1871 .ndo_open = igb_open,
1872 .ndo_stop = igb_close,
1873 .ndo_start_xmit = igb_xmit_frame,
1874 .ndo_get_stats = igb_get_stats,
1875 .ndo_set_rx_mode = igb_set_rx_mode,
1876 .ndo_set_mac_address = igb_set_mac,
1877 .ndo_change_mtu = igb_change_mtu,
1878 .ndo_do_ioctl = igb_ioctl,
1879 .ndo_tx_timeout = igb_tx_timeout,
1880 .ndo_validate_addr = eth_validate_addr,
1881 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1882 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1884 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1885 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1886 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1887 .ndo_get_vf_config = igb_ndo_get_vf_config,
1889 #ifdef CONFIG_NET_POLL_CONTROLLER
1890 .ndo_poll_controller = igb_netpoll,
1892 #ifdef HAVE_NDO_SET_FEATURES
1893 .ndo_fix_features = igb_fix_features,
1894 .ndo_set_features = igb_set_features,
1896 #ifdef HAVE_VLAN_RX_REGISTER
1897 .ndo_vlan_rx_register = igb_vlan_mode,
1901 #ifdef CONFIG_IGB_VMDQ_NETDEV
1902 static const struct net_device_ops igb_vmdq_ops = {
1903 .ndo_open = &igb_vmdq_open,
1904 .ndo_stop = &igb_vmdq_close,
1905 .ndo_start_xmit = &igb_vmdq_xmit_frame,
1906 .ndo_get_stats = &igb_vmdq_get_stats,
1907 .ndo_set_rx_mode = &igb_vmdq_set_rx_mode,
1908 .ndo_validate_addr = eth_validate_addr,
1909 .ndo_set_mac_address = &igb_vmdq_set_mac,
1910 .ndo_change_mtu = &igb_vmdq_change_mtu,
1911 .ndo_tx_timeout = &igb_vmdq_tx_timeout,
1912 .ndo_vlan_rx_register = &igb_vmdq_vlan_rx_register,
1913 .ndo_vlan_rx_add_vid = &igb_vmdq_vlan_rx_add_vid,
1914 .ndo_vlan_rx_kill_vid = &igb_vmdq_vlan_rx_kill_vid,
1917 #endif /* CONFIG_IGB_VMDQ_NETDEV */
1918 #endif /* HAVE_NET_DEVICE_OPS */
1919 #ifdef CONFIG_IGB_VMDQ_NETDEV
1920 void igb_assign_vmdq_netdev_ops(struct net_device *vnetdev)
1922 #ifdef HAVE_NET_DEVICE_OPS
1923 vnetdev->netdev_ops = &igb_vmdq_ops;
1925 dev->open = &igb_vmdq_open;
1926 dev->stop = &igb_vmdq_close;
1927 dev->hard_start_xmit = &igb_vmdq_xmit_frame;
1928 dev->get_stats = &igb_vmdq_get_stats;
1929 #ifdef HAVE_SET_RX_MODE
1930 dev->set_rx_mode = &igb_vmdq_set_rx_mode;
1932 dev->set_multicast_list = &igb_vmdq_set_rx_mode;
1933 dev->set_mac_address = &igb_vmdq_set_mac;
1934 dev->change_mtu = &igb_vmdq_change_mtu;
1935 #ifdef HAVE_TX_TIMEOUT
1936 dev->tx_timeout = &igb_vmdq_tx_timeout;
1938 #ifdef NETIF_F_HW_VLAN_TX
1939 dev->vlan_rx_register = &igb_vmdq_vlan_rx_register;
1940 dev->vlan_rx_add_vid = &igb_vmdq_vlan_rx_add_vid;
1941 dev->vlan_rx_kill_vid = &igb_vmdq_vlan_rx_kill_vid;
1944 igb_vmdq_set_ethtool_ops(vnetdev);
1945 vnetdev->watchdog_timeo = 5 * HZ;
1949 int igb_init_vmdq_netdevs(struct igb_adapter *adapter)
1951 int pool, err = 0, base_queue;
1952 struct net_device *vnetdev;
1953 struct igb_vmdq_adapter *vmdq_adapter;
1955 for (pool = 1; pool < adapter->vmdq_pools; pool++) {
1956 int qpp = (!adapter->rss_queues ? 1 : adapter->rss_queues);
1957 base_queue = pool * qpp;
1958 vnetdev = alloc_etherdev(sizeof(struct igb_vmdq_adapter));
1963 vmdq_adapter = netdev_priv(vnetdev);
1964 vmdq_adapter->vnetdev = vnetdev;
1965 vmdq_adapter->real_adapter = adapter;
1966 vmdq_adapter->rx_ring = adapter->rx_ring[base_queue];
1967 vmdq_adapter->tx_ring = adapter->tx_ring[base_queue];
1968 igb_assign_vmdq_netdev_ops(vnetdev);
1969 snprintf(vnetdev->name, IFNAMSIZ, "%sv%d",
1970 adapter->netdev->name, pool);
1971 vnetdev->features = adapter->netdev->features;
1972 #ifdef HAVE_NETDEV_VLAN_FEATURES
1973 vnetdev->vlan_features = adapter->netdev->vlan_features;
1975 adapter->vmdq_netdev[pool-1] = vnetdev;
1976 err = register_netdev(vnetdev);
1983 int igb_remove_vmdq_netdevs(struct igb_adapter *adapter)
1987 for (pool = 1; pool < adapter->vmdq_pools; pool++) {
1988 unregister_netdev(adapter->vmdq_netdev[pool-1]);
1989 free_netdev(adapter->vmdq_netdev[pool-1]);
1990 adapter->vmdq_netdev[pool-1] = NULL;
1994 #endif /* CONFIG_IGB_VMDQ_NETDEV */
1997 * igb_probe - Device Initialization Routine
1998 * @pdev: PCI device information struct
1999 * @ent: entry in igb_pci_tbl
2001 * Returns 0 on success, negative on failure
2003 * igb_probe initializes an adapter identified by a pci_dev structure.
2004 * The OS initialization, configuring of the adapter private structure,
2005 * and a hardware reset occur.
2007 static int __devinit igb_probe(struct pci_dev *pdev,
2008 const struct pci_device_id *ent)
2010 struct net_device *netdev;
2011 struct igb_adapter *adapter;
2012 struct e1000_hw *hw;
2013 u16 eeprom_data = 0;
2014 u8 pba_str[E1000_PBANUM_LENGTH];
2016 static int global_quad_port_a; /* global quad port a indication */
2017 int i, err, pci_using_dac;
2018 static int cards_found;
2020 err = pci_enable_device_mem(pdev);
2025 err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
2027 err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
2031 err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
2033 err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
2035 IGB_ERR("No usable DMA configuration, "
2042 #ifndef HAVE_ASPM_QUIRKS
2043 /* 82575 requires that the pci-e link partner disable the L0s state */
2044 switch (pdev->device) {
2045 case E1000_DEV_ID_82575EB_COPPER:
2046 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2047 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2048 pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S);
2053 #endif /* HAVE_ASPM_QUIRKS */
2054 err = pci_request_selected_regions(pdev,
2055 pci_select_bars(pdev,
2061 pci_enable_pcie_error_reporting(pdev);
2063 pci_set_master(pdev);
2067 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
2070 netdev = alloc_etherdev(sizeof(struct igb_adapter));
2071 #endif /* HAVE_TX_MQ */
2073 goto err_alloc_etherdev;
2075 SET_MODULE_OWNER(netdev);
2076 SET_NETDEV_DEV(netdev, &pdev->dev);
2078 pci_set_drvdata(pdev, netdev);
2079 adapter = netdev_priv(netdev);
2080 adapter->netdev = netdev;
2081 adapter->pdev = pdev;
2084 adapter->port_num = hw->bus.func;
2085 adapter->msg_enable = (1 << debug) - 1;
2088 err = pci_save_state(pdev);
2093 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
2094 pci_resource_len(pdev, 0));
2098 #ifdef HAVE_NET_DEVICE_OPS
2099 netdev->netdev_ops = &igb_netdev_ops;
2100 #else /* HAVE_NET_DEVICE_OPS */
2101 netdev->open = &igb_open;
2102 netdev->stop = &igb_close;
2103 netdev->get_stats = &igb_get_stats;
2104 #ifdef HAVE_SET_RX_MODE
2105 netdev->set_rx_mode = &igb_set_rx_mode;
2107 netdev->set_multicast_list = &igb_set_rx_mode;
2108 netdev->set_mac_address = &igb_set_mac;
2109 netdev->change_mtu = &igb_change_mtu;
2110 netdev->do_ioctl = &igb_ioctl;
2111 #ifdef HAVE_TX_TIMEOUT
2112 netdev->tx_timeout = &igb_tx_timeout;
2114 netdev->vlan_rx_register = igb_vlan_mode;
2115 netdev->vlan_rx_add_vid = igb_vlan_rx_add_vid;
2116 netdev->vlan_rx_kill_vid = igb_vlan_rx_kill_vid;
2117 #ifdef CONFIG_NET_POLL_CONTROLLER
2118 netdev->poll_controller = igb_netpoll;
2120 netdev->hard_start_xmit = &igb_xmit_frame;
2121 #endif /* HAVE_NET_DEVICE_OPS */
2122 igb_set_ethtool_ops(netdev);
2123 #ifdef HAVE_TX_TIMEOUT
2124 netdev->watchdog_timeo = 5 * HZ;
2127 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
2129 adapter->bd_number = cards_found;
2131 /* setup the private structure */
2132 err = igb_sw_init(adapter);
2136 e1000_get_bus_info(hw);
2138 hw->phy.autoneg_wait_to_complete = FALSE;
2139 hw->mac.adaptive_ifs = FALSE;
2141 /* Copper options */
2142 if (hw->phy.media_type == e1000_media_type_copper) {
2144 hw->phy.mdix = ETH_TP_MDI_INVALID;
2146 hw->phy.mdix = AUTO_ALL_MODES;
2147 #endif /* ETH_TP_MDI_X */
2148 hw->phy.disable_polarity_correction = FALSE;
2149 hw->phy.ms_type = e1000_ms_hw_default;
2152 if (e1000_check_reset_block(hw))
2153 dev_info(pci_dev_to_dev(pdev),
2154 "PHY reset is blocked due to SOL/IDER session.\n");
2157 * features is initialized to 0 in allocation, it might have bits
2158 * set by igb_sw_init so we should use an or instead of an
2161 netdev->features |= NETIF_F_SG |
2163 #ifdef NETIF_F_IPV6_CSUM
2171 #endif /* NETIF_F_TSO */
2172 #ifdef NETIF_F_RXHASH
2175 #ifdef HAVE_NDO_SET_FEATURES
2178 NETIF_F_HW_VLAN_RX |
2181 #ifdef HAVE_NDO_SET_FEATURES
2182 /* copy netdev features into list of user selectable features */
2183 netdev->hw_features |= netdev->features;
2186 /* give us the option of enabling LRO later */
2187 netdev->hw_features |= NETIF_F_LRO;
2192 /* this is only needed on kernels prior to 2.6.39 */
2193 netdev->features |= NETIF_F_GRO;
2197 /* set this bit last since it cannot be part of hw_features */
2198 netdev->features |= NETIF_F_HW_VLAN_FILTER;
2200 #ifdef HAVE_NETDEV_VLAN_FEATURES
2201 netdev->vlan_features |= NETIF_F_TSO |
2209 netdev->features |= NETIF_F_HIGHDMA;
2211 if (hw->mac.type >= e1000_82576)
2212 netdev->features |= NETIF_F_SCTP_CSUM;
2214 adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
2216 /* before reading the NVM, reset the controller to put the device in a
2217 * known good starting state */
2220 /* make sure the NVM is good */
2221 if (e1000_validate_nvm_checksum(hw) < 0) {
2222 dev_err(pci_dev_to_dev(pdev), "The NVM Checksum Is Not"
2228 /* copy the MAC address out of the NVM */
2229 if (e1000_read_mac_addr(hw))
2230 dev_err(pci_dev_to_dev(pdev), "NVM Read Error\n");
2231 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2232 #ifdef ETHTOOL_GPERMADDR
2233 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
2235 if (!is_valid_ether_addr(netdev->perm_addr)) {
2237 if (!is_valid_ether_addr(netdev->dev_addr)) {
2239 dev_err(pci_dev_to_dev(pdev), "Invalid MAC Address\n");
2244 memcpy(&adapter->mac_table[0].addr, hw->mac.addr, netdev->addr_len);
2245 adapter->mac_table[0].queue = adapter->vfs_allocated_count;
2246 adapter->mac_table[0].state = (IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE);
2247 igb_rar_set(adapter, 0);
2249 /* get firmware version for ethtool -i */
2250 e1000_read_nvm(&adapter->hw, 5, 1, &adapter->fw_version);
2251 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
2252 (unsigned long) adapter);
2253 if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2254 setup_timer(&adapter->dma_err_timer, &igb_dma_err_timer,
2255 (unsigned long) adapter);
2256 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
2257 (unsigned long) adapter);
2259 INIT_WORK(&adapter->reset_task, igb_reset_task);
2260 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2261 if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2262 INIT_WORK(&adapter->dma_err_task, igb_dma_err_task);
2264 /* Initialize link properties that are user-changeable */
2265 adapter->fc_autoneg = true;
2266 hw->mac.autoneg = true;
2267 hw->phy.autoneg_advertised = 0x2f;
2269 hw->fc.requested_mode = e1000_fc_default;
2270 hw->fc.current_mode = e1000_fc_default;
2272 e1000_validate_mdi_setting(hw);
2274 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2275 * enable the ACPI Magic Packet filter
2278 if (hw->bus.func == 0)
2279 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
2280 else if (hw->mac.type >= e1000_82580)
2281 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2282 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2284 else if (hw->bus.func == 1)
2285 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
2287 if (eeprom_data & IGB_EEPROM_APME)
2288 adapter->eeprom_wol |= E1000_WUFC_MAG;
2290 /* now that we have the eeprom settings, apply the special cases where
2291 * the eeprom may be wrong or the board simply won't support wake on
2292 * lan on a particular port */
2293 switch (pdev->device) {
2294 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2295 adapter->eeprom_wol = 0;
2297 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2298 case E1000_DEV_ID_82576_FIBER:
2299 case E1000_DEV_ID_82576_SERDES:
2300 /* Wake events only supported on port A for dual fiber
2301 * regardless of eeprom setting */
2302 if (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_FUNC_1)
2303 adapter->eeprom_wol = 0;
2305 case E1000_DEV_ID_82576_QUAD_COPPER:
2306 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
2307 /* if quad port adapter, disable WoL on all but port A */
2308 if (global_quad_port_a != 0)
2309 adapter->eeprom_wol = 0;
2311 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2312 /* Reset for multiple quad port adapters */
2313 if (++global_quad_port_a == 4)
2314 global_quad_port_a = 0;
2318 /* initialize the wol settings based on the eeprom settings */
2319 adapter->wol = adapter->eeprom_wol;
2320 device_set_wakeup_enable(pci_dev_to_dev(adapter->pdev), adapter->wol);
2322 /* reset the hardware with the new settings */
2325 /* let the f/w know that the h/w is now under the control of the
2327 igb_get_hw_control(adapter);
2329 strncpy(netdev->name, "eth%d", IFNAMSIZ);
2330 err = register_netdev(netdev);
2334 #ifdef CONFIG_IGB_VMDQ_NETDEV
2335 err = igb_init_vmdq_netdevs(adapter);
2339 /* carrier off reporting is important to ethtool even BEFORE open */
2340 netif_carrier_off(netdev);
2343 if (dca_add_requester(&pdev->dev) == E1000_SUCCESS) {
2344 adapter->flags |= IGB_FLAG_DCA_ENABLED;
2345 dev_info(pci_dev_to_dev(pdev), "DCA enabled\n");
2346 igb_setup_dca(adapter);
2350 #ifdef HAVE_HW_TIME_STAMP
2351 /* do hw tstamp init after resetting */
2352 igb_init_hw_timer(adapter);
2355 dev_info(pci_dev_to_dev(pdev), "Intel(R) Gigabit Ethernet Network Connection\n");
2356 /* print bus type/speed/width info */
2357 dev_info(pci_dev_to_dev(pdev), "%s: (PCIe:%s:%s) ",
2359 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5GT/s" :
2360 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0GT/s" :
2362 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4\n" :
2363 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2\n" :
2364 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1\n" :
2366 dev_info(pci_dev_to_dev(pdev), "%s: MAC: ", netdev->name);
2367 for (i = 0; i < 6; i++)
2368 printk("%2.2x%c", netdev->dev_addr[i], i == 5 ? '\n' : ':');
2370 ret_val = e1000_read_pba_string(hw, pba_str, E1000_PBANUM_LENGTH);
2372 strncpy(pba_str, "Unknown", sizeof(pba_str) - 1);
2373 dev_info(pci_dev_to_dev(pdev), "%s: PBA No: %s\n", netdev->name,
2377 /* Initialize the thermal sensor on i350 devices. */
2378 if (hw->mac.type == e1000_i350 && hw->bus.func == 0) {
2382 * Read the NVM to determine if this i350 device supports an
2383 * external thermal sensor.
2385 e1000_read_nvm(hw, NVM_ETS_CFG, 1, &ets_word);
2386 if (ets_word != 0x0000 && ets_word != 0xFFFF)
2387 adapter->ets = true;
2389 adapter->ets = false;
2391 igb_sysfs_init(adapter);
2394 igb_procfs_init(adapter);
2395 #endif /* IGB_PROCFS */
2396 #endif /* IGB_SYSFS */
2398 adapter->ets = false;
2401 switch (hw->mac.type) {
2403 /* Enable EEE for internal copper PHY devices */
2404 if (hw->phy.media_type == e1000_media_type_copper)
2405 e1000_set_eee_i350(hw);
2407 /* send driver version info to firmware */
2408 igb_init_fw(adapter);
2414 if (netdev->features & NETIF_F_LRO)
2415 dev_info(pci_dev_to_dev(pdev), "Internal LRO is enabled \n");
2417 dev_info(pci_dev_to_dev(pdev), "LRO is disabled \n");
2419 dev_info(pci_dev_to_dev(pdev),
2420 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2421 adapter->msix_entries ? "MSI-X" :
2422 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
2423 adapter->num_rx_queues, adapter->num_tx_queues);
2427 pm_runtime_put_noidle(&pdev->dev);
2431 igb_release_hw_control(adapter);
2433 if (!e1000_check_reset_block(hw))
2434 e1000_phy_hw_reset(hw);
2436 if (hw->flash_address)
2437 iounmap(hw->flash_address);
2439 igb_clear_interrupt_scheme(adapter);
2440 igb_reset_sriov_capability(adapter);
2441 iounmap(hw->hw_addr);
2443 free_netdev(netdev);
2445 pci_release_selected_regions(pdev,
2446 pci_select_bars(pdev, IORESOURCE_MEM));
2449 pci_disable_device(pdev);
2454 * igb_remove - Device Removal Routine
2455 * @pdev: PCI device information struct
2457 * igb_remove is called by the PCI subsystem to alert the driver
2458 * that it should release a PCI device. The could be caused by a
2459 * Hot-Plug event, or because the driver is going to be removed from
2462 static void __devexit igb_remove(struct pci_dev *pdev)
2464 struct net_device *netdev = pci_get_drvdata(pdev);
2465 struct igb_adapter *adapter = netdev_priv(netdev);
2466 struct e1000_hw *hw = &adapter->hw;
2468 pm_runtime_get_noresume(&pdev->dev);
2470 /* flush_scheduled work may reschedule our watchdog task, so
2471 * explicitly disable watchdog tasks from being rescheduled */
2472 set_bit(__IGB_DOWN, &adapter->state);
2473 del_timer_sync(&adapter->watchdog_timer);
2474 if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2475 del_timer_sync(&adapter->dma_err_timer);
2476 del_timer_sync(&adapter->phy_info_timer);
2478 flush_scheduled_work();
2481 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2482 dev_info(pci_dev_to_dev(pdev), "DCA disabled\n");
2483 dca_remove_requester(&pdev->dev);
2484 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2485 E1000_WRITE_REG(hw, E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_DISABLE);
2489 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2490 * would have already happened in close and is redundant. */
2491 igb_release_hw_control(adapter);
2493 unregister_netdev(netdev);
2494 #ifdef CONFIG_IGB_VMDQ_NETDEV
2495 igb_remove_vmdq_netdevs(adapter);
2498 igb_clear_interrupt_scheme(adapter);
2499 igb_reset_sriov_capability(adapter);
2501 iounmap(hw->hw_addr);
2502 if (hw->flash_address)
2503 iounmap(hw->flash_address);
2504 pci_release_selected_regions(pdev,
2505 pci_select_bars(pdev, IORESOURCE_MEM));
2507 kfree(adapter->mac_table);
2508 kfree(adapter->shadow_vfta);
2509 free_netdev(netdev);
2511 pci_disable_pcie_error_reporting(pdev);
2513 pci_disable_device(pdev);
2516 igb_sysfs_exit(adapter);
2519 igb_procfs_exit(adapter);
2520 #endif /* IGB_PROCFS */
2521 #endif /* IGB_SYSFS */
2524 #ifdef HAVE_HW_TIME_STAMP
2526 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2527 * @adapter: board private structure to initialize
2529 * igb_init_hw_timer initializes the function pointer and values for the hw
2530 * timer found in hardware.
2532 static void igb_init_hw_timer(struct igb_adapter *adapter)
2534 struct e1000_hw *hw = &adapter->hw;
2536 switch (hw->mac.type) {
2539 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2540 adapter->cycles.read = igb_read_clock;
2541 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2542 adapter->cycles.mult = 1;
2544 * The 82580 timesync updates the system timer every 8ns by 8ns
2545 * and the value cannot be shifted. Instead we need to shift
2546 * the registers to generate a 64bit timer value. As a result
2547 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2548 * 24 in order to generate a larger value for synchronization.
2550 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2551 /* disable system timer temporarily by setting bit 31 */
2552 E1000_WRITE_REG(hw, E1000_TSAUXC, 0x80000000);
2553 E1000_WRITE_FLUSH(hw);
2555 /* Set registers so that rollover occurs soon to test this. */
2556 E1000_WRITE_REG(hw, E1000_SYSTIMR, 0x00000000);
2557 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x80000000);
2558 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0x000000FF);
2559 E1000_WRITE_FLUSH(hw);
2561 /* enable system timer by clearing bit 31 */
2562 E1000_WRITE_REG(hw, E1000_TSAUXC, 0x0);
2563 E1000_WRITE_FLUSH(hw);
2565 timecounter_init(&adapter->clock,
2567 ktime_to_ns(ktime_get_real()));
2569 * Synchronize our NIC clock against system wall clock. NIC
2570 * time stamp reading requires ~3us per sample, each sample
2571 * was pretty stable even under load => only require 10
2572 * samples for each offset comparison.
2574 memset(&adapter->compare, 0, sizeof(adapter->compare));
2575 adapter->compare.source = &adapter->clock;
2576 adapter->compare.target = ktime_get_real;
2577 adapter->compare.num_samples = 10;
2578 timecompare_update(&adapter->compare, 0);
2582 * Initialize hardware timer: we keep it running just in case
2583 * that some program needs it later on.
2585 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2586 adapter->cycles.read = igb_read_clock;
2587 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2588 adapter->cycles.mult = 1;
2590 * Scale the NIC clock cycle by a large factor so that
2591 * relatively small clock corrections can be added or
2592 * subtracted at each clock tick. The drawbacks of a large
2593 * factor are a) that the clock register overflows more quickly
2594 * (not such a big deal) and b) that the increment per tick has
2595 * to fit into 24 bits. As a result we need to use a shift of
2596 * 19 so we can fit a value of 16 into the TIMINCA register.
2598 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2599 E1000_WRITE_REG(hw, E1000_TIMINCA,
2600 (1 << E1000_TIMINCA_16NS_SHIFT) |
2601 (16 << IGB_82576_TSYNC_SHIFT));
2603 /* Set registers so that rollover occurs soon to test this. */
2604 E1000_WRITE_REG(hw, E1000_SYSTIML, 0x00000000);
2605 E1000_WRITE_REG(hw, E1000_SYSTIMH, 0xFF800000);
2606 E1000_WRITE_FLUSH(hw);
2608 timecounter_init(&adapter->clock,
2610 ktime_to_ns(ktime_get_real()));
2612 * Synchronize our NIC clock against system wall clock. NIC
2613 * time stamp reading requires ~3us per sample, each sample
2614 * was pretty stable even under load => only require 10
2615 * samples for each offset comparison.
2617 memset(&adapter->compare, 0, sizeof(adapter->compare));
2618 adapter->compare.source = &adapter->clock;
2619 adapter->compare.target = ktime_get_real;
2620 adapter->compare.num_samples = 10;
2621 timecompare_update(&adapter->compare, 0);
2624 /* 82575 does not support timesync */
2630 #endif /* HAVE_HW_TIME_STAMP */
2632 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2633 * @adapter: board private structure to initialize
2635 * igb_sw_init initializes the Adapter private data structure.
2636 * Fields are initialized based on PCI device information and
2637 * OS network device settings (MTU size).
2639 static int igb_sw_init(struct igb_adapter *adapter)
2641 struct e1000_hw *hw = &adapter->hw;
2642 struct net_device *netdev = adapter->netdev;
2643 struct pci_dev *pdev = adapter->pdev;
2645 /* PCI config space info */
2647 hw->vendor_id = pdev->vendor;
2648 hw->device_id = pdev->device;
2649 hw->subsystem_vendor_id = pdev->subsystem_vendor;
2650 hw->subsystem_device_id = pdev->subsystem_device;
2652 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
2654 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2656 /* set default ring sizes */
2657 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2658 adapter->rx_ring_count = IGB_DEFAULT_RXD;
2660 /* set default work limits */
2661 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2663 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2666 /* Initialize the hardware-specific values */
2667 if (e1000_setup_init_funcs(hw, TRUE)) {
2668 dev_err(pci_dev_to_dev(pdev), "Hardware Initialization Failure\n");
2672 adapter->mac_table = kzalloc(sizeof(struct igb_mac_addr) *
2673 hw->mac.rar_entry_count,
2676 /* Setup and initialize a copy of the hw vlan table array */
2677 adapter->shadow_vfta = (u32 *)kzalloc(sizeof(u32) * E1000_VFTA_ENTRIES,
2680 /* These calls may decrease the number of queues */
2681 igb_set_sriov_capability(adapter);
2683 if (igb_init_interrupt_scheme(adapter)) {
2684 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for queues\n");
2688 /* Explicitly disable IRQ since the NIC can be in any state. */
2689 igb_irq_disable(adapter);
2691 set_bit(__IGB_DOWN, &adapter->state);
2697 * igb_open - Called when a network interface is made active
2698 * @netdev: network interface device structure
2700 * Returns 0 on success, negative value on failure
2702 * The open entry point is called when a network interface is made
2703 * active by the system (IFF_UP). At this point all resources needed
2704 * for transmit and receive operations are allocated, the interrupt
2705 * handler is registered with the OS, the watchdog timer is started,
2706 * and the stack is notified that the interface is ready.
2708 static int __igb_open(struct net_device *netdev, bool resuming)
2710 struct igb_adapter *adapter = netdev_priv(netdev);
2711 struct e1000_hw *hw = &adapter->hw;
2712 #ifdef CONFIG_PM_RUNTIME
2713 struct pci_dev *pdev = adapter->pdev;
2714 #endif /* CONFIG_PM_RUNTIME */
2718 /* disallow open during test */
2719 if (test_bit(__IGB_TESTING, &adapter->state)) {
2724 #ifdef CONFIG_PM_RUNTIME
2726 pm_runtime_get_sync(&pdev->dev);
2727 #endif /* CONFIG_PM_RUNTIME */
2729 netif_carrier_off(netdev);
2731 /* allocate transmit descriptors */
2732 err = igb_setup_all_tx_resources(adapter);
2736 /* allocate receive descriptors */
2737 err = igb_setup_all_rx_resources(adapter);
2741 igb_power_up_link(adapter);
2743 /* before we allocate an interrupt, we must be ready to handle it.
2744 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2745 * as soon as we call pci_request_irq, so we have to setup our
2746 * clean_rx handler before we do so. */
2747 igb_configure(adapter);
2749 err = igb_request_irq(adapter);
2753 /* From here on the code is the same as igb_up() */
2754 clear_bit(__IGB_DOWN, &adapter->state);
2756 for (i = 0; i < adapter->num_q_vectors; i++)
2757 napi_enable(&(adapter->q_vector[i]->napi));
2758 igb_configure_lli(adapter);
2760 /* Clear any pending interrupts. */
2761 E1000_READ_REG(hw, E1000_ICR);
2763 igb_irq_enable(adapter);
2765 /* notify VFs that reset has been completed */
2766 if (adapter->vfs_allocated_count) {
2767 u32 reg_data = E1000_READ_REG(hw, E1000_CTRL_EXT);
2768 reg_data |= E1000_CTRL_EXT_PFRSTD;
2769 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg_data);
2772 netif_tx_start_all_queues(netdev);
2774 if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
2775 schedule_work(&adapter->dma_err_task);
2777 /* start the watchdog. */
2778 hw->mac.get_link_status = 1;
2779 schedule_work(&adapter->watchdog_task);
2781 return E1000_SUCCESS;
2784 igb_release_hw_control(adapter);
2785 igb_power_down_link(adapter);
2786 igb_free_all_rx_resources(adapter);
2788 igb_free_all_tx_resources(adapter);
2792 #ifdef CONFIG_PM_RUNTIME
2794 pm_runtime_put(&pdev->dev);
2795 #endif /* CONFIG_PM_RUNTIME */
2800 static int igb_open(struct net_device *netdev)
2802 return __igb_open(netdev, false);
2806 * igb_close - Disables a network interface
2807 * @netdev: network interface device structure
2809 * Returns 0, this is not allowed to fail
2811 * The close entry point is called when an interface is de-activated
2812 * by the OS. The hardware is still under the driver's control, but
2813 * needs to be disabled. A global MAC reset is issued to stop the
2814 * hardware, and all transmit and receive resources are freed.
2816 static int __igb_close(struct net_device *netdev, bool suspending)
2818 struct igb_adapter *adapter = netdev_priv(netdev);
2819 #ifdef CONFIG_PM_RUNTIME
2820 struct pci_dev *pdev = adapter->pdev;
2821 #endif /* CONFIG_PM_RUNTIME */
2823 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2825 #ifdef CONFIG_PM_RUNTIME
2827 pm_runtime_get_sync(&pdev->dev);
2828 #endif /* CONFIG_PM_RUNTIME */
2832 igb_release_hw_control(adapter);
2834 igb_free_irq(adapter);
2836 igb_free_all_tx_resources(adapter);
2837 igb_free_all_rx_resources(adapter);
2839 #ifdef CONFIG_PM_RUNTIME
2841 pm_runtime_put_sync(&pdev->dev);
2842 #endif /* CONFIG_PM_RUNTIME */
2847 static int igb_close(struct net_device *netdev)
2849 return __igb_close(netdev, false);
2853 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2854 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2856 * Return 0 on success, negative on failure
2858 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2860 struct device *dev = tx_ring->dev;
2861 int orig_node = dev_to_node(dev);
2864 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
2865 tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
2866 if (!tx_ring->tx_buffer_info)
2867 tx_ring->tx_buffer_info = vzalloc(size);
2868 if (!tx_ring->tx_buffer_info)
2871 /* round up to nearest 4K */
2872 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2873 tx_ring->size = ALIGN(tx_ring->size, 4096);
2875 set_dev_node(dev, tx_ring->numa_node);
2876 tx_ring->desc = dma_alloc_coherent(dev,
2880 set_dev_node(dev, orig_node);
2882 tx_ring->desc = dma_alloc_coherent(dev,
2890 tx_ring->next_to_use = 0;
2891 tx_ring->next_to_clean = 0;
2896 vfree(tx_ring->tx_buffer_info);
2898 "Unable to allocate memory for the transmit descriptor ring\n");
2903 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2904 * (Descriptors) for all queues
2905 * @adapter: board private structure
2907 * Return 0 on success, negative on failure
2909 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2911 struct pci_dev *pdev = adapter->pdev;
2914 for (i = 0; i < adapter->num_tx_queues; i++) {
2915 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2917 dev_err(pci_dev_to_dev(pdev),
2918 "Allocation for Tx Queue %u failed\n", i);
2919 for (i--; i >= 0; i--)
2920 igb_free_tx_resources(adapter->tx_ring[i]);
2929 * igb_setup_tctl - configure the transmit control registers
2930 * @adapter: Board private structure
2932 void igb_setup_tctl(struct igb_adapter *adapter)
2934 struct e1000_hw *hw = &adapter->hw;
2937 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2938 E1000_WRITE_REG(hw, E1000_TXDCTL(0), 0);
2940 /* Program the Transmit Control Register */
2941 tctl = E1000_READ_REG(hw, E1000_TCTL);
2942 tctl &= ~E1000_TCTL_CT;
2943 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2944 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2946 e1000_config_collision_dist(hw);
2948 /* Enable transmits */
2949 tctl |= E1000_TCTL_EN;
2951 E1000_WRITE_REG(hw, E1000_TCTL, tctl);
2955 * igb_configure_tx_ring - Configure transmit ring after Reset
2956 * @adapter: board private structure
2957 * @ring: tx ring to configure
2959 * Configure a transmit ring after a reset.
2961 void igb_configure_tx_ring(struct igb_adapter *adapter,
2962 struct igb_ring *ring)
2964 struct e1000_hw *hw = &adapter->hw;
2966 u64 tdba = ring->dma;
2967 int reg_idx = ring->reg_idx;
2969 /* disable the queue */
2970 E1000_WRITE_REG(hw, E1000_TXDCTL(reg_idx), 0);
2971 E1000_WRITE_FLUSH(hw);
2974 E1000_WRITE_REG(hw, E1000_TDLEN(reg_idx),
2975 ring->count * sizeof(union e1000_adv_tx_desc));
2976 E1000_WRITE_REG(hw, E1000_TDBAL(reg_idx),
2977 tdba & 0x00000000ffffffffULL);
2978 E1000_WRITE_REG(hw, E1000_TDBAH(reg_idx), tdba >> 32);
2980 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2981 E1000_WRITE_REG(hw, E1000_TDH(reg_idx), 0);
2982 writel(0, ring->tail);
2984 txdctl |= IGB_TX_PTHRESH;
2985 txdctl |= IGB_TX_HTHRESH << 8;
2986 txdctl |= IGB_TX_WTHRESH << 16;
2988 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2989 E1000_WRITE_REG(hw, E1000_TXDCTL(reg_idx), txdctl);
2993 * igb_configure_tx - Configure transmit Unit after Reset
2994 * @adapter: board private structure
2996 * Configure the Tx unit of the MAC after a reset.
2998 static void igb_configure_tx(struct igb_adapter *adapter)
3002 for (i = 0; i < adapter->num_tx_queues; i++)
3003 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
3007 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
3008 * @rx_ring: rx descriptor ring (for a specific queue) to setup
3010 * Returns 0 on success, negative on failure
3012 int igb_setup_rx_resources(struct igb_ring *rx_ring)
3014 struct device *dev = rx_ring->dev;
3015 int orig_node = dev_to_node(dev);
3018 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3019 rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
3020 if (!rx_ring->rx_buffer_info)
3021 rx_ring->rx_buffer_info = vzalloc(size);
3022 if (!rx_ring->rx_buffer_info)
3025 desc_len = sizeof(union e1000_adv_rx_desc);
3027 /* Round up to nearest 4K */
3028 rx_ring->size = rx_ring->count * desc_len;
3029 rx_ring->size = ALIGN(rx_ring->size, 4096);
3031 set_dev_node(dev, rx_ring->numa_node);
3032 rx_ring->desc = dma_alloc_coherent(dev,
3036 set_dev_node(dev, orig_node);
3038 rx_ring->desc = dma_alloc_coherent(dev,
3046 rx_ring->next_to_clean = 0;
3047 rx_ring->next_to_use = 0;
3052 vfree(rx_ring->rx_buffer_info);
3053 rx_ring->rx_buffer_info = NULL;
3054 dev_err(dev, "Unable to allocate memory for the receive descriptor"
3060 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
3061 * (Descriptors) for all queues
3062 * @adapter: board private structure
3064 * Return 0 on success, negative on failure
3066 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
3068 struct pci_dev *pdev = adapter->pdev;
3071 for (i = 0; i < adapter->num_rx_queues; i++) {
3072 err = igb_setup_rx_resources(adapter->rx_ring[i]);
3074 dev_err(pci_dev_to_dev(pdev),
3075 "Allocation for Rx Queue %u failed\n", i);
3076 for (i--; i >= 0; i--)
3077 igb_free_rx_resources(adapter->rx_ring[i]);
3086 * igb_setup_mrqc - configure the multiple receive queue control registers
3087 * @adapter: Board private structure
3089 static void igb_setup_mrqc(struct igb_adapter *adapter)
3091 struct e1000_hw *hw = &adapter->hw;
3093 u32 j, num_rx_queues, shift = 0, shift2 = 0;
3098 static const u8 rsshash[40] = {
3099 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
3100 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
3101 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
3102 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
3104 /* Fill out hash function seeds */
3105 for (j = 0; j < 10; j++) {
3106 u32 rsskey = rsshash[(j * 4)];
3107 rsskey |= rsshash[(j * 4) + 1] << 8;
3108 rsskey |= rsshash[(j * 4) + 2] << 16;
3109 rsskey |= rsshash[(j * 4) + 3] << 24;
3110 E1000_WRITE_REG_ARRAY(hw, E1000_RSSRK(0), j, rsskey);
3113 num_rx_queues = adapter->rss_queues;
3115 if (adapter->vfs_allocated_count || adapter->vmdq_pools) {
3116 /* 82575 and 82576 supports 2 RSS queues for VMDq */
3117 switch (hw->mac.type) {
3134 if (hw->mac.type == e1000_82575)
3138 for (j = 0; j < (32 * 4); j++) {
3139 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
3141 reta.bytes[j & 3] |= num_rx_queues << shift2;
3143 E1000_WRITE_REG(hw, E1000_RETA(j >> 2), reta.dword);
3147 * Disable raw packet checksumming so that RSS hash is placed in
3148 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
3149 * offloads as they are enabled by default
3151 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
3152 rxcsum |= E1000_RXCSUM_PCSD;
3154 if (adapter->hw.mac.type >= e1000_82576)
3155 /* Enable Receive Checksum Offload for SCTP */
3156 rxcsum |= E1000_RXCSUM_CRCOFL;
3158 /* Don't need to set TUOFL or IPOFL, they default to 1 */
3159 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
3161 /* If VMDq is enabled then we set the appropriate mode for that, else
3162 * we default to RSS so that an RSS hash is calculated per packet even
3163 * if we are only using one queue */
3164 if (adapter->vfs_allocated_count || adapter->vmdq_pools) {
3165 if (hw->mac.type > e1000_82575) {
3166 /* Set the default pool for the PF's first queue */
3167 u32 vtctl = E1000_READ_REG(hw, E1000_VT_CTL);
3168 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
3169 E1000_VT_CTL_DISABLE_DEF_POOL);
3170 vtctl |= adapter->vfs_allocated_count <<
3171 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
3172 E1000_WRITE_REG(hw, E1000_VT_CTL, vtctl);
3173 } else if (adapter->rss_queues > 1) {
3174 /* set default queue for pool 1 to queue 2 */
3175 E1000_WRITE_REG(hw, E1000_VT_CTL,
3176 adapter->rss_queues << 7);
3178 if (adapter->rss_queues > 1)
3179 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
3181 mrqc = E1000_MRQC_ENABLE_VMDQ;
3183 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
3186 igb_vmm_control(adapter);
3189 * Generate RSS hash based on TCP port numbers and/or
3190 * IPv4/v6 src and dst addresses since UDP cannot be
3191 * hashed reliably due to IP fragmentation
3193 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
3194 E1000_MRQC_RSS_FIELD_IPV4_TCP |
3195 E1000_MRQC_RSS_FIELD_IPV6 |
3196 E1000_MRQC_RSS_FIELD_IPV6_TCP |
3197 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
3199 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
3203 * igb_setup_rctl - configure the receive control registers
3204 * @adapter: Board private structure
3206 void igb_setup_rctl(struct igb_adapter *adapter)
3208 struct e1000_hw *hw = &adapter->hw;
3211 rctl = E1000_READ_REG(hw, E1000_RCTL);
3213 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3214 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
3216 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
3217 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
3220 * enable stripping of CRC. It's unlikely this will break BMC
3221 * redirection as it did with e1000. Newer features require
3222 * that the HW strips the CRC.
3224 rctl |= E1000_RCTL_SECRC;
3226 /* disable store bad packets and clear size bits. */
3227 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
3229 /* enable LPE to prevent packets larger than max_frame_size */
3230 rctl |= E1000_RCTL_LPE;
3232 /* disable queue 0 to prevent tail write w/o re-config */
3233 E1000_WRITE_REG(hw, E1000_RXDCTL(0), 0);
3235 /* Attention!!! For SR-IOV PF driver operations you must enable
3236 * queue drop for all VF and PF queues to prevent head of line blocking
3237 * if an un-trusted VF does not provide descriptors to hardware.
3239 if (adapter->vfs_allocated_count) {
3240 /* set all queue drop enable bits */
3241 E1000_WRITE_REG(hw, E1000_QDE, ALL_QUEUES);
3244 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3247 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
3250 struct e1000_hw *hw = &adapter->hw;
3253 /* if it isn't the PF check to see if VFs are enabled and
3254 * increase the size to support vlan tags */
3255 if (vfn < adapter->vfs_allocated_count &&
3256 adapter->vf_data[vfn].vlans_enabled)
3259 #ifdef CONFIG_IGB_VMDQ_NETDEV
3260 if (vfn >= adapter->vfs_allocated_count) {
3261 int queue = vfn - adapter->vfs_allocated_count;
3262 struct igb_vmdq_adapter *vadapter;
3264 vadapter = netdev_priv(adapter->vmdq_netdev[queue-1]);
3265 if (vadapter->vlgrp)
3269 vmolr = E1000_READ_REG(hw, E1000_VMOLR(vfn));
3270 vmolr &= ~E1000_VMOLR_RLPML_MASK;
3271 vmolr |= size | E1000_VMOLR_LPE;
3272 E1000_WRITE_REG(hw, E1000_VMOLR(vfn), vmolr);
3278 * igb_rlpml_set - set maximum receive packet size
3279 * @adapter: board private structure
3281 * Configure maximum receivable packet size.
3283 static void igb_rlpml_set(struct igb_adapter *adapter)
3285 u32 max_frame_size = adapter->max_frame_size;
3286 struct e1000_hw *hw = &adapter->hw;
3287 u16 pf_id = adapter->vfs_allocated_count;
3289 if (adapter->vmdq_pools && hw->mac.type != e1000_82575) {
3291 for (i = 0; i < adapter->vmdq_pools; i++)
3292 igb_set_vf_rlpml(adapter, max_frame_size, pf_id + i);
3294 * If we're in VMDQ or SR-IOV mode, then set global RLPML
3295 * to our max jumbo frame size, in case we need to enable
3296 * jumbo frames on one of the rings later.
3297 * This will not pass over-length frames into the default
3298 * queue because it's gated by the VMOLR.RLPML.
3300 max_frame_size = MAX_JUMBO_FRAME_SIZE;
3302 /* Set VF RLPML for the PF device. */
3303 if (adapter->vfs_allocated_count)
3304 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
3306 E1000_WRITE_REG(hw, E1000_RLPML, max_frame_size);
3309 static inline void igb_set_vf_vlan_strip(struct igb_adapter *adapter,
3310 int vfn, bool enable)
3312 struct e1000_hw *hw = &adapter->hw;
3316 if (hw->mac.type < e1000_82576)
3319 if (hw->mac.type == e1000_i350)
3320 reg = hw->hw_addr + E1000_DVMOLR(vfn);
3322 reg = hw->hw_addr + E1000_VMOLR(vfn);
3326 val |= E1000_VMOLR_STRVLAN;
3328 val &= ~(E1000_VMOLR_STRVLAN);
3331 static inline void igb_set_vmolr(struct igb_adapter *adapter,
3334 struct e1000_hw *hw = &adapter->hw;
3338 * This register exists only on 82576 and newer so if we are older then
3339 * we should exit and do nothing
3341 if (hw->mac.type < e1000_82576)
3344 vmolr = E1000_READ_REG(hw, E1000_VMOLR(vfn));
3347 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3349 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
3351 /* clear all bits that might not be set */
3352 vmolr &= ~E1000_VMOLR_RSSE;
3354 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
3355 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3357 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3358 vmolr |= E1000_VMOLR_LPE; /* Accept long packets */
3360 E1000_WRITE_REG(hw, E1000_VMOLR(vfn), vmolr);
3364 * igb_configure_rx_ring - Configure a receive ring after Reset
3365 * @adapter: board private structure
3366 * @ring: receive ring to be configured
3368 * Configure the Rx unit of the MAC after a reset.
3370 void igb_configure_rx_ring(struct igb_adapter *adapter,
3371 struct igb_ring *ring)
3373 struct e1000_hw *hw = &adapter->hw;
3374 u64 rdba = ring->dma;
3375 int reg_idx = ring->reg_idx;
3376 u32 srrctl = 0, rxdctl = 0;
3378 /* disable the queue */
3379 E1000_WRITE_REG(hw, E1000_RXDCTL(reg_idx), 0);
3381 /* Set DMA base address registers */
3382 E1000_WRITE_REG(hw, E1000_RDBAL(reg_idx),
3383 rdba & 0x00000000ffffffffULL);
3384 E1000_WRITE_REG(hw, E1000_RDBAH(reg_idx), rdba >> 32);
3385 E1000_WRITE_REG(hw, E1000_RDLEN(reg_idx),
3386 ring->count * sizeof(union e1000_adv_rx_desc));
3388 /* initialize head and tail */
3389 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
3390 E1000_WRITE_REG(hw, E1000_RDH(reg_idx), 0);
3391 writel(0, ring->tail);
3393 /* set descriptor configuration */
3394 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
3395 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3396 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
3397 srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3399 srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3401 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3402 #else /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
3403 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
3404 E1000_SRRCTL_BSIZEPKT_SHIFT;
3405 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
3406 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
3407 #ifdef IGB_PER_PKT_TIMESTAMP
3408 if (hw->mac.type >= e1000_82580)
3409 srrctl |= E1000_SRRCTL_TIMESTAMP;
3412 * We should set the drop enable bit if:
3415 * Flow Control is disabled and number of RX queues > 1
3417 * This allows us to avoid head of line blocking for security
3418 * and performance reasons.
3420 if (adapter->vfs_allocated_count ||
3421 (adapter->num_rx_queues > 1 &&
3422 (hw->fc.requested_mode == e1000_fc_none ||
3423 hw->fc.requested_mode == e1000_fc_rx_pause)))
3424 srrctl |= E1000_SRRCTL_DROP_EN;
3426 E1000_WRITE_REG(hw, E1000_SRRCTL(reg_idx), srrctl);
3428 /* set filtering for VMDQ pools */
3429 igb_set_vmolr(adapter, reg_idx & 0x7, true);
3431 rxdctl |= IGB_RX_PTHRESH;
3432 rxdctl |= IGB_RX_HTHRESH << 8;
3433 rxdctl |= IGB_RX_WTHRESH << 16;
3435 /* enable receive descriptor fetching */
3436 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3437 E1000_WRITE_REG(hw, E1000_RXDCTL(reg_idx), rxdctl);
3441 * igb_configure_rx - Configure receive Unit after Reset
3442 * @adapter: board private structure
3444 * Configure the Rx unit of the MAC after a reset.
3446 static void igb_configure_rx(struct igb_adapter *adapter)
3450 /* set UTA to appropriate mode */
3451 igb_set_uta(adapter);
3453 igb_full_sync_mac_table(adapter);
3454 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3455 * the Base and Length of the Rx Descriptor Ring */
3456 for (i = 0; i < adapter->num_rx_queues; i++)
3457 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
3461 * igb_free_tx_resources - Free Tx Resources per Queue
3462 * @tx_ring: Tx descriptor ring for a specific queue
3464 * Free all transmit software resources
3466 void igb_free_tx_resources(struct igb_ring *tx_ring)
3468 igb_clean_tx_ring(tx_ring);
3470 vfree(tx_ring->tx_buffer_info);
3471 tx_ring->tx_buffer_info = NULL;
3473 /* if not set, then don't free */
3477 dma_free_coherent(tx_ring->dev, tx_ring->size,
3478 tx_ring->desc, tx_ring->dma);
3480 tx_ring->desc = NULL;
3484 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3485 * @adapter: board private structure
3487 * Free all transmit software resources
3489 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3493 for (i = 0; i < adapter->num_tx_queues; i++)
3494 igb_free_tx_resources(adapter->tx_ring[i]);
3497 void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
3498 struct igb_tx_buffer *tx_buffer)
3500 if (tx_buffer->skb) {
3501 dev_kfree_skb_any(tx_buffer->skb);
3502 if (dma_unmap_len(tx_buffer, len))
3503 dma_unmap_single(ring->dev,
3504 dma_unmap_addr(tx_buffer, dma),
3505 dma_unmap_len(tx_buffer, len),
3507 } else if (dma_unmap_len(tx_buffer, len)) {
3508 dma_unmap_page(ring->dev,
3509 dma_unmap_addr(tx_buffer, dma),
3510 dma_unmap_len(tx_buffer, len),
3513 tx_buffer->next_to_watch = NULL;
3514 tx_buffer->skb = NULL;
3515 dma_unmap_len_set(tx_buffer, len, 0);
3516 /* buffer_info must be completely set up in the transmit path */
3520 * igb_clean_tx_ring - Free Tx Buffers
3521 * @tx_ring: ring to be cleaned
3523 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3525 struct igb_tx_buffer *buffer_info;
3529 if (!tx_ring->tx_buffer_info)
3531 /* Free all the Tx ring sk_buffs */
3533 for (i = 0; i < tx_ring->count; i++) {
3534 buffer_info = &tx_ring->tx_buffer_info[i];
3535 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3539 netdev_tx_reset_queue(txring_txq(tx_ring));
3540 #endif /* CONFIG_BQL */
3542 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3543 memset(tx_ring->tx_buffer_info, 0, size);
3545 /* Zero out the descriptor ring */
3546 memset(tx_ring->desc, 0, tx_ring->size);
3548 tx_ring->next_to_use = 0;
3549 tx_ring->next_to_clean = 0;
3553 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3554 * @adapter: board private structure
3556 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3560 for (i = 0; i < adapter->num_tx_queues; i++)
3561 igb_clean_tx_ring(adapter->tx_ring[i]);
3565 * igb_free_rx_resources - Free Rx Resources
3566 * @rx_ring: ring to clean the resources from
3568 * Free all receive software resources
3570 void igb_free_rx_resources(struct igb_ring *rx_ring)
3572 igb_clean_rx_ring(rx_ring);
3574 vfree(rx_ring->rx_buffer_info);
3575 rx_ring->rx_buffer_info = NULL;
3577 /* if not set, then don't free */
3581 dma_free_coherent(rx_ring->dev, rx_ring->size,
3582 rx_ring->desc, rx_ring->dma);
3584 rx_ring->desc = NULL;
3588 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3589 * @adapter: board private structure
3591 * Free all receive software resources
3593 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3597 for (i = 0; i < adapter->num_rx_queues; i++)
3598 igb_free_rx_resources(adapter->rx_ring[i]);
3602 * igb_clean_rx_ring - Free Rx Buffers per Queue
3603 * @rx_ring: ring to free buffers from
3605 void igb_clean_rx_ring(struct igb_ring *rx_ring)
3608 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
3609 const int bufsz = rx_ring->rx_buffer_len;
3611 const int bufsz = IGB_RX_HDR_LEN;
3615 if (!rx_ring->rx_buffer_info)
3618 /* Free all the Rx ring sk_buffs */
3619 for (i = 0; i < rx_ring->count; i++) {
3620 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
3621 if (buffer_info->dma) {
3622 dma_unmap_single(rx_ring->dev,
3626 buffer_info->dma = 0;
3629 if (buffer_info->skb) {
3630 dev_kfree_skb(buffer_info->skb);
3631 buffer_info->skb = NULL;
3633 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
3634 if (buffer_info->page_dma) {
3635 dma_unmap_page(rx_ring->dev,
3636 buffer_info->page_dma,
3639 buffer_info->page_dma = 0;
3641 if (buffer_info->page) {
3642 put_page(buffer_info->page);
3643 buffer_info->page = NULL;
3644 buffer_info->page_offset = 0;
3649 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3650 memset(rx_ring->rx_buffer_info, 0, size);
3652 /* Zero out the descriptor ring */
3653 memset(rx_ring->desc, 0, rx_ring->size);
3655 rx_ring->next_to_clean = 0;
3656 rx_ring->next_to_use = 0;
3660 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3661 * @adapter: board private structure
3663 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3667 for (i = 0; i < adapter->num_rx_queues; i++)
3668 igb_clean_rx_ring(adapter->rx_ring[i]);
3672 * igb_set_mac - Change the Ethernet Address of the NIC
3673 * @netdev: network interface device structure
3674 * @p: pointer to an address structure
3676 * Returns 0 on success, negative on failure
3678 static int igb_set_mac(struct net_device *netdev, void *p)
3680 struct igb_adapter *adapter = netdev_priv(netdev);
3681 struct e1000_hw *hw = &adapter->hw;
3682 struct sockaddr *addr = p;
3684 if (!is_valid_ether_addr(addr->sa_data))
3685 return -EADDRNOTAVAIL;
3687 igb_del_mac_filter(adapter, hw->mac.addr,
3688 adapter->vfs_allocated_count);
3689 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3690 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3692 /* set the correct pool for the new PF MAC address in entry 0 */
3693 return igb_add_mac_filter(adapter, hw->mac.addr,
3694 adapter->vfs_allocated_count);
3698 * igb_write_mc_addr_list - write multicast addresses to MTA
3699 * @netdev: network interface device structure
3701 * Writes multicast address list to the MTA hash table.
3702 * Returns: -ENOMEM on failure
3703 * 0 on no addresses written
3704 * X on writing X addresses to MTA
3706 int igb_write_mc_addr_list(struct net_device *netdev)
3708 struct igb_adapter *adapter = netdev_priv(netdev);
3709 struct e1000_hw *hw = &adapter->hw;
3710 #ifdef NETDEV_HW_ADDR_T_MULTICAST
3711 struct netdev_hw_addr *ha;
3713 struct dev_mc_list *ha;
3717 #ifdef CONFIG_IGB_VMDQ_NETDEV
3720 count = netdev_mc_count(netdev);
3721 #ifdef CONFIG_IGB_VMDQ_NETDEV
3722 for (vm = 1; vm < adapter->vmdq_pools; vm++) {
3723 if (!adapter->vmdq_netdev[vm])
3725 if (!netif_running(adapter->vmdq_netdev[vm]))
3727 count += netdev_mc_count(adapter->vmdq_netdev[vm]);
3732 e1000_update_mc_addr_list(hw, NULL, 0);
3735 mta_list = kzalloc(count * 6, GFP_ATOMIC);
3739 /* The shared function expects a packed array of only addresses. */
3741 netdev_for_each_mc_addr(ha, netdev)
3742 #ifdef NETDEV_HW_ADDR_T_MULTICAST
3743 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3745 memcpy(mta_list + (i++ * ETH_ALEN), ha->dmi_addr, ETH_ALEN);
3747 #ifdef CONFIG_IGB_VMDQ_NETDEV
3748 for (vm = 1; vm < adapter->vmdq_pools; vm++) {
3749 if (!adapter->vmdq_netdev[vm])
3751 if (!netif_running(adapter->vmdq_netdev[vm]) ||
3752 !netdev_mc_count(adapter->vmdq_netdev[vm]))
3754 netdev_for_each_mc_addr(ha, adapter->vmdq_netdev[vm])
3755 #ifdef NETDEV_HW_ADDR_T_MULTICAST
3756 memcpy(mta_list + (i++ * ETH_ALEN),
3757 ha->addr, ETH_ALEN);
3759 memcpy(mta_list + (i++ * ETH_ALEN),
3760 ha->dmi_addr, ETH_ALEN);
3764 e1000_update_mc_addr_list(hw, mta_list, i);
3770 void igb_rar_set(struct igb_adapter *adapter, u32 index)
3772 u32 rar_low, rar_high;
3773 struct e1000_hw *hw = &adapter->hw;
3774 u8 *addr = adapter->mac_table[index].addr;
3775 /* HW expects these in little endian so we reverse the byte order
3776 * from network order (big endian) to little endian
3778 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
3779 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
3780 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
3782 /* Indicate to hardware the Address is Valid. */
3783 if (adapter->mac_table[index].state & IGB_MAC_STATE_IN_USE)
3784 rar_high |= E1000_RAH_AV;
3786 if (hw->mac.type == e1000_82575)
3787 rar_high |= E1000_RAH_POOL_1 * adapter->mac_table[index].queue;
3789 rar_high |= E1000_RAH_POOL_1 << adapter->mac_table[index].queue;
3791 E1000_WRITE_REG(hw, E1000_RAL(index), rar_low);
3792 E1000_WRITE_FLUSH(hw);
3793 E1000_WRITE_REG(hw, E1000_RAH(index), rar_high);
3794 E1000_WRITE_FLUSH(hw);
3797 void igb_full_sync_mac_table(struct igb_adapter *adapter)
3799 struct e1000_hw *hw = &adapter->hw;
3801 for (i = 0; i < hw->mac.rar_entry_count; i++) {
3802 igb_rar_set(adapter, i);
3806 void igb_sync_mac_table(struct igb_adapter *adapter)
3808 struct e1000_hw *hw = &adapter->hw;
3810 for (i = 0; i < hw->mac.rar_entry_count; i++) {
3811 if (adapter->mac_table[i].state & IGB_MAC_STATE_MODIFIED)
3812 igb_rar_set(adapter, i);
3813 adapter->mac_table[i].state &= ~(IGB_MAC_STATE_MODIFIED);
3817 int igb_available_rars(struct igb_adapter *adapter)
3819 struct e1000_hw *hw = &adapter->hw;
3822 for (i = 0; i < hw->mac.rar_entry_count; i++) {
3823 if (adapter->mac_table[i].state == 0)
3829 #ifdef HAVE_SET_RX_MODE
3831 * igb_write_uc_addr_list - write unicast addresses to RAR table
3832 * @netdev: network interface device structure
3834 * Writes unicast address list to the RAR table.
3835 * Returns: -ENOMEM on failure/insufficient address space
3836 * 0 on no addresses written
3837 * X on writing X addresses to the RAR table
3839 static int igb_write_uc_addr_list(struct net_device *netdev)
3841 struct igb_adapter *adapter = netdev_priv(netdev);
3842 unsigned int vfn = adapter->vfs_allocated_count;
3845 /* return ENOMEM indicating insufficient memory for addresses */
3846 if (netdev_uc_count(netdev) > igb_available_rars(adapter))
3848 if (!netdev_uc_empty(netdev)) {
3849 #ifdef NETDEV_HW_ADDR_T_UNICAST
3850 struct netdev_hw_addr *ha;
3852 struct dev_mc_list *ha;
3854 netdev_for_each_uc_addr(ha, netdev) {
3855 #ifdef NETDEV_HW_ADDR_T_UNICAST
3856 igb_del_mac_filter(adapter, ha->addr, vfn);
3857 igb_add_mac_filter(adapter, ha->addr, vfn);
3859 igb_del_mac_filter(adapter, ha->da_addr, vfn);
3860 igb_add_mac_filter(adapter, ha->da_addr, vfn);
3868 #endif /* HAVE_SET_RX_MODE */
3870 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3871 * @netdev: network interface device structure
3873 * The set_rx_mode entry point is called whenever the unicast or multicast
3874 * address lists or the network interface flags are updated. This routine is
3875 * responsible for configuring the hardware for proper unicast, multicast,
3876 * promiscuous mode, and all-multi behavior.
3878 static void igb_set_rx_mode(struct net_device *netdev)
3880 struct igb_adapter *adapter = netdev_priv(netdev);
3881 struct e1000_hw *hw = &adapter->hw;
3882 unsigned int vfn = adapter->vfs_allocated_count;
3883 u32 rctl, vmolr = 0;
3886 /* Check for Promiscuous and All Multicast modes */
3887 rctl = E1000_READ_REG(hw, E1000_RCTL);
3889 /* clear the effected bits */
3890 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3892 if (netdev->flags & IFF_PROMISC) {
3893 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3894 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3896 if (netdev->flags & IFF_ALLMULTI) {
3897 rctl |= E1000_RCTL_MPE;
3898 vmolr |= E1000_VMOLR_MPME;
3901 * Write addresses to the MTA, if the attempt fails
3902 * then we should just turn on promiscuous mode so
3903 * that we can at least receive multicast traffic
3905 count = igb_write_mc_addr_list(netdev);
3907 rctl |= E1000_RCTL_MPE;
3908 vmolr |= E1000_VMOLR_MPME;
3910 vmolr |= E1000_VMOLR_ROMPE;
3913 #ifdef HAVE_SET_RX_MODE
3915 * Write addresses to available RAR registers, if there is not
3916 * sufficient space to store all the addresses then enable
3917 * unicast promiscuous mode
3919 count = igb_write_uc_addr_list(netdev);
3921 rctl |= E1000_RCTL_UPE;
3922 vmolr |= E1000_VMOLR_ROPE;
3924 #endif /* HAVE_SET_RX_MODE */
3925 rctl |= E1000_RCTL_VFE;
3927 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
3930 * In order to support SR-IOV and eventually VMDq it is necessary to set
3931 * the VMOLR to enable the appropriate modes. Without this workaround
3932 * we will have issues with VLAN tag stripping not being done for frames
3933 * that are only arriving because we are the default pool
3935 if (hw->mac.type < e1000_82576)
3938 vmolr |= E1000_READ_REG(hw, E1000_VMOLR(vfn)) &
3939 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3940 E1000_WRITE_REG(hw, E1000_VMOLR(vfn), vmolr);
3941 igb_restore_vf_multicasts(adapter);
3944 static void igb_check_wvbr(struct igb_adapter *adapter)
3946 struct e1000_hw *hw = &adapter->hw;
3949 switch (hw->mac.type) {
3952 if (!(wvbr = E1000_READ_REG(hw, E1000_WVBR)))
3959 adapter->wvbr |= wvbr;
3962 #define IGB_STAGGERED_QUEUE_OFFSET 8
3964 static void igb_spoof_check(struct igb_adapter *adapter)
3971 for(j = 0; j < adapter->vfs_allocated_count; j++) {
3972 if (adapter->wvbr & (1 << j) ||
3973 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3974 DPRINTK(DRV, WARNING,
3975 "Spoof event(s) detected on VF %d\n", j);
3978 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3983 /* Need to wait a few seconds after link up to get diagnostic information from
3985 static void igb_update_phy_info(unsigned long data)
3987 struct igb_adapter *adapter = (struct igb_adapter *) data;
3988 e1000_get_phy_info(&adapter->hw);
3992 * igb_has_link - check shared code for link and determine up/down
3993 * @adapter: pointer to driver private info
3995 bool igb_has_link(struct igb_adapter *adapter)
3997 struct e1000_hw *hw = &adapter->hw;
3998 bool link_active = FALSE;
4000 /* get_link_status is set on LSC (link status) interrupt or
4001 * rx sequence error interrupt. get_link_status will stay
4002 * false until the e1000_check_for_link establishes link
4003 * for copper adapters ONLY
4005 switch (hw->phy.media_type) {
4006 case e1000_media_type_copper:
4007 if (!hw->mac.get_link_status)
4009 case e1000_media_type_internal_serdes:
4010 e1000_check_for_link(hw);
4011 link_active = !hw->mac.get_link_status;
4013 case e1000_media_type_unknown:
4022 * igb_watchdog - Timer Call-back
4023 * @data: pointer to adapter cast into an unsigned long
4025 static void igb_watchdog(unsigned long data)
4027 struct igb_adapter *adapter = (struct igb_adapter *)data;
4028 /* Do the rest outside of interrupt context */
4029 schedule_work(&adapter->watchdog_task);
4032 static void igb_watchdog_task(struct work_struct *work)
4034 struct igb_adapter *adapter = container_of(work,
4037 struct e1000_hw *hw = &adapter->hw;
4038 struct net_device *netdev = adapter->netdev;
4041 u32 thstat, ctrl_ext;
4044 link = igb_has_link(adapter);
4046 /* Cancel scheduled suspend requests. */
4047 pm_runtime_resume(netdev->dev.parent);
4049 if (!netif_carrier_ok(netdev)) {
4051 e1000_get_speed_and_duplex(hw,
4052 &adapter->link_speed,
4053 &adapter->link_duplex);
4055 ctrl = E1000_READ_REG(hw, E1000_CTRL);
4056 /* Links status message must follow this format */
4057 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
4058 "Flow Control: %s\n",
4060 adapter->link_speed,
4061 adapter->link_duplex == FULL_DUPLEX ?
4062 "Full Duplex" : "Half Duplex",
4063 ((ctrl & E1000_CTRL_TFCE) &&
4064 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX":
4065 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
4066 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
4067 /* adjust timeout factor according to speed/duplex */
4068 adapter->tx_timeout_factor = 1;
4069 switch (adapter->link_speed) {
4071 adapter->tx_timeout_factor = 14;
4074 /* maybe add some timeout factor ? */
4078 netif_carrier_on(netdev);
4079 netif_tx_wake_all_queues(netdev);
4081 igb_ping_all_vfs(adapter);
4083 igb_check_vf_rate_limit(adapter);
4084 #endif /* IFLA_VF_MAX */
4086 /* link state has changed, schedule phy info update */
4087 if (!test_bit(__IGB_DOWN, &adapter->state))
4088 mod_timer(&adapter->phy_info_timer,
4089 round_jiffies(jiffies + 2 * HZ));
4092 if (netif_carrier_ok(netdev)) {
4093 adapter->link_speed = 0;
4094 adapter->link_duplex = 0;
4095 /* check for thermal sensor event on i350 */
4096 if (hw->mac.type == e1000_i350) {
4097 thstat = E1000_READ_REG(hw, E1000_THSTAT);
4098 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT);
4099 if ((hw->phy.media_type ==
4100 e1000_media_type_copper) &&
4102 E1000_CTRL_EXT_LINK_MODE_SGMII)) {
4103 if (thstat & E1000_THSTAT_PWR_DOWN) {
4104 printk(KERN_ERR "igb: %s The "
4105 "network adapter was stopped "
4106 "because it overheated.\n",
4109 if (thstat & E1000_THSTAT_LINK_THROTTLE) {
4111 "igb: %s The network "
4112 "adapter supported "
4122 /* Links status message must follow this format */
4123 printk(KERN_INFO "igb: %s NIC Link is Down\n",
4125 netif_carrier_off(netdev);
4126 netif_tx_stop_all_queues(netdev);
4128 igb_ping_all_vfs(adapter);
4130 /* link state has changed, schedule phy info update */
4131 if (!test_bit(__IGB_DOWN, &adapter->state))
4132 mod_timer(&adapter->phy_info_timer,
4133 round_jiffies(jiffies + 2 * HZ));
4135 pm_schedule_suspend(netdev->dev.parent,
4140 igb_update_stats(adapter);
4142 for (i = 0; i < adapter->num_tx_queues; i++) {
4143 struct igb_ring *tx_ring = adapter->tx_ring[i];
4144 if (!netif_carrier_ok(netdev)) {
4145 /* We've lost link, so the controller stops DMA,
4146 * but we've got queued Tx work that's never going
4147 * to get done, so reset controller to flush Tx.
4148 * (Do the reset outside of interrupt context). */
4149 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
4150 adapter->tx_timeout_count++;
4151 schedule_work(&adapter->reset_task);
4152 /* return immediately since reset is imminent */
4157 /* Force detection of hung controller every watchdog period */
4158 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
4161 /* Cause software interrupt to ensure rx ring is cleaned */
4162 if (adapter->msix_entries) {
4164 for (i = 0; i < adapter->num_q_vectors; i++)
4165 eics |= adapter->q_vector[i]->eims_value;
4166 E1000_WRITE_REG(hw, E1000_EICS, eics);
4168 E1000_WRITE_REG(hw, E1000_ICS, E1000_ICS_RXDMT0);
4171 igb_spoof_check(adapter);
4173 /* Reset the timer */
4174 if (!test_bit(__IGB_DOWN, &adapter->state))
4175 mod_timer(&adapter->watchdog_timer,
4176 round_jiffies(jiffies + 2 * HZ));
4179 static void igb_dma_err_task(struct work_struct *work)
4181 struct igb_adapter *adapter = container_of(work,
4185 struct e1000_hw *hw = &adapter->hw;
4186 struct net_device *netdev = adapter->netdev;
4190 hgptc = E1000_READ_REG(hw, E1000_HGPTC);
4191 if (hgptc) /* If incrementing then no need for the check below */
4192 goto dma_timer_reset;
4194 * Check to see if a bad DMA write target from an errant or
4195 * malicious VF has caused a PCIe error. If so then we can
4196 * issue a VFLR to the offending VF(s) and then resume without
4197 * requesting a full slot reset.
4200 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4201 ciaa = (vf << 16) | 0x80000000;
4202 /* 32 bit read so align, we really want status at offset 6 */
4203 ciaa |= PCI_COMMAND;
4204 E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4205 ciad = E1000_READ_REG(hw, E1000_CIAD);
4207 /* disable debug mode asap after reading data */
4208 E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4209 /* Get the upper 16 bits which will be the PCI status reg */
4211 if (ciad & (PCI_STATUS_REC_MASTER_ABORT |
4212 PCI_STATUS_REC_TARGET_ABORT |
4213 PCI_STATUS_SIG_SYSTEM_ERROR)) {
4214 netdev_err(netdev, "VF %d suffered error\n", vf);
4216 ciaa = (vf << 16) | 0x80000000;
4218 E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4219 ciad = 0x00008000; /* VFLR */
4220 E1000_WRITE_REG(hw, E1000_CIAD, ciad);
4222 E1000_WRITE_REG(hw, E1000_CIAA, ciaa);
4226 /* Reset the timer */
4227 if (!test_bit(__IGB_DOWN, &adapter->state))
4228 mod_timer(&adapter->dma_err_timer,
4229 round_jiffies(jiffies + HZ / 10));
4233 * igb_dma_err_timer - Timer Call-back
4234 * @data: pointer to adapter cast into an unsigned long
4236 static void igb_dma_err_timer(unsigned long data)
4238 struct igb_adapter *adapter = (struct igb_adapter *)data;
4239 /* Do the rest outside of interrupt context */
4240 schedule_work(&adapter->dma_err_task);
4243 enum latency_range {
4247 latency_invalid = 255
4251 * igb_update_ring_itr - update the dynamic ITR value based on packet size
4253 * Stores a new ITR value based on strictly on packet size. This
4254 * algorithm is less sophisticated than that used in igb_update_itr,
4255 * due to the difficulty of synchronizing statistics across multiple
4256 * receive rings. The divisors and thresholds used by this function
4257 * were determined based on theoretical maximum wire speed and testing
4258 * data, in order to minimize response time while increasing bulk
4260 * This functionality is controlled by the InterruptThrottleRate module
4261 * parameter (see igb_param.c)
4262 * NOTE: This function is called only when operating in a multiqueue
4263 * receive environment.
4264 * @q_vector: pointer to q_vector
4266 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
4268 int new_val = q_vector->itr_val;
4269 int avg_wire_size = 0;
4270 struct igb_adapter *adapter = q_vector->adapter;
4271 unsigned int packets;
4273 /* For non-gigabit speeds, just fix the interrupt rate at 4000
4274 * ints/sec - ITR timer value of 120 ticks.
4276 if (adapter->link_speed != SPEED_1000) {
4277 new_val = IGB_4K_ITR;
4281 packets = q_vector->rx.total_packets;
4283 avg_wire_size = q_vector->rx.total_bytes / packets;
4285 packets = q_vector->tx.total_packets;
4287 avg_wire_size = max_t(u32, avg_wire_size,
4288 q_vector->tx.total_bytes / packets);
4290 /* if avg_wire_size isn't set no work was done */
4294 /* Add 24 bytes to size to account for CRC, preamble, and gap */
4295 avg_wire_size += 24;
4297 /* Don't starve jumbo frames */
4298 avg_wire_size = min(avg_wire_size, 3000);
4300 /* Give a little boost to mid-size frames */
4301 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
4302 new_val = avg_wire_size / 3;
4304 new_val = avg_wire_size / 2;
4306 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4307 if (new_val < IGB_20K_ITR &&
4308 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4309 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4310 new_val = IGB_20K_ITR;
4313 if (new_val != q_vector->itr_val) {
4314 q_vector->itr_val = new_val;
4315 q_vector->set_itr = 1;
4318 q_vector->rx.total_bytes = 0;
4319 q_vector->rx.total_packets = 0;
4320 q_vector->tx.total_bytes = 0;
4321 q_vector->tx.total_packets = 0;
4325 * igb_update_itr - update the dynamic ITR value based on statistics
4326 * Stores a new ITR value based on packets and byte
4327 * counts during the last interrupt. The advantage of per interrupt
4328 * computation is faster updates and more accurate ITR for the current
4329 * traffic pattern. Constants in this function were computed
4330 * based on theoretical maximum wire speed and thresholds were set based
4331 * on testing data as well as attempting to minimize response time
4332 * while increasing bulk throughput.
4333 * this functionality is controlled by the InterruptThrottleRate module
4334 * parameter (see igb_param.c)
4335 * NOTE: These calculations are only valid when operating in a single-
4336 * queue environment.
4337 * @q_vector: pointer to q_vector
4338 * @ring_container: ring info to update the itr for
4340 static void igb_update_itr(struct igb_q_vector *q_vector,
4341 struct igb_ring_container *ring_container)
4343 unsigned int packets = ring_container->total_packets;
4344 unsigned int bytes = ring_container->total_bytes;
4345 u8 itrval = ring_container->itr;
4347 /* no packets, exit with status unchanged */
4352 case lowest_latency:
4353 /* handle TSO and jumbo frames */
4354 if (bytes/packets > 8000)
4355 itrval = bulk_latency;
4356 else if ((packets < 5) && (bytes > 512))
4357 itrval = low_latency;
4359 case low_latency: /* 50 usec aka 20000 ints/s */
4360 if (bytes > 10000) {
4361 /* this if handles the TSO accounting */
4362 if (bytes/packets > 8000) {
4363 itrval = bulk_latency;
4364 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
4365 itrval = bulk_latency;
4366 } else if ((packets > 35)) {
4367 itrval = lowest_latency;
4369 } else if (bytes/packets > 2000) {
4370 itrval = bulk_latency;
4371 } else if (packets <= 2 && bytes < 512) {
4372 itrval = lowest_latency;
4375 case bulk_latency: /* 250 usec aka 4000 ints/s */
4376 if (bytes > 25000) {
4378 itrval = low_latency;
4379 } else if (bytes < 1500) {
4380 itrval = low_latency;
4385 /* clear work counters since we have the values we need */
4386 ring_container->total_bytes = 0;
4387 ring_container->total_packets = 0;
4389 /* write updated itr to ring container */
4390 ring_container->itr = itrval;
4393 static void igb_set_itr(struct igb_q_vector *q_vector)
4395 struct igb_adapter *adapter = q_vector->adapter;
4396 u32 new_itr = q_vector->itr_val;
4399 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4400 if (adapter->link_speed != SPEED_1000) {
4402 new_itr = IGB_4K_ITR;
4406 igb_update_itr(q_vector, &q_vector->tx);
4407 igb_update_itr(q_vector, &q_vector->rx);
4409 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
4411 /* conservative mode (itr 3) eliminates the lowest_latency setting */
4412 if (current_itr == lowest_latency &&
4413 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4414 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4415 current_itr = low_latency;
4417 switch (current_itr) {
4418 /* counts and packets in update_itr are dependent on these numbers */
4419 case lowest_latency:
4420 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
4423 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
4426 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */
4433 if (new_itr != q_vector->itr_val) {
4434 /* this attempts to bias the interrupt rate towards Bulk
4435 * by adding intermediate steps when interrupt rate is
4437 new_itr = new_itr > q_vector->itr_val ?
4438 max((new_itr * q_vector->itr_val) /
4439 (new_itr + (q_vector->itr_val >> 2)),
4442 /* Don't write the value here; it resets the adapter's
4443 * internal timer, and causes us to delay far longer than
4444 * we should between interrupts. Instead, we write the ITR
4445 * value at the beginning of the next interrupt so the timing
4446 * ends up being correct.
4448 q_vector->itr_val = new_itr;
4449 q_vector->set_itr = 1;
4453 void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
4454 u32 type_tucmd, u32 mss_l4len_idx)
4456 struct e1000_adv_tx_context_desc *context_desc;
4457 u16 i = tx_ring->next_to_use;
4459 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
4462 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
4464 /* set bits to identify this as an advanced context descriptor */
4465 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
4467 /* For 82575, context index must be unique per ring. */
4468 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4469 mss_l4len_idx |= tx_ring->reg_idx << 4;
4471 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
4472 context_desc->seqnum_seed = 0;
4473 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
4474 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
4477 static int igb_tso(struct igb_ring *tx_ring,
4478 struct igb_tx_buffer *first,
4482 struct sk_buff *skb = first->skb;
4483 u32 vlan_macip_lens, type_tucmd;
4484 u32 mss_l4len_idx, l4len;
4486 if (!skb_is_gso(skb))
4487 #endif /* NETIF_F_TSO */
4491 if (skb_header_cloned(skb)) {
4492 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
4497 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4498 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
4500 if (first->protocol == __constant_htons(ETH_P_IP)) {
4501 struct iphdr *iph = ip_hdr(skb);
4504 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4508 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4509 first->tx_flags |= IGB_TX_FLAGS_TSO |
4513 } else if (skb_is_gso_v6(skb)) {
4514 ipv6_hdr(skb)->payload_len = 0;
4515 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4516 &ipv6_hdr(skb)->daddr,
4518 first->tx_flags |= IGB_TX_FLAGS_TSO |
4523 /* compute header lengths */
4524 l4len = tcp_hdrlen(skb);
4525 *hdr_len = skb_transport_offset(skb) + l4len;
4527 /* update gso size and bytecount with header size */
4528 first->gso_segs = skb_shinfo(skb)->gso_segs;
4529 first->bytecount += (first->gso_segs - 1) * *hdr_len;
4532 mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
4533 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
4535 /* VLAN MACLEN IPLEN */
4536 vlan_macip_lens = skb_network_header_len(skb);
4537 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4538 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4540 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4543 #endif /* NETIF_F_TSO */
4546 static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
4548 struct sk_buff *skb = first->skb;
4549 u32 vlan_macip_lens = 0;
4550 u32 mss_l4len_idx = 0;
4553 if (skb->ip_summed != CHECKSUM_PARTIAL) {
4554 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
4558 switch (first->protocol) {
4559 case __constant_htons(ETH_P_IP):
4560 vlan_macip_lens |= skb_network_header_len(skb);
4561 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4562 l4_hdr = ip_hdr(skb)->protocol;
4564 #ifdef NETIF_F_IPV6_CSUM
4565 case __constant_htons(ETH_P_IPV6):
4566 vlan_macip_lens |= skb_network_header_len(skb);
4567 l4_hdr = ipv6_hdr(skb)->nexthdr;
4571 if (unlikely(net_ratelimit())) {
4572 dev_warn(tx_ring->dev,
4573 "partial checksum but proto=%x!\n",
4581 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4582 mss_l4len_idx = tcp_hdrlen(skb) <<
4583 E1000_ADVTXD_L4LEN_SHIFT;
4587 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4588 mss_l4len_idx = sizeof(struct sctphdr) <<
4589 E1000_ADVTXD_L4LEN_SHIFT;
4593 mss_l4len_idx = sizeof(struct udphdr) <<
4594 E1000_ADVTXD_L4LEN_SHIFT;
4597 if (unlikely(net_ratelimit())) {
4598 dev_warn(tx_ring->dev,
4599 "partial checksum but l4 proto=%x!\n",
4605 /* update TX checksum flag */
4606 first->tx_flags |= IGB_TX_FLAGS_CSUM;
4609 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4610 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4612 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4615 static __le32 igb_tx_cmd_type(u32 tx_flags)
4617 /* set type for advanced descriptor with frame checksum insertion */
4618 __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA |
4619 E1000_ADVTXD_DCMD_IFCS |
4620 E1000_ADVTXD_DCMD_DEXT);
4622 /* set HW vlan bit if vlan is present */
4623 if (tx_flags & IGB_TX_FLAGS_VLAN)
4624 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE);
4626 /* set timestamp bit if present */
4627 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4628 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP);
4630 /* set segmentation bits for TSO */
4631 if (tx_flags & IGB_TX_FLAGS_TSO)
4632 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE);
4637 static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
4638 union e1000_adv_tx_desc *tx_desc,
4639 u32 tx_flags, unsigned int paylen)
4641 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
4643 /* 82575 requires a unique index per ring if any offload is enabled */
4644 if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) &&
4645 test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4646 olinfo_status |= tx_ring->reg_idx << 4;
4648 /* insert L4 checksum */
4649 if (tx_flags & IGB_TX_FLAGS_CSUM) {
4650 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4652 /* insert IPv4 checksum */
4653 if (tx_flags & IGB_TX_FLAGS_IPV4)
4654 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4657 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4661 * The largest size we can write to the descriptor is 65535. In order to
4662 * maintain a power of two alignment we have to limit ourselves to 32K.
4664 #define IGB_MAX_TXD_PWR 15
4665 #define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
4667 static void igb_tx_map(struct igb_ring *tx_ring,
4668 struct igb_tx_buffer *first,
4671 struct sk_buff *skb = first->skb;
4672 struct igb_tx_buffer *tx_buffer;
4673 union e1000_adv_tx_desc *tx_desc;
4675 #ifdef MAX_SKB_FRAGS
4676 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
4677 unsigned int data_len = skb->data_len;
4679 unsigned int size = skb_headlen(skb);
4680 unsigned int paylen = skb->len - hdr_len;
4682 u32 tx_flags = first->tx_flags;
4683 u16 i = tx_ring->next_to_use;
4685 tx_desc = IGB_TX_DESC(tx_ring, i);
4687 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen);
4688 cmd_type = igb_tx_cmd_type(tx_flags);
4690 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4691 if (dma_mapping_error(tx_ring->dev, dma))
4694 /* record length, and DMA address */
4695 dma_unmap_len_set(first, len, size);
4696 dma_unmap_addr_set(first, dma, dma);
4697 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4699 #ifdef MAX_SKB_FRAGS
4702 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
4703 tx_desc->read.cmd_type_len =
4704 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD);
4708 if (i == tx_ring->count) {
4709 tx_desc = IGB_TX_DESC(tx_ring, 0);
4713 dma += IGB_MAX_DATA_PER_TXD;
4714 size -= IGB_MAX_DATA_PER_TXD;
4716 tx_desc->read.olinfo_status = 0;
4717 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4720 #ifdef MAX_SKB_FRAGS
4721 if (likely(!data_len))
4724 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
4728 if (i == tx_ring->count) {
4729 tx_desc = IGB_TX_DESC(tx_ring, 0);
4733 size = skb_frag_size(frag);
4736 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
4738 if (dma_mapping_error(tx_ring->dev, dma))
4741 tx_buffer = &tx_ring->tx_buffer_info[i];
4742 dma_unmap_len_set(tx_buffer, len, size);
4743 dma_unmap_addr_set(tx_buffer, dma, dma);
4745 tx_desc->read.olinfo_status = 0;
4746 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4751 #endif /* MAX_SKB_FRAGS */
4753 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
4754 #endif /* CONFIG_BQL */
4756 /* write last descriptor with RS and EOP bits */
4757 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD);
4758 tx_desc->read.cmd_type_len = cmd_type;
4760 /* set the timestamp */
4761 first->time_stamp = jiffies;
4764 * Force memory writes to complete before letting h/w know there
4765 * are new descriptors to fetch. (Only applicable for weak-ordered
4766 * memory model archs, such as IA-64).
4768 * We also need this memory barrier to make certain all of the
4769 * status bits have been updated before next_to_watch is written.
4773 /* set next_to_watch value indicating a packet is present */
4774 first->next_to_watch = tx_desc;
4777 if (i == tx_ring->count)
4780 tx_ring->next_to_use = i;
4782 writel(i, tx_ring->tail);
4784 /* we need this if more than one processor can write to our tail
4785 * at a time, it syncronizes IO on IA64/Altix systems */
4791 dev_err(tx_ring->dev, "TX DMA map failed\n");
4793 /* clear dma mappings for failed tx_buffer_info map */
4795 tx_buffer= &tx_ring->tx_buffer_info[i];
4796 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer);
4797 if (tx_buffer == first)
4804 tx_ring->next_to_use = i;
4807 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4809 struct net_device *netdev = netdev_ring(tx_ring);
4811 if (netif_is_multiqueue(netdev))
4812 netif_stop_subqueue(netdev, ring_queue_index(tx_ring));
4814 netif_stop_queue(netdev);
4816 /* Herbert's original patch had:
4817 * smp_mb__after_netif_stop_queue();
4818 * but since that doesn't exist yet, just open code it. */
4821 /* We need to check again in a case another CPU has just
4822 * made room available. */
4823 if (igb_desc_unused(tx_ring) < size)
4827 if (netif_is_multiqueue(netdev))
4828 netif_wake_subqueue(netdev, ring_queue_index(tx_ring));
4830 netif_wake_queue(netdev);
4832 tx_ring->tx_stats.restart_queue++;
4837 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4839 if (igb_desc_unused(tx_ring) >= size)
4841 return __igb_maybe_stop_tx(tx_ring, size);
4844 netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4845 struct igb_ring *tx_ring)
4847 struct igb_tx_buffer *first;
4850 __be16 protocol = vlan_get_protocol(skb);
4853 /* need: 1 descriptor per page,
4854 * + 2 desc gap to keep tail from touching head,
4855 * + 1 desc for skb->data,
4856 * + 1 desc for context descriptor,
4857 * otherwise try next time */
4858 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
4859 /* this is a hard error */
4860 return NETDEV_TX_BUSY;
4863 /* record the location of the first descriptor for this packet */
4864 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4866 first->bytecount = skb->len;
4867 first->gso_segs = 1;
4869 #ifdef HAVE_HW_TIME_STAMP
4870 #ifdef SKB_SHARED_TX_IS_UNION
4871 if (unlikely(skb_shinfo(skb)->tx_flags.flags & SKBTX_HW_TSTAMP)) {
4872 skb_shinfo(skb)->tx_flags.flags |= SKBTX_IN_PROGRESS;
4873 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4876 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4877 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4878 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4883 if (vlan_tx_tag_present(skb)) {
4884 tx_flags |= IGB_TX_FLAGS_VLAN;
4885 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4888 /* record initial flags and protocol */
4889 first->tx_flags = tx_flags;
4890 first->protocol = protocol;
4892 tso = igb_tso(tx_ring, first, &hdr_len);
4896 igb_tx_csum(tx_ring, first);
4898 igb_tx_map(tx_ring, first, hdr_len);
4900 #ifndef HAVE_TRANS_START_IN_QUEUE
4901 netdev_ring(tx_ring)->trans_start = jiffies;
4904 /* Make sure there is space in the ring for the next send. */
4905 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
4907 return NETDEV_TX_OK;
4910 igb_unmap_and_free_tx_resource(tx_ring, first);
4912 return NETDEV_TX_OK;
4916 static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4917 struct sk_buff *skb)
4919 unsigned int r_idx = skb->queue_mapping;
4921 if (r_idx >= adapter->num_tx_queues)
4922 r_idx = r_idx % adapter->num_tx_queues;
4924 return adapter->tx_ring[r_idx];
4927 #define igb_tx_queue_mapping(_adapter, _skb) (_adapter)->tx_ring[0]
4930 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4931 struct net_device *netdev)
4933 struct igb_adapter *adapter = netdev_priv(netdev);
4935 if (test_bit(__IGB_DOWN, &adapter->state)) {
4936 dev_kfree_skb_any(skb);
4937 return NETDEV_TX_OK;
4940 if (skb->len <= 0) {
4941 dev_kfree_skb_any(skb);
4942 return NETDEV_TX_OK;
4946 * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4947 * in order to meet this minimum size requirement.
4949 if (skb->len < 17) {
4950 if (skb_padto(skb, 17))
4951 return NETDEV_TX_OK;
4955 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
4959 * igb_tx_timeout - Respond to a Tx Hang
4960 * @netdev: network interface device structure
4962 static void igb_tx_timeout(struct net_device *netdev)
4964 struct igb_adapter *adapter = netdev_priv(netdev);
4965 struct e1000_hw *hw = &adapter->hw;
4967 /* Do the reset outside of interrupt context */
4968 adapter->tx_timeout_count++;
4970 if (hw->mac.type >= e1000_82580)
4971 hw->dev_spec._82575.global_device_reset = true;
4973 schedule_work(&adapter->reset_task);
4974 E1000_WRITE_REG(hw, E1000_EICS,
4975 (adapter->eims_enable_mask & ~adapter->eims_other));
4978 static void igb_reset_task(struct work_struct *work)
4980 struct igb_adapter *adapter;
4981 adapter = container_of(work, struct igb_adapter, reset_task);
4983 igb_reinit_locked(adapter);
4987 * igb_get_stats - Get System Network Statistics
4988 * @netdev: network interface device structure
4990 * Returns the address of the device statistics structure.
4991 * The statistics are updated here and also from the timer callback.
4993 static struct net_device_stats *igb_get_stats(struct net_device *netdev)
4995 struct igb_adapter *adapter = netdev_priv(netdev);
4997 if (!test_bit(__IGB_RESETTING, &adapter->state))
4998 igb_update_stats(adapter);
5000 #ifdef HAVE_NETDEV_STATS_IN_NETDEV
5001 /* only return the current stats */
5002 return &netdev->stats;
5004 /* only return the current stats */
5005 return &adapter->net_stats;
5006 #endif /* HAVE_NETDEV_STATS_IN_NETDEV */
5010 * igb_change_mtu - Change the Maximum Transfer Unit
5011 * @netdev: network interface device structure
5012 * @new_mtu: new value for maximum frame size
5014 * Returns 0 on success, negative on failure
5016 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
5018 struct igb_adapter *adapter = netdev_priv(netdev);
5019 struct pci_dev *pdev = adapter->pdev;
5020 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
5021 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
5022 u32 rx_buffer_len, i;
5025 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
5026 dev_err(pci_dev_to_dev(pdev), "Invalid MTU setting\n");
5030 #define MAX_STD_JUMBO_FRAME_SIZE 9238
5031 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
5032 dev_err(pci_dev_to_dev(pdev), "MTU > 9216 not supported.\n");
5036 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
5037 usleep_range(1000, 2000);
5039 /* igb_down has a dependency on max_frame_size */
5040 adapter->max_frame_size = max_frame;
5042 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
5043 #ifdef IGB_PER_PKT_TIMESTAMP
5044 if (adapter->hw.mac.type >= e1000_82580)
5045 max_frame += IGB_TS_HDR_LEN;
5049 * RLPML prevents us from receiving a frame larger than max_frame so
5050 * it is safe to just set the rx_buffer_len to max_frame without the
5051 * risk of an skb over panic.
5053 if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
5054 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
5056 rx_buffer_len = max_frame;
5059 if (netif_running(netdev))
5062 dev_info(pci_dev_to_dev(pdev), "changing MTU from %d to %d\n",
5063 netdev->mtu, new_mtu);
5064 netdev->mtu = new_mtu;
5066 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
5067 for (i = 0; i < adapter->num_rx_queues; i++)
5068 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
5071 if (netif_running(netdev))
5076 clear_bit(__IGB_RESETTING, &adapter->state);
5082 * igb_update_stats - Update the board statistics counters
5083 * @adapter: board private structure
5086 void igb_update_stats(struct igb_adapter *adapter)
5088 #ifdef HAVE_NETDEV_STATS_IN_NETDEV
5089 struct net_device_stats *net_stats = &adapter->netdev->stats;
5091 struct net_device_stats *net_stats = &adapter->net_stats;
5092 #endif /* HAVE_NETDEV_STATS_IN_NETDEV */
5093 struct e1000_hw *hw = &adapter->hw;
5095 struct pci_dev *pdev = adapter->pdev;
5102 u32 flushed = 0, coal = 0, recycled = 0;
5103 struct igb_q_vector *q_vector;
5106 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
5109 * Prevent stats update while adapter is being reset, or if the pci
5110 * connection is down.
5112 if (adapter->link_speed == 0)
5115 if (pci_channel_offline(pdev))
5120 for (i = 0; i < adapter->num_q_vectors; i++) {
5121 q_vector = adapter->q_vector[i];
5122 if (!q_vector || !q_vector->lrolist)
5124 flushed += q_vector->lrolist->stats.flushed;
5125 coal += q_vector->lrolist->stats.coal;
5126 recycled += q_vector->lrolist->stats.recycled;
5128 adapter->lro_stats.flushed = flushed;
5129 adapter->lro_stats.coal = coal;
5130 adapter->lro_stats.recycled = recycled;
5135 for (i = 0; i < adapter->num_rx_queues; i++) {
5136 u32 rqdpc_tmp = E1000_READ_REG(hw, E1000_RQDPC(i)) & 0x0FFF;
5137 struct igb_ring *ring = adapter->rx_ring[i];
5138 ring->rx_stats.drops += rqdpc_tmp;
5139 net_stats->rx_fifo_errors += rqdpc_tmp;
5140 #ifdef CONFIG_IGB_VMDQ_NETDEV
5141 if (!ring->vmdq_netdev) {
5142 bytes += ring->rx_stats.bytes;
5143 packets += ring->rx_stats.packets;
5146 bytes += ring->rx_stats.bytes;
5147 packets += ring->rx_stats.packets;
5151 net_stats->rx_bytes = bytes;
5152 net_stats->rx_packets = packets;
5156 for (i = 0; i < adapter->num_tx_queues; i++) {
5157 struct igb_ring *ring = adapter->tx_ring[i];
5158 #ifdef CONFIG_IGB_VMDQ_NETDEV
5159 if (!ring->vmdq_netdev) {
5160 bytes += ring->tx_stats.bytes;
5161 packets += ring->tx_stats.packets;
5164 bytes += ring->tx_stats.bytes;
5165 packets += ring->tx_stats.packets;
5168 net_stats->tx_bytes = bytes;
5169 net_stats->tx_packets = packets;
5171 /* read stats registers */
5172 adapter->stats.crcerrs += E1000_READ_REG(hw, E1000_CRCERRS);
5173 adapter->stats.gprc += E1000_READ_REG(hw, E1000_GPRC);
5174 adapter->stats.gorc += E1000_READ_REG(hw, E1000_GORCL);
5175 E1000_READ_REG(hw, E1000_GORCH); /* clear GORCL */
5176 adapter->stats.bprc += E1000_READ_REG(hw, E1000_BPRC);
5177 adapter->stats.mprc += E1000_READ_REG(hw, E1000_MPRC);
5178 adapter->stats.roc += E1000_READ_REG(hw, E1000_ROC);
5180 adapter->stats.prc64 += E1000_READ_REG(hw, E1000_PRC64);
5181 adapter->stats.prc127 += E1000_READ_REG(hw, E1000_PRC127);
5182 adapter->stats.prc255 += E1000_READ_REG(hw, E1000_PRC255);
5183 adapter->stats.prc511 += E1000_READ_REG(hw, E1000_PRC511);
5184 adapter->stats.prc1023 += E1000_READ_REG(hw, E1000_PRC1023);
5185 adapter->stats.prc1522 += E1000_READ_REG(hw, E1000_PRC1522);
5186 adapter->stats.symerrs += E1000_READ_REG(hw, E1000_SYMERRS);
5187 adapter->stats.sec += E1000_READ_REG(hw, E1000_SEC);
5189 mpc = E1000_READ_REG(hw, E1000_MPC);
5190 adapter->stats.mpc += mpc;
5191 net_stats->rx_fifo_errors += mpc;
5192 adapter->stats.scc += E1000_READ_REG(hw, E1000_SCC);
5193 adapter->stats.ecol += E1000_READ_REG(hw, E1000_ECOL);
5194 adapter->stats.mcc += E1000_READ_REG(hw, E1000_MCC);
5195 adapter->stats.latecol += E1000_READ_REG(hw, E1000_LATECOL);
5196 adapter->stats.dc += E1000_READ_REG(hw, E1000_DC);
5197 adapter->stats.rlec += E1000_READ_REG(hw, E1000_RLEC);
5198 adapter->stats.xonrxc += E1000_READ_REG(hw, E1000_XONRXC);
5199 adapter->stats.xontxc += E1000_READ_REG(hw, E1000_XONTXC);
5200 adapter->stats.xoffrxc += E1000_READ_REG(hw, E1000_XOFFRXC);
5201 adapter->stats.xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC);
5202 adapter->stats.fcruc += E1000_READ_REG(hw, E1000_FCRUC);
5203 adapter->stats.gptc += E1000_READ_REG(hw, E1000_GPTC);
5204 adapter->stats.gotc += E1000_READ_REG(hw, E1000_GOTCL);
5205 E1000_READ_REG(hw, E1000_GOTCH); /* clear GOTCL */
5206 adapter->stats.rnbc += E1000_READ_REG(hw, E1000_RNBC);
5207 adapter->stats.ruc += E1000_READ_REG(hw, E1000_RUC);
5208 adapter->stats.rfc += E1000_READ_REG(hw, E1000_RFC);
5209 adapter->stats.rjc += E1000_READ_REG(hw, E1000_RJC);
5210 adapter->stats.tor += E1000_READ_REG(hw, E1000_TORH);
5211 adapter->stats.tot += E1000_READ_REG(hw, E1000_TOTH);
5212 adapter->stats.tpr += E1000_READ_REG(hw, E1000_TPR);
5214 adapter->stats.ptc64 += E1000_READ_REG(hw, E1000_PTC64);
5215 adapter->stats.ptc127 += E1000_READ_REG(hw, E1000_PTC127);
5216 adapter->stats.ptc255 += E1000_READ_REG(hw, E1000_PTC255);
5217 adapter->stats.ptc511 += E1000_READ_REG(hw, E1000_PTC511);
5218 adapter->stats.ptc1023 += E1000_READ_REG(hw, E1000_PTC1023);
5219 adapter->stats.ptc1522 += E1000_READ_REG(hw, E1000_PTC1522);
5221 adapter->stats.mptc += E1000_READ_REG(hw, E1000_MPTC);
5222 adapter->stats.bptc += E1000_READ_REG(hw, E1000_BPTC);
5224 adapter->stats.tpt += E1000_READ_REG(hw, E1000_TPT);
5225 adapter->stats.colc += E1000_READ_REG(hw, E1000_COLC);
5227 adapter->stats.algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC);
5228 /* read internal phy sepecific stats */
5229 reg = E1000_READ_REG(hw, E1000_CTRL_EXT);
5230 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
5231 adapter->stats.rxerrc += E1000_READ_REG(hw, E1000_RXERRC);
5232 adapter->stats.tncrs += E1000_READ_REG(hw, E1000_TNCRS);
5235 adapter->stats.tsctc += E1000_READ_REG(hw, E1000_TSCTC);
5236 adapter->stats.tsctfc += E1000_READ_REG(hw, E1000_TSCTFC);
5238 adapter->stats.iac += E1000_READ_REG(hw, E1000_IAC);
5239 adapter->stats.icrxoc += E1000_READ_REG(hw, E1000_ICRXOC);
5240 adapter->stats.icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC);
5241 adapter->stats.icrxatc += E1000_READ_REG(hw, E1000_ICRXATC);
5242 adapter->stats.ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC);
5243 adapter->stats.ictxatc += E1000_READ_REG(hw, E1000_ICTXATC);
5244 adapter->stats.ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC);
5245 adapter->stats.ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC);
5246 adapter->stats.icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC);
5248 /* Fill out the OS statistics structure */
5249 net_stats->multicast = adapter->stats.mprc;
5250 net_stats->collisions = adapter->stats.colc;
5254 /* RLEC on some newer hardware can be incorrect so build
5255 * our own version based on RUC and ROC */
5256 net_stats->rx_errors = adapter->stats.rxerrc +
5257 adapter->stats.crcerrs + adapter->stats.algnerrc +
5258 adapter->stats.ruc + adapter->stats.roc +
5259 adapter->stats.cexterr;
5260 net_stats->rx_length_errors = adapter->stats.ruc +
5262 net_stats->rx_crc_errors = adapter->stats.crcerrs;
5263 net_stats->rx_frame_errors = adapter->stats.algnerrc;
5264 net_stats->rx_missed_errors = adapter->stats.mpc;
5267 net_stats->tx_errors = adapter->stats.ecol +
5268 adapter->stats.latecol;
5269 net_stats->tx_aborted_errors = adapter->stats.ecol;
5270 net_stats->tx_window_errors = adapter->stats.latecol;
5271 net_stats->tx_carrier_errors = adapter->stats.tncrs;
5273 /* Tx Dropped needs to be maintained elsewhere */
5276 if (hw->phy.media_type == e1000_media_type_copper) {
5277 if ((adapter->link_speed == SPEED_1000) &&
5278 (!e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
5279 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
5280 adapter->phy_stats.idle_errors += phy_tmp;
5284 /* Management Stats */
5285 adapter->stats.mgptc += E1000_READ_REG(hw, E1000_MGTPTC);
5286 adapter->stats.mgprc += E1000_READ_REG(hw, E1000_MGTPRC);
5287 if (hw->mac.type > e1000_82580) {
5288 adapter->stats.o2bgptc += E1000_READ_REG(hw, E1000_O2BGPTC);
5289 adapter->stats.o2bspc += E1000_READ_REG(hw, E1000_O2BSPC);
5290 adapter->stats.b2ospc += E1000_READ_REG(hw, E1000_B2OSPC);
5291 adapter->stats.b2ogprc += E1000_READ_REG(hw, E1000_B2OGPRC);
5295 static irqreturn_t igb_msix_other(int irq, void *data)
5297 struct igb_adapter *adapter = data;
5298 struct e1000_hw *hw = &adapter->hw;
5299 u32 icr = E1000_READ_REG(hw, E1000_ICR);
5300 /* reading ICR causes bit 31 of EICR to be cleared */
5302 if (icr & E1000_ICR_DRSTA)
5303 schedule_work(&adapter->reset_task);
5305 if (icr & E1000_ICR_DOUTSYNC) {
5306 /* HW is reporting DMA is out of sync */
5307 adapter->stats.doosync++;
5308 /* The DMA Out of Sync is also indication of a spoof event
5309 * in IOV mode. Check the Wrong VM Behavior register to
5310 * see if it is really a spoof event. */
5311 igb_check_wvbr(adapter);
5314 /* Check for a mailbox event */
5315 if (icr & E1000_ICR_VMMB)
5316 igb_msg_task(adapter);
5318 if (icr & E1000_ICR_LSC) {
5319 hw->mac.get_link_status = 1;
5320 /* guard against interrupt when we're going down */
5321 if (!test_bit(__IGB_DOWN, &adapter->state))
5322 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5325 /* Check for MDD event */
5326 if (icr & E1000_ICR_MDDET)
5327 igb_process_mdd_event(adapter);
5329 E1000_WRITE_REG(hw, E1000_EIMS, adapter->eims_other);
5334 static void igb_write_itr(struct igb_q_vector *q_vector)
5336 struct igb_adapter *adapter = q_vector->adapter;
5337 u32 itr_val = q_vector->itr_val & 0x7FFC;
5339 if (!q_vector->set_itr)
5345 if (adapter->hw.mac.type == e1000_82575)
5346 itr_val |= itr_val << 16;
5348 itr_val |= E1000_EITR_CNT_IGNR;
5350 writel(itr_val, q_vector->itr_register);
5351 q_vector->set_itr = 0;
5354 static irqreturn_t igb_msix_ring(int irq, void *data)
5356 struct igb_q_vector *q_vector = data;
5358 /* Write the ITR value calculated from the previous interrupt. */
5359 igb_write_itr(q_vector);
5361 napi_schedule(&q_vector->napi);
5367 static void igb_update_dca(struct igb_q_vector *q_vector)
5369 struct igb_adapter *adapter = q_vector->adapter;
5370 struct e1000_hw *hw = &adapter->hw;
5371 int cpu = get_cpu();
5373 if (q_vector->cpu == cpu)
5376 if (q_vector->tx.ring) {
5377 int q = q_vector->tx.ring->reg_idx;
5378 u32 dca_txctrl = E1000_READ_REG(hw, E1000_DCA_TXCTRL(q));
5379 if (hw->mac.type == e1000_82575) {
5380 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
5381 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
5383 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
5384 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
5385 E1000_DCA_TXCTRL_CPUID_SHIFT_82576;
5387 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
5388 E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(q), dca_txctrl);
5390 if (q_vector->rx.ring) {
5391 int q = q_vector->rx.ring->reg_idx;
5392 u32 dca_rxctrl = E1000_READ_REG(hw, E1000_DCA_RXCTRL(q));
5393 if (hw->mac.type == e1000_82575) {
5394 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
5395 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
5397 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
5398 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
5399 E1000_DCA_RXCTRL_CPUID_SHIFT_82576;
5401 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
5402 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
5403 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
5404 E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(q), dca_rxctrl);
5406 q_vector->cpu = cpu;
5411 static void igb_setup_dca(struct igb_adapter *adapter)
5413 struct e1000_hw *hw = &adapter->hw;
5416 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
5419 /* Always use CB2 mode, difference is masked in the CB driver. */
5420 E1000_WRITE_REG(hw, E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
5422 for (i = 0; i < adapter->num_q_vectors; i++) {
5423 adapter->q_vector[i]->cpu = -1;
5424 igb_update_dca(adapter->q_vector[i]);
5428 static int __igb_notify_dca(struct device *dev, void *data)
5430 struct net_device *netdev = dev_get_drvdata(dev);
5431 struct igb_adapter *adapter = netdev_priv(netdev);
5432 struct pci_dev *pdev = adapter->pdev;
5433 struct e1000_hw *hw = &adapter->hw;
5434 unsigned long event = *(unsigned long *)data;
5437 case DCA_PROVIDER_ADD:
5438 /* if already enabled, don't do it again */
5439 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
5441 if (dca_add_requester(dev) == E1000_SUCCESS) {
5442 adapter->flags |= IGB_FLAG_DCA_ENABLED;
5443 dev_info(pci_dev_to_dev(pdev), "DCA enabled\n");
5444 igb_setup_dca(adapter);
5447 /* Fall Through since DCA is disabled. */
5448 case DCA_PROVIDER_REMOVE:
5449 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
5450 /* without this a class_device is left
5451 * hanging around in the sysfs model */
5452 dca_remove_requester(dev);
5453 dev_info(pci_dev_to_dev(pdev), "DCA disabled\n");
5454 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
5455 E1000_WRITE_REG(hw, E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_DISABLE);
5460 return E1000_SUCCESS;
5463 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
5468 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
5471 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
5473 #endif /* IGB_DCA */
5475 static int igb_vf_configure(struct igb_adapter *adapter, int vf)
5477 unsigned char mac_addr[ETH_ALEN];
5478 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
5479 struct pci_dev *pdev = adapter->pdev;
5480 struct e1000_hw *hw = &adapter->hw;
5481 struct pci_dev *pvfdev;
5482 unsigned int device_id;
5486 random_ether_addr(mac_addr);
5487 igb_set_vf_mac(adapter, vf, mac_addr);
5489 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
5490 switch (adapter->hw.mac.type) {
5492 device_id = IGB_82576_VF_DEV_ID;
5493 /* VF Stride for 82576 is 2 */
5494 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 1)) |
5498 device_id = IGB_I350_VF_DEV_ID;
5499 /* VF Stride for I350 is 4 */
5500 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 2)) |
5509 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5511 if (pvfdev->devfn == thisvf_devfn)
5513 pvfdev = pci_get_device(hw->vendor_id,
5518 adapter->vf_data[vf].vfdev = pvfdev;
5521 "Couldn't find pci dev ptr for VF %4.4x\n",
5523 return pvfdev != NULL;
5529 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
5530 static int igb_find_enabled_vfs(struct igb_adapter *adapter)
5532 struct e1000_hw *hw = &adapter->hw;
5533 struct pci_dev *pdev = adapter->pdev;
5534 struct pci_dev *pvfdev;
5537 unsigned int device_id;
5540 switch (adapter->hw.mac.type) {
5542 device_id = IGB_82576_VF_DEV_ID;
5543 /* VF Stride for 82576 is 2 */
5547 device_id = IGB_I350_VF_DEV_ID;
5548 /* VF Stride for I350 is 4 */
5557 vf_devfn = pdev->devfn + 0x80;
5558 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
5560 if (pvfdev->devfn == vf_devfn)
5562 vf_devfn += vf_stride;
5563 pvfdev = pci_get_device(hw->vendor_id,
5571 static int igb_check_vf_assignment(struct igb_adapter *adapter)
5573 #ifdef HAVE_PCI_DEV_FLAGS_ASSIGNED
5575 for (i = 0; i < adapter->vfs_allocated_count; i++) {
5576 if (adapter->vf_data[i].vfdev) {
5577 if (adapter->vf_data[i].vfdev->dev_flags &
5578 PCI_DEV_FLAGS_ASSIGNED)
5586 static void igb_ping_all_vfs(struct igb_adapter *adapter)
5588 struct e1000_hw *hw = &adapter->hw;
5592 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
5593 ping = E1000_PF_CONTROL_MSG;
5594 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
5595 ping |= E1000_VT_MSGTYPE_CTS;
5596 e1000_write_mbx(hw, &ping, 1, i);
5601 * igb_mta_set_ - Set multicast filter table address
5602 * @adapter: pointer to the adapter structure
5603 * @hash_value: determines the MTA register and bit to set
5605 * The multicast table address is a register array of 32-bit registers.
5606 * The hash_value is used to determine what register the bit is in, the
5607 * current value is read, the new bit is OR'd in and the new value is
5608 * written back into the register.
5610 void igb_mta_set(struct igb_adapter *adapter, u32 hash_value)
5612 struct e1000_hw *hw = &adapter->hw;
5613 u32 hash_bit, hash_reg, mta;
5616 * The MTA is a register array of 32-bit registers. It is
5617 * treated like an array of (32*mta_reg_count) bits. We want to
5618 * set bit BitArray[hash_value]. So we figure out what register
5619 * the bit is in, read it, OR in the new bit, then write
5620 * back the new value. The (hw->mac.mta_reg_count - 1) serves as a
5621 * mask to bits 31:5 of the hash value which gives us the
5622 * register we're modifying. The hash bit within that register
5623 * is determined by the lower 5 bits of the hash value.
5625 hash_reg = (hash_value >> 5) & (hw->mac.mta_reg_count - 1);
5626 hash_bit = hash_value & 0x1F;
5628 mta = E1000_READ_REG_ARRAY(hw, E1000_MTA, hash_reg);
5630 mta |= (1 << hash_bit);
5632 E1000_WRITE_REG_ARRAY(hw, E1000_MTA, hash_reg, mta);
5633 E1000_WRITE_FLUSH(hw);
5636 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5639 struct e1000_hw *hw = &adapter->hw;
5640 u32 vmolr = E1000_READ_REG(hw, E1000_VMOLR(vf));
5641 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5643 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
5644 IGB_VF_FLAG_MULTI_PROMISC);
5645 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5647 #ifdef IGB_ENABLE_VF_PROMISC
5648 if (*msgbuf & E1000_VF_SET_PROMISC_UNICAST) {
5649 vmolr |= E1000_VMOLR_ROPE;
5650 vf_data->flags |= IGB_VF_FLAG_UNI_PROMISC;
5651 *msgbuf &= ~E1000_VF_SET_PROMISC_UNICAST;
5654 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
5655 vmolr |= E1000_VMOLR_MPME;
5656 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
5657 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
5660 * if we have hashes and we are clearing a multicast promisc
5661 * flag we need to write the hashes to the MTA as this step
5662 * was previously skipped
5664 if (vf_data->num_vf_mc_hashes > 30) {
5665 vmolr |= E1000_VMOLR_MPME;
5666 } else if (vf_data->num_vf_mc_hashes) {
5668 vmolr |= E1000_VMOLR_ROMPE;
5669 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5670 igb_mta_set(adapter, vf_data->vf_mc_hashes[j]);
5674 E1000_WRITE_REG(hw, E1000_VMOLR(vf), vmolr);
5676 /* there are flags left unprocessed, likely not supported */
5677 if (*msgbuf & E1000_VT_MSGINFO_MASK)
5684 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
5685 u32 *msgbuf, u32 vf)
5687 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5688 u16 *hash_list = (u16 *)&msgbuf[1];
5689 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5692 /* salt away the number of multicast addresses assigned
5693 * to this VF for later use to restore when the PF multi cast
5696 vf_data->num_vf_mc_hashes = n;
5698 /* only up to 30 hash values supported */
5702 /* store the hashes for later use */
5703 for (i = 0; i < n; i++)
5704 vf_data->vf_mc_hashes[i] = hash_list[i];
5706 /* Flush and reset the mta with the new values */
5707 igb_set_rx_mode(adapter->netdev);
5712 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
5714 struct e1000_hw *hw = &adapter->hw;
5715 struct vf_data_storage *vf_data;
5718 for (i = 0; i < adapter->vfs_allocated_count; i++) {
5719 u32 vmolr = E1000_READ_REG(hw, E1000_VMOLR(i));
5720 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5722 vf_data = &adapter->vf_data[i];
5724 if ((vf_data->num_vf_mc_hashes > 30) ||
5725 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5726 vmolr |= E1000_VMOLR_MPME;
5727 } else if (vf_data->num_vf_mc_hashes) {
5728 vmolr |= E1000_VMOLR_ROMPE;
5729 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5730 igb_mta_set(adapter, vf_data->vf_mc_hashes[j]);
5732 E1000_WRITE_REG(hw, E1000_VMOLR(i), vmolr);
5736 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5738 struct e1000_hw *hw = &adapter->hw;
5739 u32 pool_mask, reg, vid;
5743 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5745 /* Find the vlan filter for this id */
5746 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5747 reg = E1000_READ_REG(hw, E1000_VLVF(i));
5749 /* remove the vf from the pool */
5752 /* if pool is empty then remove entry from vfta */
5753 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5754 (reg & E1000_VLVF_VLANID_ENABLE)) {
5756 vid = reg & E1000_VLVF_VLANID_MASK;
5757 igb_vfta_set(adapter, vid, FALSE);
5760 E1000_WRITE_REG(hw, E1000_VLVF(i), reg);
5763 adapter->vf_data[vf].vlans_enabled = 0;
5765 vlan_default = adapter->vf_data[vf].default_vf_vlan_id;
5767 igb_vlvf_set(adapter, vlan_default, true, vf);
5770 s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5772 struct e1000_hw *hw = &adapter->hw;
5775 /* The vlvf table only exists on 82576 hardware and newer */
5776 if (hw->mac.type < e1000_82576)
5779 /* we only need to do this if VMDq is enabled */
5780 if (!adapter->vmdq_pools)
5783 /* Find the vlan filter for this id */
5784 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5785 reg = E1000_READ_REG(hw, E1000_VLVF(i));
5786 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5787 vid == (reg & E1000_VLVF_VLANID_MASK))
5792 if (i == E1000_VLVF_ARRAY_SIZE) {
5793 /* Did not find a matching VLAN ID entry that was
5794 * enabled. Search for a free filter entry, i.e.
5795 * one without the enable bit set
5797 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5798 reg = E1000_READ_REG(hw, E1000_VLVF(i));
5799 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5803 if (i < E1000_VLVF_ARRAY_SIZE) {
5804 /* Found an enabled/available entry */
5805 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5807 /* if !enabled we need to set this up in vfta */
5808 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
5809 /* add VID to filter table */
5810 igb_vfta_set(adapter, vid, TRUE);
5811 reg |= E1000_VLVF_VLANID_ENABLE;
5813 reg &= ~E1000_VLVF_VLANID_MASK;
5815 E1000_WRITE_REG(hw, E1000_VLVF(i), reg);
5817 /* do not modify RLPML for PF devices */
5818 if (vf >= adapter->vfs_allocated_count)
5819 return E1000_SUCCESS;
5821 if (!adapter->vf_data[vf].vlans_enabled) {
5823 reg = E1000_READ_REG(hw, E1000_VMOLR(vf));
5824 size = reg & E1000_VMOLR_RLPML_MASK;
5826 reg &= ~E1000_VMOLR_RLPML_MASK;
5828 E1000_WRITE_REG(hw, E1000_VMOLR(vf), reg);
5831 adapter->vf_data[vf].vlans_enabled++;
5834 if (i < E1000_VLVF_ARRAY_SIZE) {
5835 /* remove vf from the pool */
5836 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5837 /* if pool is empty then remove entry from vfta */
5838 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5840 igb_vfta_set(adapter, vid, FALSE);
5842 E1000_WRITE_REG(hw, E1000_VLVF(i), reg);
5844 /* do not modify RLPML for PF devices */
5845 if (vf >= adapter->vfs_allocated_count)
5846 return E1000_SUCCESS;
5848 adapter->vf_data[vf].vlans_enabled--;
5849 if (!adapter->vf_data[vf].vlans_enabled) {
5851 reg = E1000_READ_REG(hw, E1000_VMOLR(vf));
5852 size = reg & E1000_VMOLR_RLPML_MASK;
5854 reg &= ~E1000_VMOLR_RLPML_MASK;
5856 E1000_WRITE_REG(hw, E1000_VMOLR(vf), reg);
5860 return E1000_SUCCESS;
5864 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5866 struct e1000_hw *hw = &adapter->hw;
5869 E1000_WRITE_REG(hw, E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5871 E1000_WRITE_REG(hw, E1000_VMVIR(vf), 0);
5874 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5875 int vf, u16 vlan, u8 qos)
5878 struct igb_adapter *adapter = netdev_priv(netdev);
5880 /* VLAN IDs accepted range 0-4094 */
5881 if ((vf >= adapter->vfs_allocated_count) || (vlan > VLAN_VID_MASK-1) || (qos > 7))
5884 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5887 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5888 igb_set_vmolr(adapter, vf, !vlan);
5889 adapter->vf_data[vf].pf_vlan = vlan;
5890 adapter->vf_data[vf].pf_qos = qos;
5891 igb_set_vf_vlan_strip(adapter, vf, true);
5892 dev_info(&adapter->pdev->dev,
5893 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5894 if (test_bit(__IGB_DOWN, &adapter->state)) {
5895 dev_warn(&adapter->pdev->dev,
5896 "The VF VLAN has been set,"
5897 " but the PF device is not up.\n");
5898 dev_warn(&adapter->pdev->dev,
5899 "Bring the PF device up before"
5900 " attempting to use the VF device.\n");
5903 if (adapter->vf_data[vf].pf_vlan)
5904 dev_info(&adapter->pdev->dev,
5905 "Clearing VLAN on VF %d\n", vf);
5906 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5908 igb_set_vmvir(adapter, vlan, vf);
5909 igb_set_vmolr(adapter, vf, true);
5910 igb_set_vf_vlan_strip(adapter, vf, false);
5911 adapter->vf_data[vf].pf_vlan = 0;
5912 adapter->vf_data[vf].pf_qos = 0;
5919 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5921 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5922 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5925 igb_set_vf_vlan_strip(adapter, vf, true);
5927 igb_set_vf_vlan_strip(adapter, vf, false);
5929 return igb_vlvf_set(adapter, vid, add, vf);
5932 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
5934 struct e1000_hw *hw = &adapter->hw;
5936 /* clear flags except flag that the PF has set the MAC */
5937 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
5938 adapter->vf_data[vf].last_nack = jiffies;
5940 /* reset offloads to defaults */
5941 igb_set_vmolr(adapter, vf, true);
5943 /* reset vlans for device */
5944 igb_clear_vf_vfta(adapter, vf);
5946 if (adapter->vf_data[vf].pf_vlan)
5947 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5948 adapter->vf_data[vf].pf_vlan,
5949 adapter->vf_data[vf].pf_qos);
5951 igb_clear_vf_vfta(adapter, vf);
5954 /* reset multicast table array for vf */
5955 adapter->vf_data[vf].num_vf_mc_hashes = 0;
5957 /* Flush and reset the mta with the new values */
5958 igb_set_rx_mode(adapter->netdev);
5961 * Reset the VFs TDWBAL and TDWBAH registers which are not
5964 E1000_WRITE_REG(hw, E1000_TDWBAH(vf), 0);
5965 E1000_WRITE_REG(hw, E1000_TDWBAL(vf), 0);
5966 if (hw->mac.type == e1000_82576) {
5967 E1000_WRITE_REG(hw, E1000_TDWBAH(IGB_MAX_VF_FUNCTIONS + vf), 0);
5968 E1000_WRITE_REG(hw, E1000_TDWBAL(IGB_MAX_VF_FUNCTIONS + vf), 0);
5972 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5974 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5976 /* generate a new mac address as we were hotplug removed/added */
5977 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5978 random_ether_addr(vf_mac);
5980 /* process remaining reset events */
5981 igb_vf_reset(adapter, vf);
5984 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
5986 struct e1000_hw *hw = &adapter->hw;
5987 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5989 u8 *addr = (u8 *)(&msgbuf[1]);
5991 /* process all the same items cleared in a function level reset */
5992 igb_vf_reset(adapter, vf);
5994 /* set vf mac address */
5995 igb_del_mac_filter(adapter, vf_mac, vf);
5996 igb_add_mac_filter(adapter, vf_mac, vf);
5998 /* enable transmit and receive for vf */
5999 reg = E1000_READ_REG(hw, E1000_VFTE);
6000 E1000_WRITE_REG(hw, E1000_VFTE, reg | (1 << vf));
6001 reg = E1000_READ_REG(hw, E1000_VFRE);
6002 E1000_WRITE_REG(hw, E1000_VFRE, reg | (1 << vf));
6004 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
6006 /* reply to reset with ack and vf mac address */
6007 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
6008 memcpy(addr, vf_mac, 6);
6009 e1000_write_mbx(hw, msgbuf, 3, vf);
6012 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
6015 * The VF MAC Address is stored in a packed array of bytes
6016 * starting at the second 32 bit word of the msg array
6018 unsigned char *addr = (unsigned char *)&msg[1];
6021 if (is_valid_ether_addr(addr))
6022 err = igb_set_vf_mac(adapter, vf, addr);
6027 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
6029 struct e1000_hw *hw = &adapter->hw;
6030 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6031 u32 msg = E1000_VT_MSGTYPE_NACK;
6033 /* if device isn't clear to send it shouldn't be reading either */
6034 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
6035 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
6036 e1000_write_mbx(hw, &msg, 1, vf);
6037 vf_data->last_nack = jiffies;
6041 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
6043 struct pci_dev *pdev = adapter->pdev;
6044 u32 msgbuf[E1000_VFMAILBOX_SIZE];
6045 struct e1000_hw *hw = &adapter->hw;
6046 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6049 retval = e1000_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
6052 dev_err(pci_dev_to_dev(pdev), "Error receiving message from VF\n");
6056 /* this is a message we already processed, do nothing */
6057 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
6061 * until the vf completes a reset it should not be
6062 * allowed to start any configuration.
6065 if (msgbuf[0] == E1000_VF_RESET) {
6066 igb_vf_reset_msg(adapter, vf);
6070 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
6071 msgbuf[0] = E1000_VT_MSGTYPE_NACK;
6072 if (time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
6073 e1000_write_mbx(hw, msgbuf, 1, vf);
6074 vf_data->last_nack = jiffies;
6079 switch ((msgbuf[0] & 0xFFFF)) {
6080 case E1000_VF_SET_MAC_ADDR:
6082 #ifndef IGB_DISABLE_VF_MAC_SET
6083 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
6084 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
6087 "VF %d attempted to override administratively "
6088 "set MAC address\nReload the VF driver to "
6089 "resume operations\n", vf);
6092 case E1000_VF_SET_PROMISC:
6093 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
6095 case E1000_VF_SET_MULTICAST:
6096 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
6098 case E1000_VF_SET_LPE:
6099 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
6101 case E1000_VF_SET_VLAN:
6104 if (vf_data->pf_vlan)
6106 "VF %d attempted to override administratively "
6107 "set VLAN tag\nReload the VF driver to "
6108 "resume operations\n", vf);
6111 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
6114 dev_err(pci_dev_to_dev(pdev), "Unhandled Msg %08x\n", msgbuf[0]);
6115 retval = -E1000_ERR_MBX;
6119 /* notify the VF of the results of what it sent us */
6121 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
6123 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
6125 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
6127 e1000_write_mbx(hw, msgbuf, 1, vf);
6130 static void igb_msg_task(struct igb_adapter *adapter)
6132 struct e1000_hw *hw = &adapter->hw;
6135 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
6136 /* process any reset requests */
6137 if (!e1000_check_for_rst(hw, vf))
6138 igb_vf_reset_event(adapter, vf);
6140 /* process any messages pending */
6141 if (!e1000_check_for_msg(hw, vf))
6142 igb_rcv_msg_from_vf(adapter, vf);
6144 /* process any acks */
6145 if (!e1000_check_for_ack(hw, vf))
6146 igb_rcv_ack_from_vf(adapter, vf);
6151 * igb_set_uta - Set unicast filter table address
6152 * @adapter: board private structure
6154 * The unicast table address is a register array of 32-bit registers.
6155 * The table is meant to be used in a way similar to how the MTA is used
6156 * however due to certain limitations in the hardware it is necessary to
6157 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
6158 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
6160 static void igb_set_uta(struct igb_adapter *adapter)
6162 struct e1000_hw *hw = &adapter->hw;
6165 /* The UTA table only exists on 82576 hardware and newer */
6166 if (hw->mac.type < e1000_82576)
6169 /* we only need to do this if VMDq is enabled */
6170 if (!adapter->vmdq_pools)
6173 for (i = 0; i < hw->mac.uta_reg_count; i++)
6174 E1000_WRITE_REG_ARRAY(hw, E1000_UTA, i, ~0);
6178 * igb_intr_msi - Interrupt Handler
6179 * @irq: interrupt number
6180 * @data: pointer to a network interface device structure
6182 static irqreturn_t igb_intr_msi(int irq, void *data)
6184 struct igb_adapter *adapter = data;
6185 struct igb_q_vector *q_vector = adapter->q_vector[0];
6186 struct e1000_hw *hw = &adapter->hw;
6187 /* read ICR disables interrupts using IAM */
6188 u32 icr = E1000_READ_REG(hw, E1000_ICR);
6190 igb_write_itr(q_vector);
6192 if (icr & E1000_ICR_DRSTA)
6193 schedule_work(&adapter->reset_task);
6195 if (icr & E1000_ICR_DOUTSYNC) {
6196 /* HW is reporting DMA is out of sync */
6197 adapter->stats.doosync++;
6200 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
6201 hw->mac.get_link_status = 1;
6202 if (!test_bit(__IGB_DOWN, &adapter->state))
6203 mod_timer(&adapter->watchdog_timer, jiffies + 1);
6206 napi_schedule(&q_vector->napi);
6212 * igb_intr - Legacy Interrupt Handler
6213 * @irq: interrupt number
6214 * @data: pointer to a network interface device structure
6216 static irqreturn_t igb_intr(int irq, void *data)
6218 struct igb_adapter *adapter = data;
6219 struct igb_q_vector *q_vector = adapter->q_vector[0];
6220 struct e1000_hw *hw = &adapter->hw;
6221 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
6222 * need for the IMC write */
6223 u32 icr = E1000_READ_REG(hw, E1000_ICR);
6225 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
6226 * not set, then the adapter didn't send an interrupt */
6227 if (!(icr & E1000_ICR_INT_ASSERTED))
6230 igb_write_itr(q_vector);
6232 if (icr & E1000_ICR_DRSTA)
6233 schedule_work(&adapter->reset_task);
6235 if (icr & E1000_ICR_DOUTSYNC) {
6236 /* HW is reporting DMA is out of sync */
6237 adapter->stats.doosync++;
6240 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
6241 hw->mac.get_link_status = 1;
6242 /* guard against interrupt when we're going down */
6243 if (!test_bit(__IGB_DOWN, &adapter->state))
6244 mod_timer(&adapter->watchdog_timer, jiffies + 1);
6247 napi_schedule(&q_vector->napi);
6252 void igb_ring_irq_enable(struct igb_q_vector *q_vector)
6254 struct igb_adapter *adapter = q_vector->adapter;
6255 struct e1000_hw *hw = &adapter->hw;
6257 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
6258 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
6259 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
6260 igb_set_itr(q_vector);
6262 igb_update_ring_itr(q_vector);
6265 if (!test_bit(__IGB_DOWN, &adapter->state)) {
6266 if (adapter->msix_entries)
6267 E1000_WRITE_REG(hw, E1000_EIMS, q_vector->eims_value);
6269 igb_irq_enable(adapter);
6274 * igb_poll - NAPI Rx polling callback
6275 * @napi: napi polling structure
6276 * @budget: count of how many packets we should handle
6278 static int igb_poll(struct napi_struct *napi, int budget)
6280 struct igb_q_vector *q_vector = container_of(napi, struct igb_q_vector, napi);
6281 bool clean_complete = true;
6284 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
6285 igb_update_dca(q_vector);
6287 if (q_vector->tx.ring)
6288 clean_complete = igb_clean_tx_irq(q_vector);
6290 if (q_vector->rx.ring)
6291 clean_complete &= igb_clean_rx_irq(q_vector, budget);
6293 #ifndef HAVE_NETDEV_NAPI_LIST
6294 /* if netdev is disabled we need to stop polling */
6295 if (!netif_running(q_vector->adapter->netdev))
6296 clean_complete = true;
6299 /* If all work not completed, return budget and keep polling */
6300 if (!clean_complete)
6303 /* If not enough Rx work done, exit the polling mode */
6304 napi_complete(napi);
6305 igb_ring_irq_enable(q_vector);
6310 #ifdef HAVE_HW_TIME_STAMP
6312 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
6313 * @adapter: board private structure
6314 * @shhwtstamps: timestamp structure to update
6315 * @regval: unsigned 64bit system time value.
6317 * We need to convert the system time value stored in the RX/TXSTMP registers
6318 * into a hwtstamp which can be used by the upper level timestamping functions
6320 static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
6321 struct skb_shared_hwtstamps *shhwtstamps,
6327 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
6328 * 24 to match clock shift we setup earlier.
6330 if (adapter->hw.mac.type >= e1000_82580)
6331 regval <<= IGB_82580_TSYNC_SHIFT;
6333 ns = timecounter_cyc2time(&adapter->clock, regval);
6336 * force a timecompare_update here (even if less than a second
6337 * has passed) in order to prevent the case when ptpd or other
6338 * software jumps the clock offset. othwerise there is a small
6339 * window when the timestamp would be based on previous skew
6340 * and invalid results would be pushed to the network stack.
6342 timecompare_update(&adapter->compare, 0);
6343 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
6344 shhwtstamps->hwtstamp = ns_to_ktime(ns);
6345 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
6349 * igb_tx_hwtstamp - utility function which checks for TX time stamp
6350 * @q_vector: pointer to q_vector containing needed info
6351 * @buffer: pointer to igb_tx_buffer structure
6353 * If we were asked to do hardware stamping and such a time stamp is
6354 * available, then it must have been for this skb here because we only
6355 * allow only one such packet into the queue.
6357 static void igb_tx_hwtstamp(struct igb_q_vector *q_vector,
6358 struct igb_tx_buffer *buffer_info)
6360 struct igb_adapter *adapter = q_vector->adapter;
6361 struct e1000_hw *hw = &adapter->hw;
6362 struct skb_shared_hwtstamps shhwtstamps;
6365 /* if skb does not support hw timestamp or TX stamp not valid exit */
6366 if (likely(!(buffer_info->tx_flags & IGB_TX_FLAGS_TSTAMP)) ||
6367 !(E1000_READ_REG(hw, E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
6370 regval = E1000_READ_REG(hw, E1000_TXSTMPL);
6371 regval |= (u64)E1000_READ_REG(hw, E1000_TXSTMPH) << 32;
6373 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
6374 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
6379 * igb_clean_tx_irq - Reclaim resources after transmit completes
6380 * @q_vector: pointer to q_vector containing needed info
6381 * returns TRUE if ring is completely cleaned
6383 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
6385 struct igb_adapter *adapter = q_vector->adapter;
6386 struct igb_ring *tx_ring = q_vector->tx.ring;
6387 struct igb_tx_buffer *tx_buffer;
6388 union e1000_adv_tx_desc *tx_desc, *eop_desc;
6389 unsigned int total_bytes = 0, total_packets = 0;
6390 unsigned int budget = q_vector->tx.work_limit;
6391 unsigned int i = tx_ring->next_to_clean;
6393 if (test_bit(__IGB_DOWN, &adapter->state))
6396 tx_buffer = &tx_ring->tx_buffer_info[i];
6397 tx_desc = IGB_TX_DESC(tx_ring, i);
6398 i -= tx_ring->count;
6400 for (; budget; budget--) {
6401 eop_desc = tx_buffer->next_to_watch;
6403 /* prevent any other reads prior to eop_desc */
6406 /* if next_to_watch is not set then there is no work pending */
6410 /* if DD is not set pending work has not been completed */
6411 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
6414 /* clear next_to_watch to prevent false hangs */
6415 tx_buffer->next_to_watch = NULL;
6417 /* update the statistics for this packet */
6418 total_bytes += tx_buffer->bytecount;
6419 total_packets += tx_buffer->gso_segs;
6421 #ifdef HAVE_HW_TIME_STAMP
6422 /* retrieve hardware timestamp */
6423 igb_tx_hwtstamp(q_vector, tx_buffer);
6427 dev_kfree_skb_any(tx_buffer->skb);
6429 /* unmap skb header data */
6430 dma_unmap_single(tx_ring->dev,
6431 dma_unmap_addr(tx_buffer, dma),
6432 dma_unmap_len(tx_buffer, len),
6435 /* clear tx_buffer data */
6436 tx_buffer->skb = NULL;
6437 dma_unmap_len_set(tx_buffer, len, 0);
6439 /* clear last DMA location and unmap remaining buffers */
6440 while (tx_desc != eop_desc) {
6445 i -= tx_ring->count;
6446 tx_buffer = tx_ring->tx_buffer_info;
6447 tx_desc = IGB_TX_DESC(tx_ring, 0);
6450 /* unmap any remaining paged data */
6451 if (dma_unmap_len(tx_buffer, len)) {
6452 dma_unmap_page(tx_ring->dev,
6453 dma_unmap_addr(tx_buffer, dma),
6454 dma_unmap_len(tx_buffer, len),
6456 dma_unmap_len_set(tx_buffer, len, 0);
6460 /* move us one more past the eop_desc for start of next pkt */
6465 i -= tx_ring->count;
6466 tx_buffer = tx_ring->tx_buffer_info;
6467 tx_desc = IGB_TX_DESC(tx_ring, 0);
6472 netdev_tx_completed_queue(txring_txq(tx_ring),
6473 total_packets, total_bytes);
6474 #endif /* CONFIG_BQL */
6476 i += tx_ring->count;
6477 tx_ring->next_to_clean = i;
6478 tx_ring->tx_stats.bytes += total_bytes;
6479 tx_ring->tx_stats.packets += total_packets;
6480 q_vector->tx.total_bytes += total_bytes;
6481 q_vector->tx.total_packets += total_packets;
6483 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
6484 struct e1000_hw *hw = &adapter->hw;
6486 eop_desc = tx_buffer->next_to_watch;
6488 /* Detect a transmit hang in hardware, this serializes the
6489 * check with the clearing of time_stamp and movement of i */
6490 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
6492 time_after(jiffies, tx_buffer->time_stamp +
6493 (adapter->tx_timeout_factor * HZ))
6494 && !(E1000_READ_REG(hw, E1000_STATUS) &
6495 E1000_STATUS_TXOFF)) {
6497 /* detected Tx unit hang */
6498 dev_err(tx_ring->dev,
6499 "Detected Tx Unit Hang\n"
6503 " next_to_use <%x>\n"
6504 " next_to_clean <%x>\n"
6505 "buffer_info[next_to_clean]\n"
6506 " time_stamp <%lx>\n"
6507 " next_to_watch <%p>\n"
6509 " desc.status <%x>\n",
6510 tx_ring->queue_index,
6511 E1000_READ_REG(hw, E1000_TDH(tx_ring->reg_idx)),
6512 readl(tx_ring->tail),
6513 tx_ring->next_to_use,
6514 tx_ring->next_to_clean,
6515 tx_buffer->time_stamp,
6518 eop_desc->wb.status);
6519 if (netif_is_multiqueue(netdev_ring(tx_ring)))
6520 netif_stop_subqueue(netdev_ring(tx_ring),
6521 ring_queue_index(tx_ring));
6523 netif_stop_queue(netdev_ring(tx_ring));
6525 /* we are about to reset, no point in enabling stuff */
6530 if (unlikely(total_packets &&
6531 netif_carrier_ok(netdev_ring(tx_ring)) &&
6532 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
6533 /* Make sure that anybody stopping the queue after this
6534 * sees the new next_to_clean.
6537 if (netif_is_multiqueue(netdev_ring(tx_ring))) {
6538 if (__netif_subqueue_stopped(netdev_ring(tx_ring),
6539 ring_queue_index(tx_ring)) &&
6540 !(test_bit(__IGB_DOWN, &adapter->state))) {
6541 netif_wake_subqueue(netdev_ring(tx_ring),
6542 ring_queue_index(tx_ring));
6543 tx_ring->tx_stats.restart_queue++;
6546 if (netif_queue_stopped(netdev_ring(tx_ring)) &&
6547 !(test_bit(__IGB_DOWN, &adapter->state))) {
6548 netif_wake_queue(netdev_ring(tx_ring));
6549 tx_ring->tx_stats.restart_queue++;
6557 #ifdef HAVE_VLAN_RX_REGISTER
6559 * igb_receive_skb - helper function to handle rx indications
6560 * @q_vector: structure containing interrupt and ring information
6561 * @skb: packet to send up
6563 static void igb_receive_skb(struct igb_q_vector *q_vector,
6564 struct sk_buff *skb)
6566 struct vlan_group **vlgrp = netdev_priv(skb->dev);
6568 if (IGB_CB(skb)->vid) {
6570 vlan_gro_receive(&q_vector->napi, *vlgrp,
6571 IGB_CB(skb)->vid, skb);
6573 dev_kfree_skb_any(skb);
6576 napi_gro_receive(&q_vector->napi, skb);
6580 #endif /* HAVE_VLAN_RX_REGISTER */
6581 static inline void igb_rx_checksum(struct igb_ring *ring,
6582 union e1000_adv_rx_desc *rx_desc,
6583 struct sk_buff *skb)
6585 skb_checksum_none_assert(skb);
6587 /* Ignore Checksum bit is set */
6588 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
6591 /* Rx checksum disabled via ethtool */
6592 #ifdef HAVE_NDO_SET_FEATURES
6593 if (!(netdev_ring(ring)->features & NETIF_F_RXCSUM))
6595 if (!test_bit(IGB_RING_FLAG_RX_CSUM, &ring->flags))
6599 /* TCP/UDP checksum error bit is set */
6600 if (igb_test_staterr(rx_desc,
6601 E1000_RXDEXT_STATERR_TCPE |
6602 E1000_RXDEXT_STATERR_IPE)) {
6604 * work around errata with sctp packets where the TCPE aka
6605 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
6606 * packets, (aka let the stack check the crc32c)
6608 if (!((skb->len == 60) &&
6609 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags)))
6610 ring->rx_stats.csum_err++;
6612 /* let the stack verify checksum errors */
6615 /* It must be a TCP or UDP packet with a valid checksum */
6616 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
6617 E1000_RXD_STAT_UDPCS))
6618 skb->ip_summed = CHECKSUM_UNNECESSARY;
6621 #ifdef NETIF_F_RXHASH
6622 static inline void igb_rx_hash(struct igb_ring *ring,
6623 union e1000_adv_rx_desc *rx_desc,
6624 struct sk_buff *skb)
6626 if (netdev_ring(ring)->features & NETIF_F_RXHASH)
6627 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
6631 #ifdef HAVE_HW_TIME_STAMP
6632 static void igb_rx_hwtstamp(struct igb_q_vector *q_vector,
6633 union e1000_adv_rx_desc *rx_desc,
6634 struct sk_buff *skb)
6636 struct igb_adapter *adapter = q_vector->adapter;
6637 struct e1000_hw *hw = &adapter->hw;
6640 if (!igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP |
6641 E1000_RXDADV_STAT_TS))
6645 * If this bit is set, then the RX registers contain the time stamp. No
6646 * other packet will be time stamped until we read these registers, so
6647 * read the registers to make them available again. Because only one
6648 * packet can be time stamped at a time, we know that the register
6649 * values must belong to this one here and therefore we don't need to
6650 * compare any of the additional attributes stored for it.
6652 * If nothing went wrong, then it should have a skb_shared_tx that we
6653 * can turn into a skb_shared_hwtstamps.
6655 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
6656 u32 *stamp = (u32 *)skb->data;
6657 regval = le32_to_cpu(*(stamp + 2));
6658 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
6659 skb_pull(skb, IGB_TS_HDR_LEN);
6661 if(!(E1000_READ_REG(hw, E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
6664 regval = E1000_READ_REG(hw, E1000_RXSTMPL);
6665 regval |= (u64)E1000_READ_REG(hw, E1000_RXSTMPH) << 32;
6668 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
6671 static void igb_rx_vlan(struct igb_ring *ring,
6672 union e1000_adv_rx_desc *rx_desc,
6673 struct sk_buff *skb)
6675 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
6677 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
6678 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags))
6679 vid = be16_to_cpu(rx_desc->wb.upper.vlan);
6681 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
6682 #ifdef HAVE_VLAN_RX_REGISTER
6683 IGB_CB(skb)->vid = vid;
6685 IGB_CB(skb)->vid = 0;
6688 #ifdef HAVE_VLAN_PROTOCOL
6689 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
6691 __vlan_hwaccel_put_tag(skb, vid);
6699 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
6700 static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
6702 /* HW will not DMA in data larger than the given buffer, even if it
6703 * parses the (NFS, of course) header to be larger. In that case, it
6704 * fills the header buffer and spills the rest into the page.
6706 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info) &
6707 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
6708 if (hlen > IGB_RX_HDR_LEN)
6709 hlen = IGB_RX_HDR_LEN;
6716 * igb_merge_active_tail - merge active tail into lro skb
6717 * @tail: pointer to active tail in frag_list
6719 * This function merges the length and data of an active tail into the
6720 * skb containing the frag_list. It resets the tail's pointer to the head,
6721 * but it leaves the heads pointer to tail intact.
6723 static inline struct sk_buff *igb_merge_active_tail(struct sk_buff *tail)
6725 struct sk_buff *head = IGB_CB(tail)->head;
6730 head->len += tail->len;
6731 head->data_len += tail->len;
6732 head->truesize += tail->len;
6734 IGB_CB(tail)->head = NULL;
6740 * igb_add_active_tail - adds an active tail into the skb frag_list
6741 * @head: pointer to the start of the skb
6742 * @tail: pointer to active tail to add to frag_list
6744 * This function adds an active tail to the end of the frag list. This tail
6745 * will still be receiving data so we cannot yet ad it's stats to the main
6746 * skb. That is done via igb_merge_active_tail.
6748 static inline void igb_add_active_tail(struct sk_buff *head, struct sk_buff *tail)
6750 struct sk_buff *old_tail = IGB_CB(head)->tail;
6753 igb_merge_active_tail(old_tail);
6754 old_tail->next = tail;
6756 skb_shinfo(head)->frag_list = tail;
6759 IGB_CB(tail)->head = head;
6760 IGB_CB(head)->tail = tail;
6762 IGB_CB(head)->append_cnt++;
6766 * igb_close_active_frag_list - cleanup pointers on a frag_list skb
6767 * @head: pointer to head of an active frag list
6769 * This function will clear the frag_tail_tracker pointer on an active
6770 * frag_list and returns true if the pointer was actually set
6772 static inline bool igb_close_active_frag_list(struct sk_buff *head)
6774 struct sk_buff *tail = IGB_CB(head)->tail;
6779 igb_merge_active_tail(tail);
6781 IGB_CB(head)->tail = NULL;
6787 * igb_can_lro - returns true if packet is TCP/IPV4 and LRO is enabled
6788 * @adapter: board private structure
6789 * @rx_desc: pointer to the rx descriptor
6790 * @skb: pointer to the skb to be merged
6793 static inline bool igb_can_lro(struct igb_ring *rx_ring,
6794 union e1000_adv_rx_desc *rx_desc,
6795 struct sk_buff *skb)
6797 struct iphdr *iph = (struct iphdr *)skb->data;
6798 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
6800 /* verify LRO is enabled */
6801 if (!(netdev_ring(rx_ring)->features & NETIF_F_LRO))
6804 /* verify hardware indicates this is IPv4/TCP */
6805 if((!(pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_TCP)) ||
6806 !(pkt_info & cpu_to_le16(E1000_RXDADV_PKTTYPE_IPV4))))
6809 /* verify the header is large enough for us to read IP/TCP fields */
6810 if (!pskb_may_pull(skb, sizeof(struct igb_lrohdr)))
6813 /* verify there are no VLANs on packet */
6814 if (skb->protocol != __constant_htons(ETH_P_IP))
6817 /* ensure we are version 4 with no options */
6818 if (*(u8 *)iph != 0x45)
6821 /* .. and the packet is not fragmented */
6822 if (iph->frag_off & htons(IP_MF | IP_OFFSET))
6825 /* .. and that next header is TCP */
6826 if (iph->protocol != IPPROTO_TCP)
6832 static inline struct igb_lrohdr *igb_lro_hdr(struct sk_buff *skb)
6834 return (struct igb_lrohdr *)skb->data;
6838 * igb_lro_flush - Indicate packets to upper layer.
6840 * Update IP and TCP header part of head skb if more than one
6841 * skb's chained and indicate packets to upper layer.
6843 static void igb_lro_flush(struct igb_q_vector *q_vector,
6844 struct sk_buff *skb)
6846 struct igb_lro_list *lrolist = q_vector->lrolist;
6848 __skb_unlink(skb, &lrolist->active);
6850 if (IGB_CB(skb)->append_cnt) {
6851 struct igb_lrohdr *lroh = igb_lro_hdr(skb);
6853 /* close any active lro contexts */
6854 igb_close_active_frag_list(skb);
6856 /* incorporate ip header and re-calculate checksum */
6857 lroh->iph.tot_len = ntohs(skb->len);
6858 lroh->iph.check = 0;
6860 /* header length is 5 since we know no options exist */
6861 lroh->iph.check = ip_fast_csum((u8 *)lroh, 5);
6863 /* clear TCP checksum to indicate we are an LRO frame */
6866 /* incorporate latest timestamp into the tcp header */
6867 if (IGB_CB(skb)->tsecr) {
6868 lroh->ts[2] = IGB_CB(skb)->tsecr;
6869 lroh->ts[1] = htonl(IGB_CB(skb)->tsval);
6873 skb_shinfo(skb)->gso_size = IGB_CB(skb)->mss;
6877 #ifdef HAVE_VLAN_RX_REGISTER
6878 igb_receive_skb(q_vector, skb);
6880 napi_gro_receive(&q_vector->napi, skb);
6882 lrolist->stats.flushed++;
6885 static void igb_lro_flush_all(struct igb_q_vector *q_vector)
6887 struct igb_lro_list *lrolist = q_vector->lrolist;
6888 struct sk_buff *skb, *tmp;
6890 skb_queue_reverse_walk_safe(&lrolist->active, skb, tmp)
6891 igb_lro_flush(q_vector, skb);
6895 * igb_lro_header_ok - Main LRO function.
6897 static void igb_lro_header_ok(struct sk_buff *skb)
6899 struct igb_lrohdr *lroh = igb_lro_hdr(skb);
6900 u16 opt_bytes, data_len;
6902 IGB_CB(skb)->tail = NULL;
6903 IGB_CB(skb)->tsecr = 0;
6904 IGB_CB(skb)->append_cnt = 0;
6905 IGB_CB(skb)->mss = 0;
6907 /* ensure that the checksum is valid */
6908 if (skb->ip_summed != CHECKSUM_UNNECESSARY)
6911 /* If we see CE codepoint in IP header, packet is not mergeable */
6912 if (INET_ECN_is_ce(ipv4_get_dsfield(&lroh->iph)))
6915 /* ensure no bits set besides ack or psh */
6916 if (lroh->th.fin || lroh->th.syn || lroh->th.rst ||
6917 lroh->th.urg || lroh->th.ece || lroh->th.cwr ||
6921 /* store the total packet length */
6922 data_len = ntohs(lroh->iph.tot_len);
6924 /* remove any padding from the end of the skb */
6925 __pskb_trim(skb, data_len);
6927 /* remove header length from data length */
6928 data_len -= sizeof(struct igb_lrohdr);
6931 * check for timestamps. Since the only option we handle are timestamps,
6932 * we only have to handle the simple case of aligned timestamps
6934 opt_bytes = (lroh->th.doff << 2) - sizeof(struct tcphdr);
6935 if (opt_bytes != 0) {
6936 if ((opt_bytes != TCPOLEN_TSTAMP_ALIGNED) ||
6937 !pskb_may_pull(skb, sizeof(struct igb_lrohdr) +
6938 TCPOLEN_TSTAMP_ALIGNED) ||
6939 (lroh->ts[0] != htonl((TCPOPT_NOP << 24) |
6940 (TCPOPT_NOP << 16) |
6941 (TCPOPT_TIMESTAMP << 8) |
6942 TCPOLEN_TIMESTAMP)) ||
6943 (lroh->ts[2] == 0)) {
6947 IGB_CB(skb)->tsval = ntohl(lroh->ts[1]);
6948 IGB_CB(skb)->tsecr = lroh->ts[2];
6950 data_len -= TCPOLEN_TSTAMP_ALIGNED;
6953 /* record data_len as mss for the packet */
6954 IGB_CB(skb)->mss = data_len;
6955 IGB_CB(skb)->next_seq = ntohl(lroh->th.seq);
6958 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
6959 static bool igb_merge_frags(struct sk_buff *lro_skb, struct sk_buff *new_skb)
6961 struct sk_buff *tail;
6962 struct skb_shared_info *tail_info;
6963 struct skb_shared_info *new_skb_info;
6966 /* header must be empty to pull frags into current skb */
6967 if (skb_headlen(new_skb))
6970 if (IGB_CB(lro_skb)->tail)
6971 tail = IGB_CB(lro_skb)->tail;
6975 tail_info = skb_shinfo(tail);
6976 new_skb_info = skb_shinfo(new_skb);
6978 /* make sure we have room in frags list */
6979 if (new_skb_info->nr_frags >= (MAX_SKB_FRAGS - tail_info->nr_frags))
6982 /* bump append count */
6983 IGB_CB(lro_skb)->append_cnt++;
6985 /* copy frags into the last skb */
6986 memcpy(tail_info->frags + tail_info->nr_frags,
6987 new_skb_info->frags,
6988 new_skb_info->nr_frags * sizeof(skb_frag_t));
6990 /* copy size data over */
6991 tail_info->nr_frags += new_skb_info->nr_frags;
6992 data_len = IGB_CB(new_skb)->mss;
6993 tail->len += data_len;
6994 tail->data_len += data_len;
6995 tail->truesize += data_len;
6997 /* wipe record of data from new_skb */
6998 new_skb_info->nr_frags = 0;
6999 new_skb->len = new_skb->data_len = 0;
7000 new_skb->truesize -= data_len;
7001 new_skb->data = new_skb->head + NET_SKB_PAD + NET_IP_ALIGN;
7002 skb_reset_tail_pointer(new_skb);
7003 new_skb->protocol = 0;
7004 new_skb->ip_summed = CHECKSUM_NONE;
7005 #ifdef HAVE_VLAN_RX_REGISTER
7006 IGB_CB(new_skb)->vid = 0;
7008 new_skb->vlan_tci = 0;
7014 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7016 * igb_lro_queue - if able, queue skb into lro chain
7017 * @q_vector: structure containing interrupt and ring information
7018 * @new_skb: pointer to current skb being checked
7020 * Checks whether the skb given is eligible for LRO and if that's
7021 * fine chains it to the existing lro_skb based on flowid. If an LRO for
7022 * the flow doesn't exist create one.
7024 static struct sk_buff *igb_lro_queue(struct igb_q_vector *q_vector,
7025 struct sk_buff *new_skb)
7027 struct sk_buff *lro_skb;
7028 struct igb_lro_list *lrolist = q_vector->lrolist;
7029 struct igb_lrohdr *lroh = igb_lro_hdr(new_skb);
7030 __be32 saddr = lroh->iph.saddr;
7031 __be32 daddr = lroh->iph.daddr;
7032 __be32 tcp_ports = *(__be32 *)&lroh->th;
7034 #ifdef HAVE_VLAN_RX_REGISTER
7035 u16 vid = IGB_CB(new_skb)->vid;
7037 u16 vid = new_skb->vlan_tci;
7040 igb_lro_header_ok(new_skb);
7043 * we have a packet that might be eligible for LRO,
7044 * so see if it matches anything we might expect
7046 skb_queue_walk(&lrolist->active, lro_skb) {
7047 if (*(__be32 *)&igb_lro_hdr(lro_skb)->th != tcp_ports ||
7048 igb_lro_hdr(lro_skb)->iph.saddr != saddr ||
7049 igb_lro_hdr(lro_skb)->iph.daddr != daddr)
7052 #ifdef HAVE_VLAN_RX_REGISTER
7053 if (IGB_CB(lro_skb)->vid != vid)
7055 if (lro_skb->vlan_tci != vid)
7059 /* out of order packet */
7060 if (IGB_CB(lro_skb)->next_seq != IGB_CB(new_skb)->next_seq) {
7061 igb_lro_flush(q_vector, lro_skb);
7062 IGB_CB(new_skb)->mss = 0;
7066 /* TCP timestamp options have changed */
7067 if (!IGB_CB(lro_skb)->tsecr != !IGB_CB(new_skb)->tsecr) {
7068 igb_lro_flush(q_vector, lro_skb);
7072 /* make sure timestamp values are increasing */
7073 if (IGB_CB(lro_skb)->tsecr &&
7074 IGB_CB(lro_skb)->tsval > IGB_CB(new_skb)->tsval) {
7075 igb_lro_flush(q_vector, lro_skb);
7076 IGB_CB(new_skb)->mss = 0;
7080 data_len = IGB_CB(new_skb)->mss;
7083 * malformed header, no tcp data, resultant packet would
7084 * be too large, or new skb is larger than our current mss.
7086 if (data_len == 0 ||
7087 data_len > IGB_CB(lro_skb)->mss ||
7088 data_len > IGB_CB(lro_skb)->free) {
7089 igb_lro_flush(q_vector, lro_skb);
7093 /* ack sequence numbers or window size has changed */
7094 if (igb_lro_hdr(lro_skb)->th.ack_seq != lroh->th.ack_seq ||
7095 igb_lro_hdr(lro_skb)->th.window != lroh->th.window) {
7096 igb_lro_flush(q_vector, lro_skb);
7100 /* Remove IP and TCP header*/
7101 skb_pull(new_skb, new_skb->len - data_len);
7103 /* update timestamp and timestamp echo response */
7104 IGB_CB(lro_skb)->tsval = IGB_CB(new_skb)->tsval;
7105 IGB_CB(lro_skb)->tsecr = IGB_CB(new_skb)->tsecr;
7107 /* update sequence and free space */
7108 IGB_CB(lro_skb)->next_seq += data_len;
7109 IGB_CB(lro_skb)->free -= data_len;
7111 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7112 /* if header is empty pull pages into current skb */
7113 if (igb_merge_frags(lro_skb, new_skb)) {
7114 lrolist->stats.recycled++;
7117 /* chain this new skb in frag_list */
7118 igb_add_active_tail(lro_skb, new_skb);
7120 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7124 if ((data_len < IGB_CB(lro_skb)->mss) || lroh->th.psh) {
7125 igb_lro_hdr(lro_skb)->th.psh |= lroh->th.psh;
7126 igb_lro_flush(q_vector, lro_skb);
7129 lrolist->stats.coal++;
7133 if (IGB_CB(new_skb)->mss && !lroh->th.psh) {
7134 /* if we are at capacity flush the tail */
7135 if (skb_queue_len(&lrolist->active) >= IGB_LRO_MAX) {
7136 lro_skb = skb_peek_tail(&lrolist->active);
7138 igb_lro_flush(q_vector, lro_skb);
7141 /* update sequence and free space */
7142 IGB_CB(new_skb)->next_seq += IGB_CB(new_skb)->mss;
7143 IGB_CB(new_skb)->free = 65521 - new_skb->len;
7145 /* .. and insert at the front of the active list */
7146 __skb_queue_head(&lrolist->active, new_skb);
7148 lrolist->stats.coal++;
7152 /* packet not handled by any of the above, pass it to the stack */
7153 #ifdef HAVE_VLAN_RX_REGISTER
7154 igb_receive_skb(q_vector, new_skb);
7156 napi_gro_receive(&q_vector->napi, new_skb);
7161 #endif /* IGB_NO_LRO */
7162 static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
7164 struct igb_ring *rx_ring = q_vector->rx.ring;
7165 union e1000_adv_rx_desc *rx_desc;
7166 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7167 const int current_node = numa_node_id();
7169 unsigned int total_bytes = 0, total_packets = 0;
7170 u16 cleaned_count = igb_desc_unused(rx_ring);
7171 u16 i = rx_ring->next_to_clean;
7173 rx_desc = IGB_RX_DESC(rx_ring, i);
7175 while (igb_test_staterr(rx_desc, E1000_RXD_STAT_DD)) {
7176 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
7177 struct sk_buff *skb = buffer_info->skb;
7178 union e1000_adv_rx_desc *next_rxd;
7180 buffer_info->skb = NULL;
7181 prefetch(skb->data);
7184 if (i == rx_ring->count)
7187 next_rxd = IGB_RX_DESC(rx_ring, i);
7191 * This memory barrier is needed to keep us from reading
7192 * any other fields out of the rx_desc until we know the
7193 * RXD_STAT_DD bit is set
7197 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7198 __skb_put(skb, le16_to_cpu(rx_desc->wb.upper.length));
7199 dma_unmap_single(rx_ring->dev, buffer_info->dma,
7200 rx_ring->rx_buffer_len,
7202 buffer_info->dma = 0;
7205 if (!skb_is_nonlinear(skb)) {
7206 __skb_put(skb, igb_get_hlen(rx_desc));
7207 dma_unmap_single(rx_ring->dev, buffer_info->dma,
7210 buffer_info->dma = 0;
7213 if (rx_desc->wb.upper.length) {
7214 u16 length = le16_to_cpu(rx_desc->wb.upper.length);
7216 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
7218 buffer_info->page_offset,
7222 skb->data_len += length;
7223 skb->truesize += length;
7225 if ((page_count(buffer_info->page) != 1) ||
7226 (page_to_nid(buffer_info->page) != current_node))
7227 buffer_info->page = NULL;
7229 get_page(buffer_info->page);
7231 dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
7232 PAGE_SIZE / 2, DMA_FROM_DEVICE);
7233 buffer_info->page_dma = 0;
7236 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)) {
7237 struct igb_rx_buffer *next_buffer;
7238 next_buffer = &rx_ring->rx_buffer_info[i];
7239 buffer_info->skb = next_buffer->skb;
7240 buffer_info->dma = next_buffer->dma;
7241 next_buffer->skb = skb;
7242 next_buffer->dma = 0;
7246 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7247 if (igb_test_staterr(rx_desc,
7248 E1000_RXDEXT_ERR_FRAME_ERR_MASK)) {
7249 dev_kfree_skb_any(skb);
7253 #ifdef HAVE_HW_TIME_STAMP
7254 igb_rx_hwtstamp(q_vector, rx_desc, skb);
7256 #ifdef NETIF_F_RXHASH
7257 igb_rx_hash(rx_ring, rx_desc, skb);
7259 igb_rx_checksum(rx_ring, rx_desc, skb);
7260 igb_rx_vlan(rx_ring, rx_desc, skb);
7262 total_bytes += skb->len;
7265 skb->protocol = eth_type_trans(skb, netdev_ring(rx_ring));
7268 if (igb_can_lro(rx_ring, rx_desc, skb))
7269 buffer_info->skb = igb_lro_queue(q_vector, skb);
7272 #ifdef HAVE_VLAN_RX_REGISTER
7273 igb_receive_skb(q_vector, skb);
7275 napi_gro_receive(&q_vector->napi, skb);
7279 netdev_ring(rx_ring)->last_rx = jiffies;
7289 /* return some buffers to hardware, one at a time is too slow */
7290 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
7291 igb_alloc_rx_buffers(rx_ring, cleaned_count);
7295 /* use prefetched values */
7299 rx_ring->next_to_clean = i;
7300 rx_ring->rx_stats.packets += total_packets;
7301 rx_ring->rx_stats.bytes += total_bytes;
7302 q_vector->rx.total_packets += total_packets;
7303 q_vector->rx.total_bytes += total_bytes;
7306 igb_alloc_rx_buffers(rx_ring, cleaned_count);
7309 if (netdev_ring(rx_ring)->features & NETIF_F_LRO)
7310 igb_lro_flush_all(q_vector);
7312 #endif /* IGB_NO_LRO */
7316 static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
7317 struct igb_rx_buffer *bi)
7319 struct sk_buff *skb = bi->skb;
7320 dma_addr_t dma = bi->dma;
7326 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7327 skb = netdev_alloc_skb_ip_align(netdev_ring(rx_ring),
7328 rx_ring->rx_buffer_len);
7330 skb = netdev_alloc_skb_ip_align(netdev_ring(rx_ring),
7335 rx_ring->rx_stats.alloc_failed++;
7339 /* initialize skb for ring */
7340 skb_record_rx_queue(skb, ring_queue_index(rx_ring));
7343 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7344 dma = dma_map_single(rx_ring->dev, skb->data,
7345 rx_ring->rx_buffer_len, DMA_FROM_DEVICE);
7347 dma = dma_map_single(rx_ring->dev, skb->data,
7348 IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
7351 if (dma_mapping_error(rx_ring->dev, dma)) {
7352 rx_ring->rx_stats.alloc_failed++;
7360 #ifndef CONFIG_IGB_DISABLE_PACKET_SPLIT
7361 static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
7362 struct igb_rx_buffer *bi)
7364 struct page *page = bi->page;
7365 dma_addr_t page_dma = bi->page_dma;
7366 unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
7372 page = alloc_page(GFP_ATOMIC | __GFP_COLD);
7374 if (unlikely(!page)) {
7375 rx_ring->rx_stats.alloc_failed++;
7380 page_dma = dma_map_page(rx_ring->dev, page,
7381 page_offset, PAGE_SIZE / 2,
7384 if (dma_mapping_error(rx_ring->dev, page_dma)) {
7385 rx_ring->rx_stats.alloc_failed++;
7389 bi->page_dma = page_dma;
7390 bi->page_offset = page_offset;
7394 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7396 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
7397 * @adapter: address of board private structure
7399 void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
7401 union e1000_adv_rx_desc *rx_desc;
7402 struct igb_rx_buffer *bi;
7403 u16 i = rx_ring->next_to_use;
7405 rx_desc = IGB_RX_DESC(rx_ring, i);
7406 bi = &rx_ring->rx_buffer_info[i];
7407 i -= rx_ring->count;
7409 while (cleaned_count--) {
7410 if (!igb_alloc_mapped_skb(rx_ring, bi))
7413 /* Refresh the desc even if buffer_addrs didn't change
7414 * because each write-back erases this info. */
7415 #ifdef CONFIG_IGB_DISABLE_PACKET_SPLIT
7416 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
7418 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
7420 if (!igb_alloc_mapped_page(rx_ring, bi))
7423 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
7425 #endif /* CONFIG_IGB_DISABLE_PACKET_SPLIT */
7430 rx_desc = IGB_RX_DESC(rx_ring, 0);
7431 bi = rx_ring->rx_buffer_info;
7432 i -= rx_ring->count;
7435 /* clear the hdr_addr for the next_to_use descriptor */
7436 rx_desc->read.hdr_addr = 0;
7439 i += rx_ring->count;
7441 if (rx_ring->next_to_use != i) {
7442 rx_ring->next_to_use = i;
7444 /* Force memory writes to complete before letting h/w
7445 * know there are new descriptors to fetch. (Only
7446 * applicable for weak-ordered memory model archs,
7447 * such as IA-64). */
7449 writel(i, rx_ring->tail);
7460 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
7462 struct igb_adapter *adapter = netdev_priv(netdev);
7463 struct mii_ioctl_data *data = if_mii(ifr);
7465 if (adapter->hw.phy.media_type != e1000_media_type_copper)
7470 data->phy_id = adapter->hw.phy.addr;
7473 if (!capable(CAP_NET_ADMIN))
7475 if (e1000_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
7483 return E1000_SUCCESS;
7487 #ifdef HAVE_HW_TIME_STAMP
7489 * igb_hwtstamp_ioctl - control hardware time stamping
7494 * Outgoing time stamping can be enabled and disabled. Play nice and
7495 * disable it when requested, although it shouldn't case any overhead
7496 * when no packet needs it. At most one packet in the queue may be
7497 * marked for time stamping, otherwise it would be impossible to tell
7498 * for sure to which packet the hardware time stamp belongs.
7500 * Incoming time stamping has to be configured via the hardware
7501 * filters. Not all combinations are supported, in particular event
7502 * type has to be specified. Matching the kind of event packet is
7503 * not supported, with the exception of "all V2 events regardless of
7507 static int igb_hwtstamp_ioctl(struct net_device *netdev,
7508 struct ifreq *ifr, int cmd)
7510 struct igb_adapter *adapter = netdev_priv(netdev);
7511 struct e1000_hw *hw = &adapter->hw;
7512 struct hwtstamp_config config;
7513 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
7514 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
7515 u32 tsync_rx_cfg = 0;
7520 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
7523 /* reserved for future extensions */
7527 switch (config.tx_type) {
7528 case HWTSTAMP_TX_OFF:
7530 case HWTSTAMP_TX_ON:
7536 switch (config.rx_filter) {
7537 case HWTSTAMP_FILTER_NONE:
7540 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
7541 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
7542 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
7543 case HWTSTAMP_FILTER_ALL:
7545 * register TSYNCRXCFG must be set, therefore it is not
7546 * possible to time stamp both Sync and Delay_Req messages
7547 * => fall back to time stamping all packets
7549 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
7550 config.rx_filter = HWTSTAMP_FILTER_ALL;
7552 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
7553 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
7554 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
7557 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
7558 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
7559 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
7562 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
7563 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
7564 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
7565 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
7568 config.rx_filter = HWTSTAMP_FILTER_SOME;
7570 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
7571 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
7572 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
7573 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
7576 config.rx_filter = HWTSTAMP_FILTER_SOME;
7578 case HWTSTAMP_FILTER_PTP_V2_EVENT:
7579 case HWTSTAMP_FILTER_PTP_V2_SYNC:
7580 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
7581 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
7582 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
7590 if (hw->mac.type == e1000_82575) {
7591 if (tsync_rx_ctl | tsync_tx_ctl)
7596 #ifdef IGB_PER_PKT_TIMESTAMP
7598 * Per-packet timestamping only works if all packets are
7599 * timestamped, so enable timestamping in all packets as
7600 * long as one rx filter was configured.
7602 if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) {
7603 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
7604 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
7608 /* enable/disable TX */
7609 regval = E1000_READ_REG(hw, E1000_TSYNCTXCTL);
7610 regval &= ~E1000_TSYNCTXCTL_ENABLED;
7611 regval |= tsync_tx_ctl;
7612 E1000_WRITE_REG(hw, E1000_TSYNCTXCTL, regval);
7614 /* enable/disable RX */
7615 regval = E1000_READ_REG(hw, E1000_TSYNCRXCTL);
7616 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
7617 regval |= tsync_rx_ctl;
7618 E1000_WRITE_REG(hw, E1000_TSYNCRXCTL, regval);
7620 /* define which PTP packets are time stamped */
7621 E1000_WRITE_REG(hw, E1000_TSYNCRXCFG, tsync_rx_cfg);
7623 /* define ethertype filter for timestamped packets */
7625 E1000_WRITE_REG(hw, E1000_ETQF(3),
7626 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
7627 E1000_ETQF_1588 | /* enable timestamping */
7628 ETH_P_1588)); /* 1588 eth protocol type */
7630 E1000_WRITE_REG(hw, E1000_ETQF(3), 0);
7632 #define PTP_PORT 319
7633 /* L4 Queue Filter[3]: filter by destination port and protocol */
7635 u32 ftqf = (IPPROTO_UDP /* UDP */
7636 | E1000_FTQF_VF_BP /* VF not compared */
7637 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
7638 | E1000_FTQF_MASK); /* mask all inputs */
7639 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
7641 E1000_WRITE_REG(hw, E1000_IMIR(3), htons(PTP_PORT));
7642 E1000_WRITE_REG(hw, E1000_IMIREXT(3),
7643 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
7644 if (hw->mac.type == e1000_82576) {
7645 /* enable source port check */
7646 E1000_WRITE_REG(hw, E1000_SPQF(3), htons(PTP_PORT));
7647 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
7649 E1000_WRITE_REG(hw, E1000_FTQF(3), ftqf);
7651 E1000_WRITE_REG(hw, E1000_FTQF(3), E1000_FTQF_MASK);
7653 E1000_WRITE_FLUSH(hw);
7655 adapter->hwtstamp_config = config;
7657 /* clear TX/RX time stamp registers, just to be sure */
7658 regval = E1000_READ_REG(hw, E1000_TXSTMPH);
7659 regval = E1000_READ_REG(hw, E1000_RXSTMPH);
7661 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
7672 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
7679 return igb_mii_ioctl(netdev, ifr, cmd);
7681 #ifdef HAVE_HW_TIME_STAMP
7683 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
7685 #ifdef ETHTOOL_OPS_COMPAT
7687 return ethtool_ioctl(ifr);
7694 s32 e1000_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
7696 struct igb_adapter *adapter = hw->back;
7699 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
7701 return -E1000_ERR_CONFIG;
7703 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
7705 return E1000_SUCCESS;
7708 s32 e1000_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
7710 struct igb_adapter *adapter = hw->back;
7713 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
7715 return -E1000_ERR_CONFIG;
7717 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
7719 return E1000_SUCCESS;
7722 #ifdef HAVE_VLAN_RX_REGISTER
7723 static void igb_vlan_mode(struct net_device *netdev, struct vlan_group *vlgrp)
7725 void igb_vlan_mode(struct net_device *netdev, u32 features)
7728 struct igb_adapter *adapter = netdev_priv(netdev);
7729 struct e1000_hw *hw = &adapter->hw;
7732 #ifdef HAVE_VLAN_RX_REGISTER
7733 bool enable = !!vlgrp;
7735 igb_irq_disable(adapter);
7737 adapter->vlgrp = vlgrp;
7739 if (!test_bit(__IGB_DOWN, &adapter->state))
7740 igb_irq_enable(adapter);
7742 bool enable = !!(features & NETIF_F_HW_VLAN_RX);
7746 /* enable VLAN tag insert/strip */
7747 ctrl = E1000_READ_REG(hw, E1000_CTRL);
7748 ctrl |= E1000_CTRL_VME;
7749 E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
7751 /* Disable CFI check */
7752 rctl = E1000_READ_REG(hw, E1000_RCTL);
7753 rctl &= ~E1000_RCTL_CFIEN;
7754 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
7756 /* disable VLAN tag insert/strip */
7757 ctrl = E1000_READ_REG(hw, E1000_CTRL);
7758 ctrl &= ~E1000_CTRL_VME;
7759 E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
7762 #ifndef CONFIG_IGB_VMDQ_NETDEV
7763 for (i = 0; i < adapter->vmdq_pools; i++) {
7764 igb_set_vf_vlan_strip(adapter,
7765 adapter->vfs_allocated_count + i,
7770 igb_set_vf_vlan_strip(adapter,
7771 adapter->vfs_allocated_count,
7774 for (i = 1; i < adapter->vmdq_pools; i++) {
7775 #ifdef HAVE_VLAN_RX_REGISTER
7776 struct igb_vmdq_adapter *vadapter;
7777 vadapter = netdev_priv(adapter->vmdq_netdev[i-1]);
7778 enable = !!vadapter->vlgrp;
7780 struct net_device *vnetdev;
7781 vnetdev = adapter->vmdq_netdev[i-1];
7782 enable = !!(vnetdev->features & NETIF_F_HW_VLAN_RX);
7784 igb_set_vf_vlan_strip(adapter,
7785 adapter->vfs_allocated_count + i,
7790 igb_rlpml_set(adapter);
7793 #ifdef HAVE_VLAN_PROTOCOL
7794 static int igb_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
7795 #elif defined HAVE_INT_NDO_VLAN_RX_ADD_VID
7796 static int igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
7798 static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
7801 struct igb_adapter *adapter = netdev_priv(netdev);
7802 int pf_id = adapter->vfs_allocated_count;
7804 /* attempt to add filter to vlvf array */
7805 igb_vlvf_set(adapter, vid, TRUE, pf_id);
7807 /* add the filter since PF can receive vlans w/o entry in vlvf */
7808 igb_vfta_set(adapter, vid, TRUE);
7809 #ifndef HAVE_NETDEV_VLAN_FEATURES
7811 /* Copy feature flags from netdev to the vlan netdev for this vid.
7812 * This allows things like TSO to bubble down to our vlan device.
7813 * There is no need to update netdev for vlan 0 (DCB), since it
7814 * wouldn't has v_netdev.
7816 if (adapter->vlgrp) {
7817 struct vlan_group *vlgrp = adapter->vlgrp;
7818 struct net_device *v_netdev = vlan_group_get_device(vlgrp, vid);
7820 v_netdev->features |= netdev->features;
7821 vlan_group_set_device(vlgrp, vid, v_netdev);
7825 #ifndef HAVE_VLAN_RX_REGISTER
7827 set_bit(vid, adapter->active_vlans);
7829 #ifdef HAVE_INT_NDO_VLAN_RX_ADD_VID
7834 #ifdef HAVE_VLAN_PROTOCOL
7835 static int igb_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
7836 #elif defined HAVE_INT_NDO_VLAN_RX_ADD_VID
7837 static int igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
7839 static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
7842 struct igb_adapter *adapter = netdev_priv(netdev);
7843 int pf_id = adapter->vfs_allocated_count;
7846 #ifdef HAVE_VLAN_RX_REGISTER
7847 igb_irq_disable(adapter);
7849 vlan_group_set_device(adapter->vlgrp, vid, NULL);
7851 if (!test_bit(__IGB_DOWN, &adapter->state))
7852 igb_irq_enable(adapter);
7854 #endif /* HAVE_VLAN_RX_REGISTER */
7855 /* remove vlan from VLVF table array */
7856 err = igb_vlvf_set(adapter, vid, FALSE, pf_id);
7858 /* if vid was not present in VLVF just remove it from table */
7860 igb_vfta_set(adapter, vid, FALSE);
7861 #ifndef HAVE_VLAN_RX_REGISTER
7863 clear_bit(vid, adapter->active_vlans);
7865 #ifdef HAVE_INT_NDO_VLAN_RX_ADD_VID
7870 static void igb_restore_vlan(struct igb_adapter *adapter)
7872 #ifdef HAVE_VLAN_RX_REGISTER
7873 igb_vlan_mode(adapter->netdev, adapter->vlgrp);
7875 if (adapter->vlgrp) {
7877 for (vid = 0; vid < VLAN_N_VID; vid++) {
7878 if (!vlan_group_get_device(adapter->vlgrp, vid))
7880 igb_vlan_rx_add_vid(adapter->netdev, vid);
7886 igb_vlan_mode(adapter->netdev, adapter->netdev->features);
7888 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
7889 #ifdef HAVE_VLAN_PROTOCOL
7890 igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
7892 igb_vlan_rx_add_vid(adapter->netdev, vid);
7898 int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
7900 struct pci_dev *pdev = adapter->pdev;
7901 struct e1000_mac_info *mac = &adapter->hw.mac;
7905 /* Fiber NIC's only allow 1000 gbps Full duplex */
7906 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes ) &&
7907 spddplx != (SPEED_1000 + DUPLEX_FULL)) {
7908 dev_err(pci_dev_to_dev(pdev),
7909 "Unsupported Speed/Duplex configuration\n");
7914 case SPEED_10 + DUPLEX_HALF:
7915 mac->forced_speed_duplex = ADVERTISE_10_HALF;
7917 case SPEED_10 + DUPLEX_FULL:
7918 mac->forced_speed_duplex = ADVERTISE_10_FULL;
7920 case SPEED_100 + DUPLEX_HALF:
7921 mac->forced_speed_duplex = ADVERTISE_100_HALF;
7923 case SPEED_100 + DUPLEX_FULL:
7924 mac->forced_speed_duplex = ADVERTISE_100_FULL;
7926 case SPEED_1000 + DUPLEX_FULL:
7928 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
7930 case SPEED_1000 + DUPLEX_HALF: /* not supported */
7932 dev_err(pci_dev_to_dev(pdev), "Unsupported Speed/Duplex configuration\n");
7938 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
7941 struct net_device *netdev = pci_get_drvdata(pdev);
7942 struct igb_adapter *adapter = netdev_priv(netdev);
7943 struct e1000_hw *hw = &adapter->hw;
7944 u32 ctrl, rctl, status;
7945 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
7950 netif_device_detach(netdev);
7952 if (netif_running(netdev))
7953 __igb_close(netdev, true);
7955 igb_clear_interrupt_scheme(adapter);
7958 retval = pci_save_state(pdev);
7963 status = E1000_READ_REG(hw, E1000_STATUS);
7964 if (status & E1000_STATUS_LU)
7965 wufc &= ~E1000_WUFC_LNKC;
7968 igb_setup_rctl(adapter);
7969 igb_set_rx_mode(netdev);
7971 /* turn on all-multi mode if wake on multicast is enabled */
7972 if (wufc & E1000_WUFC_MC) {
7973 rctl = E1000_READ_REG(hw, E1000_RCTL);
7974 rctl |= E1000_RCTL_MPE;
7975 E1000_WRITE_REG(hw, E1000_RCTL, rctl);
7978 ctrl = E1000_READ_REG(hw, E1000_CTRL);
7979 /* phy power management enable */
7980 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
7981 ctrl |= E1000_CTRL_ADVD3WUC;
7982 E1000_WRITE_REG(hw, E1000_CTRL, ctrl);
7984 /* Allow time for pending master requests to run */
7985 e1000_disable_pcie_master(hw);
7987 E1000_WRITE_REG(hw, E1000_WUC, E1000_WUC_PME_EN);
7988 E1000_WRITE_REG(hw, E1000_WUFC, wufc);
7990 E1000_WRITE_REG(hw, E1000_WUC, 0);
7991 E1000_WRITE_REG(hw, E1000_WUFC, 0);
7994 *enable_wake = wufc || adapter->en_mng_pt;
7996 igb_power_down_link(adapter);
7998 igb_power_up_link(adapter);
8000 /* Release control of h/w to f/w. If f/w is AMT enabled, this
8001 * would have already happened in close and is redundant. */
8002 igb_release_hw_control(adapter);
8004 pci_disable_device(pdev);
8010 #ifdef HAVE_SYSTEM_SLEEP_PM_OPS
8011 static int igb_suspend(struct device *dev)
8015 struct pci_dev *pdev = to_pci_dev(dev);
8017 retval = __igb_shutdown(pdev, &wake, 0);
8022 pci_prepare_to_sleep(pdev);
8024 pci_wake_from_d3(pdev, false);
8025 pci_set_power_state(pdev, PCI_D3hot);
8031 static int igb_resume(struct device *dev)
8033 struct pci_dev *pdev = to_pci_dev(dev);
8034 struct net_device *netdev = pci_get_drvdata(pdev);
8035 struct igb_adapter *adapter = netdev_priv(netdev);
8036 struct e1000_hw *hw = &adapter->hw;
8039 pci_set_power_state(pdev, PCI_D0);
8040 pci_restore_state(pdev);
8041 pci_save_state(pdev);
8043 err = pci_enable_device_mem(pdev);
8045 dev_err(pci_dev_to_dev(pdev),
8046 "igb: Cannot enable PCI device from suspend\n");
8049 pci_set_master(pdev);
8051 pci_enable_wake(pdev, PCI_D3hot, 0);
8052 pci_enable_wake(pdev, PCI_D3cold, 0);
8054 #ifdef CONFIG_PM_RUNTIME
8055 if (!rtnl_is_locked()) {
8057 * shut up ASSERT_RTNL() warning in
8058 * netif_set_real_num_tx/rx_queues.
8061 err = igb_init_interrupt_scheme(adapter);
8064 err = igb_init_interrupt_scheme(adapter);
8068 if (igb_init_interrupt_scheme(adapter)) {
8069 #endif /* CONFIG_PM_RUNTIME */
8070 dev_err(pci_dev_to_dev(pdev), "Unable to allocate memory for queues\n");
8076 /* let the f/w know that the h/w is now under the control of the
8078 igb_get_hw_control(adapter);
8080 E1000_WRITE_REG(hw, E1000_WUS, ~0);
8082 if (netdev->flags & IFF_UP) {
8083 err = __igb_open(netdev, true);
8088 netif_device_attach(netdev);
8093 #ifdef CONFIG_PM_RUNTIME
8094 static int igb_runtime_idle(struct device *dev)
8096 struct pci_dev *pdev = to_pci_dev(dev);
8097 struct net_device *netdev = pci_get_drvdata(pdev);
8098 struct igb_adapter *adapter = netdev_priv(netdev);
8100 if (!igb_has_link(adapter))
8101 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
8106 static int igb_runtime_suspend(struct device *dev)
8108 struct pci_dev *pdev = to_pci_dev(dev);
8112 retval = __igb_shutdown(pdev, &wake, 1);
8117 pci_prepare_to_sleep(pdev);
8119 pci_wake_from_d3(pdev, false);
8120 pci_set_power_state(pdev, PCI_D3hot);
8126 static int igb_runtime_resume(struct device *dev)
8128 return igb_resume(dev);
8130 #endif /* CONFIG_PM_RUNTIME */
8131 #endif /* HAVE_SYSTEM_SLEEP_PM_OPS */
8132 #endif /* CONFIG_PM */
8134 #ifdef USE_REBOOT_NOTIFIER
8135 /* only want to do this for 2.4 kernels? */
8136 static int igb_notify_reboot(struct notifier_block *nb, unsigned long event,
8139 struct pci_dev *pdev = NULL;
8146 while ((pdev = pci_find_device(PCI_ANY_ID, PCI_ANY_ID, pdev))) {
8147 if (pci_dev_driver(pdev) == &igb_driver) {
8148 __igb_shutdown(pdev, &wake, 0);
8149 if (event == SYS_POWER_OFF) {
8150 pci_wake_from_d3(pdev, wake);
8151 pci_set_power_state(pdev, PCI_D3hot);
8159 static void igb_shutdown(struct pci_dev *pdev)
8163 __igb_shutdown(pdev, &wake, 0);
8165 if (system_state == SYSTEM_POWER_OFF) {
8166 pci_wake_from_d3(pdev, wake);
8167 pci_set_power_state(pdev, PCI_D3hot);
8170 #endif /* USE_REBOOT_NOTIFIER */
8172 #ifdef CONFIG_NET_POLL_CONTROLLER
8174 * Polling 'interrupt' - used by things like netconsole to send skbs
8175 * without having to re-enable interrupts. It's not called while
8176 * the interrupt routine is executing.
8178 static void igb_netpoll(struct net_device *netdev)
8180 struct igb_adapter *adapter = netdev_priv(netdev);
8181 struct e1000_hw *hw = &adapter->hw;
8182 struct igb_q_vector *q_vector;
8185 for (i = 0; i < adapter->num_q_vectors; i++) {
8186 q_vector = adapter->q_vector[i];
8187 if (adapter->msix_entries)
8188 E1000_WRITE_REG(hw, E1000_EIMC, q_vector->eims_value);
8190 igb_irq_disable(adapter);
8191 napi_schedule(&q_vector->napi);
8194 #endif /* CONFIG_NET_POLL_CONTROLLER */
8197 #define E1000_DEV_ID_82576_VF 0x10CA
8199 * igb_io_error_detected - called when PCI error is detected
8200 * @pdev: Pointer to PCI device
8201 * @state: The current pci connection state
8203 * This function is called after a PCI bus error affecting
8204 * this device has been detected.
8206 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
8207 pci_channel_state_t state)
8209 struct net_device *netdev = pci_get_drvdata(pdev);
8210 struct igb_adapter *adapter = netdev_priv(netdev);
8212 #ifdef CONFIG_PCI_IOV__UNUSED
8213 struct pci_dev *bdev, *vfdev;
8214 u32 dw0, dw1, dw2, dw3;
8216 u16 req_id, pf_func;
8218 if (!(adapter->flags & IGB_FLAG_DETECT_BAD_DMA))
8219 goto skip_bad_vf_detection;
8221 bdev = pdev->bus->self;
8222 while (bdev && (bdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT))
8223 bdev = bdev->bus->self;
8226 goto skip_bad_vf_detection;
8228 pos = pci_find_ext_capability(bdev, PCI_EXT_CAP_ID_ERR);
8230 goto skip_bad_vf_detection;
8232 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG, &dw0);
8233 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 4, &dw1);
8234 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 8, &dw2);
8235 pci_read_config_dword(bdev, pos + PCI_ERR_HEADER_LOG + 12, &dw3);
8238 /* On the 82576 if bit 7 of the requestor ID is set then it's a VF */
8239 if (!(req_id & 0x0080))
8240 goto skip_bad_vf_detection;
8242 pf_func = req_id & 0x01;
8243 if ((pf_func & 1) == (pdev->devfn & 1)) {
8245 vf = (req_id & 0x7F) >> 1;
8246 dev_err(pci_dev_to_dev(pdev),
8247 "VF %d has caused a PCIe error\n", vf);
8248 dev_err(pci_dev_to_dev(pdev),
8249 "TLP: dw0: %8.8x\tdw1: %8.8x\tdw2: "
8250 "%8.8x\tdw3: %8.8x\n",
8251 dw0, dw1, dw2, dw3);
8253 /* Find the pci device of the offending VF */
8254 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL,
8255 E1000_DEV_ID_82576_VF, NULL);
8257 if (vfdev->devfn == (req_id & 0xFF))
8259 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL,
8260 E1000_DEV_ID_82576_VF, vfdev);
8263 * There's a slim chance the VF could have been hot plugged,
8264 * so if it is no longer present we don't need to issue the
8265 * VFLR. Just clean up the AER in that case.
8268 dev_err(pci_dev_to_dev(pdev),
8269 "Issuing VFLR to VF %d\n", vf);
8270 pci_write_config_dword(vfdev, 0xA8, 0x00008000);
8273 pci_cleanup_aer_uncorrect_error_status(pdev);
8277 * Even though the error may have occurred on the other port
8278 * we still need to increment the vf error reference count for
8279 * both ports because the I/O resume function will be called
8282 adapter->vferr_refcount++;
8284 return PCI_ERS_RESULT_RECOVERED;
8286 skip_bad_vf_detection:
8287 #endif /* CONFIG_PCI_IOV */
8289 netif_device_detach(netdev);
8291 if (state == pci_channel_io_perm_failure)
8292 return PCI_ERS_RESULT_DISCONNECT;
8294 if (netif_running(netdev))
8296 pci_disable_device(pdev);
8298 /* Request a slot slot reset. */
8299 return PCI_ERS_RESULT_NEED_RESET;
8303 * igb_io_slot_reset - called after the pci bus has been reset.
8304 * @pdev: Pointer to PCI device
8306 * Restart the card from scratch, as if from a cold-boot. Implementation
8307 * resembles the first-half of the igb_resume routine.
8309 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
8311 struct net_device *netdev = pci_get_drvdata(pdev);
8312 struct igb_adapter *adapter = netdev_priv(netdev);
8313 struct e1000_hw *hw = &adapter->hw;
8314 pci_ers_result_t result;
8316 if (pci_enable_device_mem(pdev)) {
8317 dev_err(pci_dev_to_dev(pdev),
8318 "Cannot re-enable PCI device after reset.\n");
8319 result = PCI_ERS_RESULT_DISCONNECT;
8321 pci_set_master(pdev);
8322 pci_restore_state(pdev);
8323 pci_save_state(pdev);
8325 pci_enable_wake(pdev, PCI_D3hot, 0);
8326 pci_enable_wake(pdev, PCI_D3cold, 0);
8328 schedule_work(&adapter->reset_task);
8329 E1000_WRITE_REG(hw, E1000_WUS, ~0);
8330 result = PCI_ERS_RESULT_RECOVERED;
8333 pci_cleanup_aer_uncorrect_error_status(pdev);
8339 * igb_io_resume - called when traffic can start flowing again.
8340 * @pdev: Pointer to PCI device
8342 * This callback is called when the error recovery driver tells us that
8343 * its OK to resume normal operation. Implementation resembles the
8344 * second-half of the igb_resume routine.
8346 static void igb_io_resume(struct pci_dev *pdev)
8348 struct net_device *netdev = pci_get_drvdata(pdev);
8349 struct igb_adapter *adapter = netdev_priv(netdev);
8351 if (adapter->vferr_refcount) {
8352 dev_info(pci_dev_to_dev(pdev), "Resuming after VF err\n");
8353 adapter->vferr_refcount--;
8357 if (netif_running(netdev)) {
8358 if (igb_up(adapter)) {
8359 dev_err(pci_dev_to_dev(pdev), "igb_up failed after reset\n");
8364 netif_device_attach(netdev);
8366 /* let the f/w know that the h/w is now under the control of the
8368 igb_get_hw_control(adapter);
8371 #endif /* HAVE_PCI_ERS */
8373 int igb_add_mac_filter(struct igb_adapter *adapter, u8 *addr, u16 queue)
8375 struct e1000_hw *hw = &adapter->hw;
8378 if (is_zero_ether_addr(addr))
8381 for (i = 0; i < hw->mac.rar_entry_count; i++) {
8382 if (adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE)
8384 adapter->mac_table[i].state = (IGB_MAC_STATE_MODIFIED |
8385 IGB_MAC_STATE_IN_USE);
8386 memcpy(adapter->mac_table[i].addr, addr, ETH_ALEN);
8387 adapter->mac_table[i].queue = queue;
8388 igb_sync_mac_table(adapter);
8393 int igb_del_mac_filter(struct igb_adapter *adapter, u8* addr, u16 queue)
8395 /* search table for addr, if found, set to 0 and sync */
8397 struct e1000_hw *hw = &adapter->hw;
8399 if (is_zero_ether_addr(addr))
8401 for (i = 0; i < hw->mac.rar_entry_count; i++) {
8402 if (!compare_ether_addr(addr, adapter->mac_table[i].addr) &&
8403 adapter->mac_table[i].queue == queue) {
8404 adapter->mac_table[i].state = IGB_MAC_STATE_MODIFIED;
8405 memset(adapter->mac_table[i].addr, 0, ETH_ALEN);
8406 adapter->mac_table[i].queue = 0;
8407 igb_sync_mac_table(adapter);
8413 static int igb_set_vf_mac(struct igb_adapter *adapter,
8414 int vf, unsigned char *mac_addr)
8416 igb_del_mac_filter(adapter, adapter->vf_data[vf].vf_mac_addresses, vf);
8417 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
8419 igb_add_mac_filter(adapter, mac_addr, vf);
8425 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
8427 struct igb_adapter *adapter = netdev_priv(netdev);
8428 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
8430 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
8431 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
8432 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
8433 " change effective.\n");
8434 if (test_bit(__IGB_DOWN, &adapter->state)) {
8435 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
8436 " but the PF device is not up.\n");
8437 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
8438 " attempting to use the VF device.\n");
8440 return igb_set_vf_mac(adapter, vf, mac);
8443 static int igb_link_mbps(int internal_link_speed)
8445 switch (internal_link_speed) {
8455 static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
8462 /* Calculate the rate factor values to set */
8463 rf_int = link_speed / tx_rate;
8464 rf_dec = (link_speed - (rf_int * tx_rate));
8465 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
8467 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
8468 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
8469 E1000_RTTBCNRC_RF_INT_MASK);
8470 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
8475 E1000_WRITE_REG(hw, E1000_RTTDQSEL, vf); /* vf X uses queue X */
8477 * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
8478 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
8480 E1000_WRITE_REG(hw, E1000_RTTBCNRM(0), 0x14);
8481 E1000_WRITE_REG(hw, E1000_RTTBCNRC, bcnrc_val);
8484 static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
8486 int actual_link_speed, i;
8487 bool reset_rate = false;
8489 /* VF TX rate limit was not set */
8490 if ((adapter->vf_rate_link_speed == 0) ||
8491 (adapter->hw.mac.type != e1000_82576))
8494 actual_link_speed = igb_link_mbps(adapter->link_speed);
8495 if (actual_link_speed != adapter->vf_rate_link_speed) {
8497 adapter->vf_rate_link_speed = 0;
8498 dev_info(&adapter->pdev->dev,
8499 "Link speed has been changed. VF Transmit rate is disabled\n");
8502 for (i = 0; i < adapter->vfs_allocated_count; i++) {
8504 adapter->vf_data[i].tx_rate = 0;
8506 igb_set_vf_rate_limit(&adapter->hw, i,
8507 adapter->vf_data[i].tx_rate, actual_link_speed);
8511 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
8513 struct igb_adapter *adapter = netdev_priv(netdev);
8514 struct e1000_hw *hw = &adapter->hw;
8515 int actual_link_speed;
8517 if (hw->mac.type != e1000_82576)
8520 actual_link_speed = igb_link_mbps(adapter->link_speed);
8521 if ((vf >= adapter->vfs_allocated_count) ||
8522 (!(E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) ||
8523 (tx_rate < 0) || (tx_rate > actual_link_speed))
8526 adapter->vf_rate_link_speed = actual_link_speed;
8527 adapter->vf_data[vf].tx_rate = (u16)tx_rate;
8528 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
8533 static int igb_ndo_get_vf_config(struct net_device *netdev,
8534 int vf, struct ifla_vf_info *ivi)
8536 struct igb_adapter *adapter = netdev_priv(netdev);
8537 if (vf >= adapter->vfs_allocated_count)
8540 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
8541 ivi->tx_rate = adapter->vf_data[vf].tx_rate;
8542 ivi->vlan = adapter->vf_data[vf].pf_vlan;
8543 ivi->qos = adapter->vf_data[vf].pf_qos;
8547 static void igb_vmm_control(struct igb_adapter *adapter)
8549 struct e1000_hw *hw = &adapter->hw;
8552 switch (hw->mac.type) {
8555 /* replication is not supported for 82575 */
8558 /* notify HW that the MAC is adding vlan tags */
8559 reg = E1000_READ_REG(hw, E1000_DTXCTL);
8560 reg |= (E1000_DTXCTL_VLAN_ADDED |
8561 E1000_DTXCTL_SPOOF_INT);
8562 E1000_WRITE_REG(hw, E1000_DTXCTL, reg);
8564 /* enable replication vlan tag stripping */
8565 reg = E1000_READ_REG(hw, E1000_RPLOLR);
8566 reg |= E1000_RPLOLR_STRVLAN;
8567 E1000_WRITE_REG(hw, E1000_RPLOLR, reg);
8569 /* none of the above registers are supported by i350 */
8573 /* Enable Malicious Driver Detection */
8574 if ((hw->mac.type == e1000_i350) && (adapter->vfs_allocated_count) &&
8576 igb_enable_mdd(adapter);
8578 /* enable replication and loopback support */
8579 e1000_vmdq_set_loopback_pf(hw, adapter->vfs_allocated_count ||
8580 adapter->vmdq_pools);
8582 e1000_vmdq_set_anti_spoofing_pf(hw, adapter->vfs_allocated_count ||
8583 adapter->vmdq_pools,
8584 adapter->vfs_allocated_count);
8585 e1000_vmdq_set_replication_pf(hw, adapter->vfs_allocated_count ||
8586 adapter->vmdq_pools);
8589 static void igb_init_fw(struct igb_adapter *adapter)
8591 struct e1000_fw_drv_info fw_cmd;
8592 struct e1000_hw *hw = &adapter->hw;
8596 mask = E1000_SWFW_PHY0_SM;
8598 if (!hw->mac.ops.acquire_swfw_sync(hw, mask)) {
8599 for (i = 0; i <= FW_MAX_RETRIES; i++) {
8600 E1000_WRITE_REG(hw, E1000_FWSTS, E1000_FWSTS_FWRI);
8601 fw_cmd.hdr.cmd = FW_CMD_DRV_INFO;
8602 fw_cmd.hdr.buf_len = FW_CMD_DRV_INFO_LEN;
8603 fw_cmd.hdr.cmd_or_resp.cmd_resv = FW_CMD_RESERVED;
8604 fw_cmd.port_num = hw->bus.func;
8605 fw_cmd.drv_version = FW_FAMILY_DRV_VER;
8606 fw_cmd.hdr.checksum = 0;
8607 fw_cmd.hdr.checksum = e1000_calculate_checksum((u8 *)&fw_cmd,
8609 fw_cmd.hdr.buf_len));
8610 e1000_host_interface_command(hw, (u8*)&fw_cmd,
8612 if (fw_cmd.hdr.cmd_or_resp.ret_status == FW_STATUS_SUCCESS)
8616 dev_warn(pci_dev_to_dev(adapter->pdev),
8617 "Unable to get semaphore, firmware init failed.\n");
8618 hw->mac.ops.release_swfw_sync(hw, mask);
8621 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
8623 struct e1000_hw *hw = &adapter->hw;
8627 if (hw->mac.type > e1000_82580) {
8628 if (adapter->dmac != IGB_DMAC_DISABLE) {
8631 /* force threshold to 0. */
8632 E1000_WRITE_REG(hw, E1000_DMCTXTH, 0);
8635 * DMA Coalescing high water mark needs to be greater
8636 * than the Rx threshold. Set hwm to PBA - max frame
8637 * size in 16B units, capping it at PBA - 6KB.
8639 hwm = 64 * pba - adapter->max_frame_size / 16;
8640 if (hwm < 64 * (pba - 6))
8641 hwm = 64 * (pba - 6);
8642 reg = E1000_READ_REG(hw, E1000_FCRTC);
8643 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
8644 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
8645 & E1000_FCRTC_RTH_COAL_MASK);
8646 E1000_WRITE_REG(hw, E1000_FCRTC, reg);
8649 * Set the DMA Coalescing Rx threshold to PBA - 2 * max
8650 * frame size, capping it at PBA - 10KB.
8652 dmac_thr = pba - adapter->max_frame_size / 512;
8653 if (dmac_thr < pba - 10)
8654 dmac_thr = pba - 10;
8655 reg = E1000_READ_REG(hw, E1000_DMACR);
8656 reg &= ~E1000_DMACR_DMACTHR_MASK;
8657 reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
8658 & E1000_DMACR_DMACTHR_MASK);
8660 /* transition to L0x or L1 if available..*/
8661 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
8663 /* watchdog timer= msec values in 32usec intervals */
8664 reg |= ((adapter->dmac) >> 5);
8665 E1000_WRITE_REG(hw, E1000_DMACR, reg);
8667 /* no lower threshold to disable coalescing(smart fifb)-UTRESH=0*/
8668 E1000_WRITE_REG(hw, E1000_DMCRTRH, 0);
8671 * This sets the time to wait before requesting transition to
8672 * low power state to number of usecs needed to receive 1 512
8673 * byte frame at gigabit line rate
8675 reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
8677 E1000_WRITE_REG(hw, E1000_DMCTLX, reg);
8679 /* free space in tx packet buffer to wake from DMA coal */
8680 E1000_WRITE_REG(hw, E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
8681 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
8683 /* make low power state decision controlled by DMA coal */
8684 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
8685 reg &= ~E1000_PCIEMISC_LX_DECISION;
8686 E1000_WRITE_REG(hw, E1000_PCIEMISC, reg);
8687 } /* endif adapter->dmac is not disabled */
8688 } else if (hw->mac.type == e1000_82580) {
8689 u32 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
8690 E1000_WRITE_REG(hw, E1000_PCIEMISC,
8691 reg & ~E1000_PCIEMISC_LX_DECISION);
8692 E1000_WRITE_REG(hw, E1000_DMACR, 0);
8700 * igb_probe - Device Initialization Routine
8701 * @pdev: PCI device information struct
8702 * @ent: entry in igb_pci_tbl
8704 * Returns 0 on success, negative on failure
8706 * igb_probe initializes an adapter identified by a pci_dev structure.
8707 * The OS initialization, configuring of the adapter private structure,
8708 * and a hardware reset occur.
8710 int igb_kni_probe(struct pci_dev *pdev,
8711 struct net_device **lad_dev)
8713 struct net_device *netdev;
8714 struct igb_adapter *adapter;
8715 struct e1000_hw *hw;
8716 u16 eeprom_data = 0;
8717 u8 pba_str[E1000_PBANUM_LENGTH];
8719 static int global_quad_port_a; /* global quad port a indication */
8720 int i, err, pci_using_dac = 0;
8721 static int cards_found;
8723 err = pci_enable_device_mem(pdev);
8729 err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
8731 err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(64));
8735 err = dma_set_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
8737 err = dma_set_coherent_mask(pci_dev_to_dev(pdev), DMA_BIT_MASK(32));
8739 IGB_ERR("No usable DMA configuration, "
8746 #ifndef HAVE_ASPM_QUIRKS
8747 /* 82575 requires that the pci-e link partner disable the L0s state */
8748 switch (pdev->device) {
8749 case E1000_DEV_ID_82575EB_COPPER:
8750 case E1000_DEV_ID_82575EB_FIBER_SERDES:
8751 case E1000_DEV_ID_82575GB_QUAD_COPPER:
8752 pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S);
8757 #endif /* HAVE_ASPM_QUIRKS */
8758 err = pci_request_selected_regions(pdev,
8759 pci_select_bars(pdev,
8765 pci_enable_pcie_error_reporting(pdev);
8767 pci_set_master(pdev);
8771 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
8774 netdev = alloc_etherdev(sizeof(struct igb_adapter));
8775 #endif /* HAVE_TX_MQ */
8777 goto err_alloc_etherdev;
8780 SET_MODULE_OWNER(netdev);
8781 SET_NETDEV_DEV(netdev, &pdev->dev);
8783 //pci_set_drvdata(pdev, netdev);
8785 adapter = netdev_priv(netdev);
8786 adapter->netdev = netdev;
8787 adapter->pdev = pdev;
8790 adapter->port_num = hw->bus.func;
8791 adapter->msg_enable = (1 << debug) - 1;
8794 err = pci_save_state(pdev);
8799 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
8800 pci_resource_len(pdev, 0));
8804 #ifdef HAVE_NET_DEVICE_OPS
8805 netdev->netdev_ops = &igb_netdev_ops;
8806 #else /* HAVE_NET_DEVICE_OPS */
8807 netdev->open = &igb_open;
8808 netdev->stop = &igb_close;
8809 netdev->get_stats = &igb_get_stats;
8810 #ifdef HAVE_SET_RX_MODE
8811 netdev->set_rx_mode = &igb_set_rx_mode;
8813 netdev->set_multicast_list = &igb_set_rx_mode;
8814 netdev->set_mac_address = &igb_set_mac;
8815 netdev->change_mtu = &igb_change_mtu;
8816 netdev->do_ioctl = &igb_ioctl;
8817 #ifdef HAVE_TX_TIMEOUT
8818 netdev->tx_timeout = &igb_tx_timeout;
8820 netdev->vlan_rx_register = igb_vlan_mode;
8821 netdev->vlan_rx_add_vid = igb_vlan_rx_add_vid;
8822 netdev->vlan_rx_kill_vid = igb_vlan_rx_kill_vid;
8823 #ifdef CONFIG_NET_POLL_CONTROLLER
8824 netdev->poll_controller = igb_netpoll;
8826 netdev->hard_start_xmit = &igb_xmit_frame;
8827 #endif /* HAVE_NET_DEVICE_OPS */
8828 igb_set_ethtool_ops(netdev);
8829 #ifdef HAVE_TX_TIMEOUT
8830 netdev->watchdog_timeo = 5 * HZ;
8833 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
8835 adapter->bd_number = cards_found;
8837 /* setup the private structure */
8838 err = igb_sw_init(adapter);
8842 e1000_get_bus_info(hw);
8844 hw->phy.autoneg_wait_to_complete = FALSE;
8845 hw->mac.adaptive_ifs = FALSE;
8847 /* Copper options */
8848 if (hw->phy.media_type == e1000_media_type_copper) {
8850 hw->phy.mdix = ETH_TP_MDI_INVALID;
8852 hw->phy.mdix = AUTO_ALL_MODES;
8853 #endif /* ETH_TP_MDI_X */
8854 hw->phy.disable_polarity_correction = FALSE;
8855 hw->phy.ms_type = e1000_ms_hw_default;
8858 if (e1000_check_reset_block(hw))
8859 dev_info(pci_dev_to_dev(pdev),
8860 "PHY reset is blocked due to SOL/IDER session.\n");
8863 * features is initialized to 0 in allocation, it might have bits
8864 * set by igb_sw_init so we should use an or instead of an
8867 netdev->features |= NETIF_F_SG |
8869 #ifdef NETIF_F_IPV6_CSUM
8877 #endif /* NETIF_F_TSO */
8878 #ifdef NETIF_F_RXHASH
8881 #ifdef HAVE_NDO_SET_FEATURES
8884 NETIF_F_HW_VLAN_RX |
8887 #ifdef HAVE_NDO_SET_FEATURES
8888 /* copy netdev features into list of user selectable features */
8889 netdev->hw_features |= netdev->features;
8892 /* give us the option of enabling LRO later */
8893 netdev->hw_features |= NETIF_F_LRO;
8898 /* this is only needed on kernels prior to 2.6.39 */
8899 netdev->features |= NETIF_F_GRO;
8903 /* set this bit last since it cannot be part of hw_features */
8904 netdev->features |= NETIF_F_HW_VLAN_FILTER;
8906 #ifdef HAVE_NETDEV_VLAN_FEATURES
8907 netdev->vlan_features |= NETIF_F_TSO |
8915 netdev->features |= NETIF_F_HIGHDMA;
8917 if (hw->mac.type >= e1000_82576)
8918 netdev->features |= NETIF_F_SCTP_CSUM;
8921 adapter->en_mng_pt = e1000_enable_mng_pass_thru(hw);
8923 /* before reading the NVM, reset the controller to put the device in a
8924 * known good starting state */
8928 /* make sure the NVM is good */
8929 if (e1000_validate_nvm_checksum(hw) < 0) {
8930 dev_err(pci_dev_to_dev(pdev), "The NVM Checksum Is Not"
8936 /* copy the MAC address out of the NVM */
8937 if (e1000_read_mac_addr(hw))
8938 dev_err(pci_dev_to_dev(pdev), "NVM Read Error\n");
8939 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
8940 #ifdef ETHTOOL_GPERMADDR
8941 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
8943 if (!is_valid_ether_addr(netdev->perm_addr)) {
8945 if (!is_valid_ether_addr(netdev->dev_addr)) {
8947 dev_err(pci_dev_to_dev(pdev), "Invalid MAC Address\n");
8952 memcpy(&adapter->mac_table[0].addr, hw->mac.addr, netdev->addr_len);
8953 adapter->mac_table[0].queue = adapter->vfs_allocated_count;
8954 adapter->mac_table[0].state = (IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE);
8955 igb_rar_set(adapter, 0);
8957 /* get firmware version for ethtool -i */
8958 e1000_read_nvm(&adapter->hw, 5, 1, &adapter->fw_version);
8960 setup_timer(&adapter->watchdog_timer, &igb_watchdog,
8961 (unsigned long) adapter);
8962 if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
8963 setup_timer(&adapter->dma_err_timer, &igb_dma_err_timer,
8964 (unsigned long) adapter);
8965 setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
8966 (unsigned long) adapter);
8968 INIT_WORK(&adapter->reset_task, igb_reset_task);
8969 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
8970 if (adapter->flags & IGB_FLAG_DETECT_BAD_DMA)
8971 INIT_WORK(&adapter->dma_err_task, igb_dma_err_task);
8974 /* Initialize link properties that are user-changeable */
8975 adapter->fc_autoneg = true;
8976 hw->mac.autoneg = true;
8977 hw->phy.autoneg_advertised = 0x2f;
8979 hw->fc.requested_mode = e1000_fc_default;
8980 hw->fc.current_mode = e1000_fc_default;
8982 e1000_validate_mdi_setting(hw);
8984 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
8985 * enable the ACPI Magic Packet filter
8988 if (hw->bus.func == 0)
8989 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
8990 else if (hw->mac.type >= e1000_82580)
8991 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
8992 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
8994 else if (hw->bus.func == 1)
8995 e1000_read_nvm(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
8997 if (eeprom_data & IGB_EEPROM_APME)
8998 adapter->eeprom_wol |= E1000_WUFC_MAG;
9000 /* now that we have the eeprom settings, apply the special cases where
9001 * the eeprom may be wrong or the board simply won't support wake on
9002 * lan on a particular port */
9003 switch (pdev->device) {
9004 case E1000_DEV_ID_82575GB_QUAD_COPPER:
9005 adapter->eeprom_wol = 0;
9007 case E1000_DEV_ID_82575EB_FIBER_SERDES:
9008 case E1000_DEV_ID_82576_FIBER:
9009 case E1000_DEV_ID_82576_SERDES:
9010 /* Wake events only supported on port A for dual fiber
9011 * regardless of eeprom setting */
9012 if (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_FUNC_1)
9013 adapter->eeprom_wol = 0;
9015 case E1000_DEV_ID_82576_QUAD_COPPER:
9016 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
9017 /* if quad port adapter, disable WoL on all but port A */
9018 if (global_quad_port_a != 0)
9019 adapter->eeprom_wol = 0;
9021 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
9022 /* Reset for multiple quad port adapters */
9023 if (++global_quad_port_a == 4)
9024 global_quad_port_a = 0;
9028 /* initialize the wol settings based on the eeprom settings */
9029 adapter->wol = adapter->eeprom_wol;
9031 device_set_wakeup_enable(pci_dev_to_dev(adapter->pdev), adapter->wol);
9033 /* reset the hardware with the new settings */
9036 /* let the f/w know that the h/w is now under the control of the
9038 igb_get_hw_control(adapter);
9040 strncpy(netdev->name, "eth%d", IFNAMSIZ);
9041 err = register_netdev(netdev);
9045 #ifdef CONFIG_IGB_VMDQ_NETDEV
9046 err = igb_init_vmdq_netdevs(adapter);
9050 /* carrier off reporting is important to ethtool even BEFORE open */
9051 netif_carrier_off(netdev);
9054 if (dca_add_requester(&pdev->dev) == E1000_SUCCESS) {
9055 adapter->flags |= IGB_FLAG_DCA_ENABLED;
9056 dev_info(pci_dev_to_dev(pdev), "DCA enabled\n");
9057 igb_setup_dca(adapter);
9061 #ifdef HAVE_HW_TIME_STAMP
9062 /* do hw tstamp init after resetting */
9063 igb_init_hw_timer(adapter);
9068 dev_info(pci_dev_to_dev(pdev), "Intel(R) Gigabit Ethernet Network Connection\n");
9069 /* print bus type/speed/width info */
9070 dev_info(pci_dev_to_dev(pdev), "%s: (PCIe:%s:%s) ",
9072 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5GT/s" :
9073 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0GT/s" :
9075 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4\n" :
9076 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2\n" :
9077 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1\n" :
9079 dev_info(pci_dev_to_dev(pdev), "%s: MAC: ", netdev->name);
9080 for (i = 0; i < 6; i++)
9081 printk("%2.2x%c", netdev->dev_addr[i], i == 5 ? '\n' : ':');
9083 ret_val = e1000_read_pba_string(hw, pba_str, E1000_PBANUM_LENGTH);
9085 strncpy(pba_str, "Unknown", sizeof(pba_str) - 1);
9086 dev_info(pci_dev_to_dev(pdev), "%s: PBA No: %s\n", netdev->name,
9089 /* Initialize the thermal sensor on i350 devices. */
9090 if (hw->mac.type == e1000_i350 && hw->bus.func == 0) {
9094 * Read the NVM to determine if this i350 device supports an
9095 * external thermal sensor.
9097 e1000_read_nvm(hw, NVM_ETS_CFG, 1, &ets_word);
9098 if (ets_word != 0x0000 && ets_word != 0xFFFF)
9099 adapter->ets = true;
9101 adapter->ets = false;
9104 igb_sysfs_init(adapter);
9107 igb_procfs_init(adapter);
9108 #endif /* IGB_PROCFS */
9109 #endif /* IGB_SYSFS */
9112 adapter->ets = false;
9115 switch (hw->mac.type) {
9117 /* Enable EEE for internal copper PHY devices */
9118 if (hw->phy.media_type == e1000_media_type_copper)
9119 e1000_set_eee_i350(hw);
9121 /* send driver version info to firmware */
9122 igb_init_fw(adapter);
9128 if (netdev->features & NETIF_F_LRO)
9129 dev_info(pci_dev_to_dev(pdev), "Internal LRO is enabled \n");
9131 dev_info(pci_dev_to_dev(pdev), "LRO is disabled \n");
9133 dev_info(pci_dev_to_dev(pdev),
9134 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
9135 adapter->msix_entries ? "MSI-X" :
9136 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
9137 adapter->num_rx_queues, adapter->num_tx_queues);
9142 pm_runtime_put_noidle(&pdev->dev);
9146 //igb_release_hw_control(adapter);
9148 //if (!e1000_check_reset_block(hw))
9149 // e1000_phy_hw_reset(hw);
9151 if (hw->flash_address)
9152 iounmap(hw->flash_address);
9154 //igb_clear_interrupt_scheme(adapter);
9155 //igb_reset_sriov_capability(adapter);
9156 iounmap(hw->hw_addr);
9158 free_netdev(netdev);
9160 //pci_release_selected_regions(pdev,
9161 // pci_select_bars(pdev, IORESOURCE_MEM));
9164 pci_disable_device(pdev);
9169 void igb_kni_remove(struct pci_dev *pdev)
9171 pci_disable_device(pdev);