1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019-2020 Intel Corporation
8 #include <rte_string_fns.h>
10 #include <rte_bus_pci.h>
11 #include <rte_ethdev_driver.h>
12 #include <rte_ethdev_pci.h>
13 #include <rte_malloc.h>
14 #include <rte_alarm.h>
19 #define IGC_INTEL_VENDOR_ID 0x8086
22 * The overhead from MTU to max frame size.
23 * Considering VLAN so tag needs to be counted.
25 #define IGC_ETH_OVERHEAD (RTE_ETHER_HDR_LEN + \
26 RTE_ETHER_CRC_LEN + VLAN_TAG_SIZE)
28 #define IGC_FC_PAUSE_TIME 0x0680
29 #define IGC_LINK_UPDATE_CHECK_TIMEOUT 90 /* 9s */
30 #define IGC_LINK_UPDATE_CHECK_INTERVAL 100 /* ms */
32 #define IGC_MISC_VEC_ID RTE_INTR_VEC_ZERO_OFFSET
33 #define IGC_RX_VEC_START RTE_INTR_VEC_RXTX_OFFSET
34 #define IGC_MSIX_OTHER_INTR_VEC 0 /* MSI-X other interrupt vector */
35 #define IGC_FLAG_NEED_LINK_UPDATE (1u << 0) /* need update link */
37 #define IGC_DEFAULT_RX_FREE_THRESH 32
39 #define IGC_DEFAULT_RX_PTHRESH 8
40 #define IGC_DEFAULT_RX_HTHRESH 8
41 #define IGC_DEFAULT_RX_WTHRESH 4
43 #define IGC_DEFAULT_TX_PTHRESH 8
44 #define IGC_DEFAULT_TX_HTHRESH 1
45 #define IGC_DEFAULT_TX_WTHRESH 16
47 /* MSI-X other interrupt vector */
48 #define IGC_MSIX_OTHER_INTR_VEC 0
50 /* External VLAN Enable bit mask */
51 #define IGC_CTRL_EXT_EXT_VLAN (1u << 26)
53 /* Per Queue Good Packets Received Count */
54 #define IGC_PQGPRC(idx) (0x10010 + 0x100 * (idx))
55 /* Per Queue Good Octets Received Count */
56 #define IGC_PQGORC(idx) (0x10018 + 0x100 * (idx))
57 /* Per Queue Good Octets Transmitted Count */
58 #define IGC_PQGOTC(idx) (0x10034 + 0x100 * (idx))
59 /* Per Queue Multicast Packets Received Count */
60 #define IGC_PQMPRC(idx) (0x10038 + 0x100 * (idx))
61 /* Transmit Queue Drop Packet Count */
62 #define IGC_TQDPC(idx) (0xe030 + 0x40 * (idx))
64 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
65 #define U32_0_IN_U64 0 /* lower bytes of u64 */
66 #define U32_1_IN_U64 1 /* higher bytes of u64 */
68 #define U32_0_IN_U64 1
69 #define U32_1_IN_U64 0
72 #define IGC_ALARM_INTERVAL 8000000u
73 /* us, about 13.6s some per-queue registers will wrap around back to 0. */
75 static const struct rte_eth_desc_lim rx_desc_lim = {
76 .nb_max = IGC_MAX_RXD,
77 .nb_min = IGC_MIN_RXD,
78 .nb_align = IGC_RXD_ALIGN,
81 static const struct rte_eth_desc_lim tx_desc_lim = {
82 .nb_max = IGC_MAX_TXD,
83 .nb_min = IGC_MIN_TXD,
84 .nb_align = IGC_TXD_ALIGN,
85 .nb_seg_max = IGC_TX_MAX_SEG,
86 .nb_mtu_seg_max = IGC_TX_MAX_MTU_SEG,
89 static const struct rte_pci_id pci_id_igc_map[] = {
90 { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_LM) },
91 { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_V) },
92 { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_I) },
93 { RTE_PCI_DEVICE(IGC_INTEL_VENDOR_ID, IGC_DEV_ID_I225_K) },
94 { .vendor_id = 0, /* sentinel */ },
97 /* store statistics names and its offset in stats structure */
98 struct rte_igc_xstats_name_off {
99 char name[RTE_ETH_XSTATS_NAME_SIZE];
103 static const struct rte_igc_xstats_name_off rte_igc_stats_strings[] = {
104 {"rx_crc_errors", offsetof(struct igc_hw_stats, crcerrs)},
105 {"rx_align_errors", offsetof(struct igc_hw_stats, algnerrc)},
106 {"rx_errors", offsetof(struct igc_hw_stats, rxerrc)},
107 {"rx_missed_packets", offsetof(struct igc_hw_stats, mpc)},
108 {"tx_single_collision_packets", offsetof(struct igc_hw_stats, scc)},
109 {"tx_multiple_collision_packets", offsetof(struct igc_hw_stats, mcc)},
110 {"tx_excessive_collision_packets", offsetof(struct igc_hw_stats,
112 {"tx_late_collisions", offsetof(struct igc_hw_stats, latecol)},
113 {"tx_total_collisions", offsetof(struct igc_hw_stats, colc)},
114 {"tx_deferred_packets", offsetof(struct igc_hw_stats, dc)},
115 {"tx_no_carrier_sense_packets", offsetof(struct igc_hw_stats, tncrs)},
116 {"tx_discarded_packets", offsetof(struct igc_hw_stats, htdpmc)},
117 {"rx_length_errors", offsetof(struct igc_hw_stats, rlec)},
118 {"rx_xon_packets", offsetof(struct igc_hw_stats, xonrxc)},
119 {"tx_xon_packets", offsetof(struct igc_hw_stats, xontxc)},
120 {"rx_xoff_packets", offsetof(struct igc_hw_stats, xoffrxc)},
121 {"tx_xoff_packets", offsetof(struct igc_hw_stats, xofftxc)},
122 {"rx_flow_control_unsupported_packets", offsetof(struct igc_hw_stats,
124 {"rx_size_64_packets", offsetof(struct igc_hw_stats, prc64)},
125 {"rx_size_65_to_127_packets", offsetof(struct igc_hw_stats, prc127)},
126 {"rx_size_128_to_255_packets", offsetof(struct igc_hw_stats, prc255)},
127 {"rx_size_256_to_511_packets", offsetof(struct igc_hw_stats, prc511)},
128 {"rx_size_512_to_1023_packets", offsetof(struct igc_hw_stats,
130 {"rx_size_1024_to_max_packets", offsetof(struct igc_hw_stats,
132 {"rx_broadcast_packets", offsetof(struct igc_hw_stats, bprc)},
133 {"rx_multicast_packets", offsetof(struct igc_hw_stats, mprc)},
134 {"rx_undersize_errors", offsetof(struct igc_hw_stats, ruc)},
135 {"rx_fragment_errors", offsetof(struct igc_hw_stats, rfc)},
136 {"rx_oversize_errors", offsetof(struct igc_hw_stats, roc)},
137 {"rx_jabber_errors", offsetof(struct igc_hw_stats, rjc)},
138 {"rx_no_buffers", offsetof(struct igc_hw_stats, rnbc)},
139 {"rx_management_packets", offsetof(struct igc_hw_stats, mgprc)},
140 {"rx_management_dropped", offsetof(struct igc_hw_stats, mgpdc)},
141 {"tx_management_packets", offsetof(struct igc_hw_stats, mgptc)},
142 {"rx_total_packets", offsetof(struct igc_hw_stats, tpr)},
143 {"tx_total_packets", offsetof(struct igc_hw_stats, tpt)},
144 {"rx_total_bytes", offsetof(struct igc_hw_stats, tor)},
145 {"tx_total_bytes", offsetof(struct igc_hw_stats, tot)},
146 {"tx_size_64_packets", offsetof(struct igc_hw_stats, ptc64)},
147 {"tx_size_65_to_127_packets", offsetof(struct igc_hw_stats, ptc127)},
148 {"tx_size_128_to_255_packets", offsetof(struct igc_hw_stats, ptc255)},
149 {"tx_size_256_to_511_packets", offsetof(struct igc_hw_stats, ptc511)},
150 {"tx_size_512_to_1023_packets", offsetof(struct igc_hw_stats,
152 {"tx_size_1023_to_max_packets", offsetof(struct igc_hw_stats,
154 {"tx_multicast_packets", offsetof(struct igc_hw_stats, mptc)},
155 {"tx_broadcast_packets", offsetof(struct igc_hw_stats, bptc)},
156 {"tx_tso_packets", offsetof(struct igc_hw_stats, tsctc)},
157 {"rx_sent_to_host_packets", offsetof(struct igc_hw_stats, rpthc)},
158 {"tx_sent_by_host_packets", offsetof(struct igc_hw_stats, hgptc)},
159 {"interrupt_assert_count", offsetof(struct igc_hw_stats, iac)},
160 {"rx_descriptor_lower_threshold",
161 offsetof(struct igc_hw_stats, icrxdmtc)},
164 #define IGC_NB_XSTATS (sizeof(rte_igc_stats_strings) / \
165 sizeof(rte_igc_stats_strings[0]))
167 static int eth_igc_configure(struct rte_eth_dev *dev);
168 static int eth_igc_link_update(struct rte_eth_dev *dev, int wait_to_complete);
169 static void eth_igc_stop(struct rte_eth_dev *dev);
170 static int eth_igc_start(struct rte_eth_dev *dev);
171 static int eth_igc_set_link_up(struct rte_eth_dev *dev);
172 static int eth_igc_set_link_down(struct rte_eth_dev *dev);
173 static void eth_igc_close(struct rte_eth_dev *dev);
174 static int eth_igc_reset(struct rte_eth_dev *dev);
175 static int eth_igc_promiscuous_enable(struct rte_eth_dev *dev);
176 static int eth_igc_promiscuous_disable(struct rte_eth_dev *dev);
177 static int eth_igc_fw_version_get(struct rte_eth_dev *dev,
178 char *fw_version, size_t fw_size);
179 static int eth_igc_infos_get(struct rte_eth_dev *dev,
180 struct rte_eth_dev_info *dev_info);
181 static int eth_igc_led_on(struct rte_eth_dev *dev);
182 static int eth_igc_led_off(struct rte_eth_dev *dev);
183 static const uint32_t *eth_igc_supported_ptypes_get(struct rte_eth_dev *dev);
184 static int eth_igc_rar_set(struct rte_eth_dev *dev,
185 struct rte_ether_addr *mac_addr, uint32_t index, uint32_t pool);
186 static void eth_igc_rar_clear(struct rte_eth_dev *dev, uint32_t index);
187 static int eth_igc_default_mac_addr_set(struct rte_eth_dev *dev,
188 struct rte_ether_addr *addr);
189 static int eth_igc_set_mc_addr_list(struct rte_eth_dev *dev,
190 struct rte_ether_addr *mc_addr_set,
191 uint32_t nb_mc_addr);
192 static int eth_igc_allmulticast_enable(struct rte_eth_dev *dev);
193 static int eth_igc_allmulticast_disable(struct rte_eth_dev *dev);
194 static int eth_igc_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
195 static int eth_igc_stats_get(struct rte_eth_dev *dev,
196 struct rte_eth_stats *rte_stats);
197 static int eth_igc_xstats_get(struct rte_eth_dev *dev,
198 struct rte_eth_xstat *xstats, unsigned int n);
199 static int eth_igc_xstats_get_by_id(struct rte_eth_dev *dev,
201 uint64_t *values, unsigned int n);
202 static int eth_igc_xstats_get_names(struct rte_eth_dev *dev,
203 struct rte_eth_xstat_name *xstats_names,
205 static int eth_igc_xstats_get_names_by_id(struct rte_eth_dev *dev,
206 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids,
208 static int eth_igc_xstats_reset(struct rte_eth_dev *dev);
210 eth_igc_queue_stats_mapping_set(struct rte_eth_dev *dev,
211 uint16_t queue_id, uint8_t stat_idx, uint8_t is_rx);
213 static const struct eth_dev_ops eth_igc_ops = {
214 .dev_configure = eth_igc_configure,
215 .link_update = eth_igc_link_update,
216 .dev_stop = eth_igc_stop,
217 .dev_start = eth_igc_start,
218 .dev_close = eth_igc_close,
219 .dev_reset = eth_igc_reset,
220 .dev_set_link_up = eth_igc_set_link_up,
221 .dev_set_link_down = eth_igc_set_link_down,
222 .promiscuous_enable = eth_igc_promiscuous_enable,
223 .promiscuous_disable = eth_igc_promiscuous_disable,
224 .allmulticast_enable = eth_igc_allmulticast_enable,
225 .allmulticast_disable = eth_igc_allmulticast_disable,
226 .fw_version_get = eth_igc_fw_version_get,
227 .dev_infos_get = eth_igc_infos_get,
228 .dev_led_on = eth_igc_led_on,
229 .dev_led_off = eth_igc_led_off,
230 .dev_supported_ptypes_get = eth_igc_supported_ptypes_get,
231 .mtu_set = eth_igc_mtu_set,
232 .mac_addr_add = eth_igc_rar_set,
233 .mac_addr_remove = eth_igc_rar_clear,
234 .mac_addr_set = eth_igc_default_mac_addr_set,
235 .set_mc_addr_list = eth_igc_set_mc_addr_list,
237 .rx_queue_setup = eth_igc_rx_queue_setup,
238 .rx_queue_release = eth_igc_rx_queue_release,
239 .rx_queue_count = eth_igc_rx_queue_count,
240 .rx_descriptor_done = eth_igc_rx_descriptor_done,
241 .rx_descriptor_status = eth_igc_rx_descriptor_status,
242 .tx_descriptor_status = eth_igc_tx_descriptor_status,
243 .tx_queue_setup = eth_igc_tx_queue_setup,
244 .tx_queue_release = eth_igc_tx_queue_release,
245 .tx_done_cleanup = eth_igc_tx_done_cleanup,
246 .rxq_info_get = eth_igc_rxq_info_get,
247 .txq_info_get = eth_igc_txq_info_get,
248 .stats_get = eth_igc_stats_get,
249 .xstats_get = eth_igc_xstats_get,
250 .xstats_get_by_id = eth_igc_xstats_get_by_id,
251 .xstats_get_names_by_id = eth_igc_xstats_get_names_by_id,
252 .xstats_get_names = eth_igc_xstats_get_names,
253 .stats_reset = eth_igc_xstats_reset,
254 .xstats_reset = eth_igc_xstats_reset,
255 .queue_stats_mapping_set = eth_igc_queue_stats_mapping_set,
259 * multiple queue mode checking
262 igc_check_mq_mode(struct rte_eth_dev *dev)
264 enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
265 enum rte_eth_tx_mq_mode tx_mq_mode = dev->data->dev_conf.txmode.mq_mode;
267 if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
268 PMD_INIT_LOG(ERR, "SRIOV is not supported.");
272 if (rx_mq_mode != ETH_MQ_RX_NONE &&
273 rx_mq_mode != ETH_MQ_RX_RSS) {
274 /* RSS together with VMDq not supported*/
275 PMD_INIT_LOG(ERR, "RX mode %d is not supported.",
280 /* To no break software that set invalid mode, only display
281 * warning if invalid mode is used.
283 if (tx_mq_mode != ETH_MQ_TX_NONE)
284 PMD_INIT_LOG(WARNING,
285 "TX mode %d is not supported. Due to meaningless in this driver, just ignore",
292 eth_igc_configure(struct rte_eth_dev *dev)
294 struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
297 PMD_INIT_FUNC_TRACE();
299 ret = igc_check_mq_mode(dev);
303 intr->flags |= IGC_FLAG_NEED_LINK_UPDATE;
308 eth_igc_set_link_up(struct rte_eth_dev *dev)
310 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
312 if (hw->phy.media_type == igc_media_type_copper)
313 igc_power_up_phy(hw);
315 igc_power_up_fiber_serdes_link(hw);
320 eth_igc_set_link_down(struct rte_eth_dev *dev)
322 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
324 if (hw->phy.media_type == igc_media_type_copper)
325 igc_power_down_phy(hw);
327 igc_shutdown_fiber_serdes_link(hw);
332 * disable other interrupt
335 igc_intr_other_disable(struct rte_eth_dev *dev)
337 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
338 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
339 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
341 if (rte_intr_allow_others(intr_handle) &&
342 dev->data->dev_conf.intr_conf.lsc) {
343 IGC_WRITE_REG(hw, IGC_EIMC, 1u << IGC_MSIX_OTHER_INTR_VEC);
346 IGC_WRITE_REG(hw, IGC_IMC, ~0);
351 * enable other interrupt
354 igc_intr_other_enable(struct rte_eth_dev *dev)
356 struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
357 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
358 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
359 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
361 if (rte_intr_allow_others(intr_handle) &&
362 dev->data->dev_conf.intr_conf.lsc) {
363 IGC_WRITE_REG(hw, IGC_EIMS, 1u << IGC_MSIX_OTHER_INTR_VEC);
366 IGC_WRITE_REG(hw, IGC_IMS, intr->mask);
371 * It reads ICR and gets interrupt causes, check it and set a bit flag
372 * to update link status.
375 eth_igc_interrupt_get_status(struct rte_eth_dev *dev)
378 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
379 struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
381 /* read-on-clear nic registers here */
382 icr = IGC_READ_REG(hw, IGC_ICR);
385 if (icr & IGC_ICR_LSC)
386 intr->flags |= IGC_FLAG_NEED_LINK_UPDATE;
389 /* return 0 means link status changed, -1 means not changed */
391 eth_igc_link_update(struct rte_eth_dev *dev, int wait_to_complete)
393 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
394 struct rte_eth_link link;
395 int link_check, count;
398 hw->mac.get_link_status = 1;
400 /* possible wait-to-complete in up to 9 seconds */
401 for (count = 0; count < IGC_LINK_UPDATE_CHECK_TIMEOUT; count++) {
402 /* Read the real link status */
403 switch (hw->phy.media_type) {
404 case igc_media_type_copper:
405 /* Do the work to read phy */
406 igc_check_for_link(hw);
407 link_check = !hw->mac.get_link_status;
410 case igc_media_type_fiber:
411 igc_check_for_link(hw);
412 link_check = (IGC_READ_REG(hw, IGC_STATUS) &
416 case igc_media_type_internal_serdes:
417 igc_check_for_link(hw);
418 link_check = hw->mac.serdes_has_link;
424 if (link_check || wait_to_complete == 0)
426 rte_delay_ms(IGC_LINK_UPDATE_CHECK_INTERVAL);
428 memset(&link, 0, sizeof(link));
430 /* Now we check if a transition has happened */
432 uint16_t duplex, speed;
433 hw->mac.ops.get_link_up_info(hw, &speed, &duplex);
434 link.link_duplex = (duplex == FULL_DUPLEX) ?
435 ETH_LINK_FULL_DUPLEX :
436 ETH_LINK_HALF_DUPLEX;
437 link.link_speed = speed;
438 link.link_status = ETH_LINK_UP;
439 link.link_autoneg = !(dev->data->dev_conf.link_speeds &
440 ETH_LINK_SPEED_FIXED);
442 if (speed == SPEED_2500) {
443 uint32_t tipg = IGC_READ_REG(hw, IGC_TIPG);
444 if ((tipg & IGC_TIPG_IPGT_MASK) != 0x0b) {
445 tipg &= ~IGC_TIPG_IPGT_MASK;
447 IGC_WRITE_REG(hw, IGC_TIPG, tipg);
452 link.link_duplex = ETH_LINK_HALF_DUPLEX;
453 link.link_status = ETH_LINK_DOWN;
454 link.link_autoneg = ETH_LINK_FIXED;
457 return rte_eth_linkstatus_set(dev, &link);
461 * It executes link_update after knowing an interrupt is present.
464 eth_igc_interrupt_action(struct rte_eth_dev *dev)
466 struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
467 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
468 struct rte_eth_link link;
471 if (intr->flags & IGC_FLAG_NEED_LINK_UPDATE) {
472 intr->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
474 /* set get_link_status to check register later */
475 ret = eth_igc_link_update(dev, 0);
477 /* check if link has changed */
481 rte_eth_linkstatus_get(dev, &link);
482 if (link.link_status)
484 " Port %d: Link Up - speed %u Mbps - %s",
486 (unsigned int)link.link_speed,
487 link.link_duplex == ETH_LINK_FULL_DUPLEX ?
488 "full-duplex" : "half-duplex");
490 PMD_DRV_LOG(INFO, " Port %d: Link Down",
493 PMD_DRV_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT,
494 pci_dev->addr.domain,
497 pci_dev->addr.function);
498 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
504 * Interrupt handler which shall be registered at first.
507 * Pointer to interrupt handle.
509 * The address of parameter (struct rte_eth_dev *) registered before.
512 eth_igc_interrupt_handler(void *param)
514 struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
516 eth_igc_interrupt_get_status(dev);
517 eth_igc_interrupt_action(dev);
520 static void igc_read_queue_stats_register(struct rte_eth_dev *dev);
523 * Update the queue status every IGC_ALARM_INTERVAL time.
525 * The address of parameter (struct rte_eth_dev *) registered before.
528 igc_update_queue_stats_handler(void *param)
530 struct rte_eth_dev *dev = param;
531 igc_read_queue_stats_register(dev);
532 rte_eal_alarm_set(IGC_ALARM_INTERVAL,
533 igc_update_queue_stats_handler, dev);
537 * rx,tx enable/disable
540 eth_igc_rxtx_control(struct rte_eth_dev *dev, bool enable)
542 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
545 tctl = IGC_READ_REG(hw, IGC_TCTL);
546 rctl = IGC_READ_REG(hw, IGC_RCTL);
554 tctl &= ~IGC_TCTL_EN;
555 rctl &= ~IGC_RCTL_EN;
557 IGC_WRITE_REG(hw, IGC_TCTL, tctl);
558 IGC_WRITE_REG(hw, IGC_RCTL, rctl);
563 * This routine disables all traffic on the adapter by issuing a
564 * global reset on the MAC.
567 eth_igc_stop(struct rte_eth_dev *dev)
569 struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
570 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
571 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
572 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
573 struct rte_eth_link link;
575 adapter->stopped = 1;
577 /* disable receive and transmit */
578 eth_igc_rxtx_control(dev, false);
580 /* disable all MSI-X interrupts */
581 IGC_WRITE_REG(hw, IGC_EIMC, 0x1f);
584 /* clear all MSI-X interrupts */
585 IGC_WRITE_REG(hw, IGC_EICR, 0x1f);
587 igc_intr_other_disable(dev);
589 rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
591 /* disable intr eventfd mapping */
592 rte_intr_disable(intr_handle);
596 /* disable all wake up */
597 IGC_WRITE_REG(hw, IGC_WUC, 0);
599 /* Set bit for Go Link disconnect */
600 igc_read_reg_check_set_bits(hw, IGC_82580_PHY_POWER_MGMT,
601 IGC_82580_PM_GO_LINKD);
603 /* Power down the phy. Needed to make the link go Down */
604 eth_igc_set_link_down(dev);
606 igc_dev_clear_queues(dev);
608 /* clear the recorded link status */
609 memset(&link, 0, sizeof(link));
610 rte_eth_linkstatus_set(dev, &link);
612 if (!rte_intr_allow_others(intr_handle))
613 /* resume to the default handler */
614 rte_intr_callback_register(intr_handle,
615 eth_igc_interrupt_handler,
618 /* Clean datapath event and queue/vec mapping */
619 rte_intr_efd_disable(intr_handle);
622 /* Sets up the hardware to generate MSI-X interrupts properly
624 * board private structure
627 igc_configure_msix_intr(struct rte_eth_dev *dev)
629 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
630 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
631 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
635 /* won't configure msix register if no mapping is done
636 * between intr vector and event fd
638 if (!rte_intr_dp_is_en(intr_handle) ||
639 !dev->data->dev_conf.intr_conf.lsc)
642 /* turn on MSI-X capability first */
643 IGC_WRITE_REG(hw, IGC_GPIE, IGC_GPIE_MSIX_MODE |
644 IGC_GPIE_PBA | IGC_GPIE_EIAME |
647 intr_mask = (1u << IGC_MSIX_OTHER_INTR_VEC);
649 /* enable msix auto-clear */
650 igc_read_reg_check_set_bits(hw, IGC_EIAC, intr_mask);
652 /* set other cause interrupt vector */
653 igc_read_reg_check_set_bits(hw, IGC_IVAR_MISC,
654 (uint32_t)(IGC_MSIX_OTHER_INTR_VEC | IGC_IVAR_VALID) << 8);
656 /* enable auto-mask */
657 igc_read_reg_check_set_bits(hw, IGC_EIAM, intr_mask);
663 * It enables the interrupt mask and then enable the interrupt.
666 * Pointer to struct rte_eth_dev.
671 igc_lsc_interrupt_setup(struct rte_eth_dev *dev, uint8_t on)
673 struct igc_interrupt *intr = IGC_DEV_PRIVATE_INTR(dev);
676 intr->mask |= IGC_ICR_LSC;
678 intr->mask &= ~IGC_ICR_LSC;
682 * Get hardware rx-buffer size.
685 igc_get_rx_buffer_size(struct igc_hw *hw)
687 return (IGC_READ_REG(hw, IGC_RXPBS) & 0x3f) << 10;
691 * igc_hw_control_acquire sets CTRL_EXT:DRV_LOAD bit.
692 * For ASF and Pass Through versions of f/w this means
693 * that the driver is loaded.
696 igc_hw_control_acquire(struct igc_hw *hw)
700 /* Let firmware know the driver has taken over */
701 ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
702 IGC_WRITE_REG(hw, IGC_CTRL_EXT, ctrl_ext | IGC_CTRL_EXT_DRV_LOAD);
706 * igc_hw_control_release resets CTRL_EXT:DRV_LOAD bit.
707 * For ASF and Pass Through versions of f/w this means that the
708 * driver is no longer loaded.
711 igc_hw_control_release(struct igc_hw *hw)
715 /* Let firmware taken over control of h/w */
716 ctrl_ext = IGC_READ_REG(hw, IGC_CTRL_EXT);
717 IGC_WRITE_REG(hw, IGC_CTRL_EXT,
718 ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD);
722 igc_hardware_init(struct igc_hw *hw)
724 uint32_t rx_buf_size;
727 /* Let the firmware know the OS is in control */
728 igc_hw_control_acquire(hw);
730 /* Issue a global reset */
733 /* disable all wake up */
734 IGC_WRITE_REG(hw, IGC_WUC, 0);
737 * Hardware flow control
738 * - High water mark should allow for at least two standard size (1518)
739 * frames to be received after sending an XOFF.
740 * - Low water mark works best when it is very near the high water mark.
741 * This allows the receiver to restart by sending XON when it has
742 * drained a bit. Here we use an arbitrary value of 1500 which will
743 * restart after one full frame is pulled from the buffer. There
744 * could be several smaller frames in the buffer and if so they will
745 * not trigger the XON until their total number reduces the buffer
748 rx_buf_size = igc_get_rx_buffer_size(hw);
749 hw->fc.high_water = rx_buf_size - (RTE_ETHER_MAX_LEN * 2);
750 hw->fc.low_water = hw->fc.high_water - 1500;
751 hw->fc.pause_time = IGC_FC_PAUSE_TIME;
753 hw->fc.requested_mode = igc_fc_full;
755 diag = igc_init_hw(hw);
759 igc_get_phy_info(hw);
760 igc_check_for_link(hw);
766 eth_igc_start(struct rte_eth_dev *dev)
768 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
769 struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
770 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
771 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
775 PMD_INIT_FUNC_TRACE();
777 /* disable all MSI-X interrupts */
778 IGC_WRITE_REG(hw, IGC_EIMC, 0x1f);
781 /* clear all MSI-X interrupts */
782 IGC_WRITE_REG(hw, IGC_EICR, 0x1f);
784 /* disable uio/vfio intr/eventfd mapping */
785 if (!adapter->stopped)
786 rte_intr_disable(intr_handle);
788 /* Power up the phy. Needed to make the link go Up */
789 eth_igc_set_link_up(dev);
791 /* Put the address into the Receive Address Array */
792 igc_rar_set(hw, hw->mac.addr, 0);
794 /* Initialize the hardware */
795 if (igc_hardware_init(hw)) {
796 PMD_DRV_LOG(ERR, "Unable to initialize the hardware");
799 adapter->stopped = 0;
801 /* confiugre msix for rx interrupt */
802 igc_configure_msix_intr(dev);
806 /* This can fail when allocating mbufs for descriptor rings */
807 ret = igc_rx_init(dev);
809 PMD_DRV_LOG(ERR, "Unable to initialize RX hardware");
810 igc_dev_clear_queues(dev);
814 igc_clear_hw_cntrs_base_generic(hw);
816 /* Setup link speed and duplex */
817 speeds = &dev->data->dev_conf.link_speeds;
818 if (*speeds == ETH_LINK_SPEED_AUTONEG) {
819 hw->phy.autoneg_advertised = IGC_ALL_SPEED_DUPLEX_2500;
823 bool autoneg = (*speeds & ETH_LINK_SPEED_FIXED) == 0;
826 hw->phy.autoneg_advertised = 0;
828 if (*speeds & ~(ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M |
829 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M |
830 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G |
831 ETH_LINK_SPEED_FIXED)) {
833 goto error_invalid_config;
835 if (*speeds & ETH_LINK_SPEED_10M_HD) {
836 hw->phy.autoneg_advertised |= ADVERTISE_10_HALF;
839 if (*speeds & ETH_LINK_SPEED_10M) {
840 hw->phy.autoneg_advertised |= ADVERTISE_10_FULL;
843 if (*speeds & ETH_LINK_SPEED_100M_HD) {
844 hw->phy.autoneg_advertised |= ADVERTISE_100_HALF;
847 if (*speeds & ETH_LINK_SPEED_100M) {
848 hw->phy.autoneg_advertised |= ADVERTISE_100_FULL;
851 if (*speeds & ETH_LINK_SPEED_1G) {
852 hw->phy.autoneg_advertised |= ADVERTISE_1000_FULL;
855 if (*speeds & ETH_LINK_SPEED_2_5G) {
856 hw->phy.autoneg_advertised |= ADVERTISE_2500_FULL;
859 if (num_speeds == 0 || (!autoneg && num_speeds > 1))
860 goto error_invalid_config;
862 /* Set/reset the mac.autoneg based on the link speed,
867 hw->mac.forced_speed_duplex =
868 hw->phy.autoneg_advertised;
876 if (rte_intr_allow_others(intr_handle)) {
877 /* check if lsc interrupt is enabled */
878 if (dev->data->dev_conf.intr_conf.lsc)
879 igc_lsc_interrupt_setup(dev, 1);
881 igc_lsc_interrupt_setup(dev, 0);
883 rte_intr_callback_unregister(intr_handle,
884 eth_igc_interrupt_handler,
886 if (dev->data->dev_conf.intr_conf.lsc)
888 "LSC won't enable because of no intr multiplex");
891 /* enable uio/vfio intr/eventfd mapping */
892 rte_intr_enable(intr_handle);
894 rte_eal_alarm_set(IGC_ALARM_INTERVAL,
895 igc_update_queue_stats_handler, dev);
897 /* resume enabled intr since hw reset */
898 igc_intr_other_enable(dev);
900 eth_igc_rxtx_control(dev, true);
901 eth_igc_link_update(dev, 0);
905 error_invalid_config:
906 PMD_DRV_LOG(ERR, "Invalid advertised speeds (%u) for port %u",
907 dev->data->dev_conf.link_speeds, dev->data->port_id);
908 igc_dev_clear_queues(dev);
913 igc_reset_swfw_lock(struct igc_hw *hw)
918 * Do mac ops initialization manually here, since we will need
919 * some function pointers set by this call.
921 ret_val = igc_init_mac_params(hw);
926 * SMBI lock should not fail in this early stage. If this is the case,
927 * it is due to an improper exit of the application.
928 * So force the release of the faulty lock.
930 if (igc_get_hw_semaphore_generic(hw) < 0)
931 PMD_DRV_LOG(DEBUG, "SMBI lock released");
933 igc_put_hw_semaphore_generic(hw);
935 if (hw->mac.ops.acquire_swfw_sync != NULL) {
939 * Phy lock should not fail in this early stage.
940 * If this is the case, it is due to an improper exit of the
941 * application. So force the release of the faulty lock.
943 mask = IGC_SWFW_PHY0_SM;
944 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0) {
945 PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released",
948 hw->mac.ops.release_swfw_sync(hw, mask);
951 * This one is more tricky since it is common to all ports; but
952 * swfw_sync retries last long enough (1s) to be almost sure
953 * that if lock can not be taken it is due to an improper lock
956 mask = IGC_SWFW_EEP_SM;
957 if (hw->mac.ops.acquire_swfw_sync(hw, mask) < 0)
958 PMD_DRV_LOG(DEBUG, "SWFW common locks released");
960 hw->mac.ops.release_swfw_sync(hw, mask);
967 * free all rx/tx queues.
970 igc_dev_free_queues(struct rte_eth_dev *dev)
974 for (i = 0; i < dev->data->nb_rx_queues; i++) {
975 eth_igc_rx_queue_release(dev->data->rx_queues[i]);
976 dev->data->rx_queues[i] = NULL;
978 dev->data->nb_rx_queues = 0;
980 for (i = 0; i < dev->data->nb_tx_queues; i++) {
981 eth_igc_tx_queue_release(dev->data->tx_queues[i]);
982 dev->data->tx_queues[i] = NULL;
984 dev->data->nb_tx_queues = 0;
988 eth_igc_close(struct rte_eth_dev *dev)
990 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
991 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
992 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
993 struct igc_adapter *adapter = IGC_DEV_PRIVATE(dev);
996 PMD_INIT_FUNC_TRACE();
998 if (!adapter->stopped)
1001 igc_intr_other_disable(dev);
1003 int ret = rte_intr_callback_unregister(intr_handle,
1004 eth_igc_interrupt_handler, dev);
1005 if (ret >= 0 || ret == -ENOENT || ret == -EINVAL)
1008 PMD_DRV_LOG(ERR, "intr callback unregister failed: %d", ret);
1009 DELAY(200 * 1000); /* delay 200ms */
1010 } while (retry++ < 5);
1012 igc_phy_hw_reset(hw);
1013 igc_hw_control_release(hw);
1014 igc_dev_free_queues(dev);
1016 /* Reset any pending lock */
1017 igc_reset_swfw_lock(hw);
1021 igc_identify_hardware(struct rte_eth_dev *dev, struct rte_pci_device *pci_dev)
1023 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1025 hw->vendor_id = pci_dev->id.vendor_id;
1026 hw->device_id = pci_dev->id.device_id;
1027 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
1028 hw->subsystem_device_id = pci_dev->id.subsystem_device_id;
1032 eth_igc_dev_init(struct rte_eth_dev *dev)
1034 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1035 struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
1036 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1039 PMD_INIT_FUNC_TRACE();
1040 dev->dev_ops = ð_igc_ops;
1043 * for secondary processes, we don't initialize any further as primary
1044 * has already done this work. Only check we don't need a different
1047 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1050 rte_eth_copy_pci_info(dev, pci_dev);
1053 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1055 igc_identify_hardware(dev, pci_dev);
1056 if (igc_setup_init_funcs(hw, false) != IGC_SUCCESS) {
1061 igc_get_bus_info(hw);
1063 /* Reset any pending lock */
1064 if (igc_reset_swfw_lock(hw) != IGC_SUCCESS) {
1069 /* Finish initialization */
1070 if (igc_setup_init_funcs(hw, true) != IGC_SUCCESS) {
1075 hw->mac.autoneg = 1;
1076 hw->phy.autoneg_wait_to_complete = 0;
1077 hw->phy.autoneg_advertised = IGC_ALL_SPEED_DUPLEX_2500;
1079 /* Copper options */
1080 if (hw->phy.media_type == igc_media_type_copper) {
1081 hw->phy.mdix = 0; /* AUTO_ALL_MODES */
1082 hw->phy.disable_polarity_correction = 0;
1083 hw->phy.ms_type = igc_ms_hw_default;
1087 * Start from a known state, this is important in reading the nvm
1088 * and mac from that.
1092 /* Make sure we have a good EEPROM before we read from it */
1093 if (igc_validate_nvm_checksum(hw) < 0) {
1095 * Some PCI-E parts fail the first check due to
1096 * the link being in sleep state, call it again,
1097 * if it fails a second time its a real issue.
1099 if (igc_validate_nvm_checksum(hw) < 0) {
1100 PMD_INIT_LOG(ERR, "EEPROM checksum invalid");
1106 /* Read the permanent MAC address out of the EEPROM */
1107 if (igc_read_mac_addr(hw) != 0) {
1108 PMD_INIT_LOG(ERR, "EEPROM error while reading MAC address");
1113 /* Allocate memory for storing MAC addresses */
1114 dev->data->mac_addrs = rte_zmalloc("igc",
1115 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count, 0);
1116 if (dev->data->mac_addrs == NULL) {
1117 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes for storing MAC",
1118 RTE_ETHER_ADDR_LEN * hw->mac.rar_entry_count);
1123 /* Copy the permanent MAC address */
1124 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr,
1125 &dev->data->mac_addrs[0]);
1127 /* Now initialize the hardware */
1128 if (igc_hardware_init(hw) != 0) {
1129 PMD_INIT_LOG(ERR, "Hardware initialization failed");
1130 rte_free(dev->data->mac_addrs);
1131 dev->data->mac_addrs = NULL;
1136 /* Pass the information to the rte_eth_dev_close() that it should also
1137 * release the private port resources.
1139 dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
1141 hw->mac.get_link_status = 1;
1144 /* Indicate SOL/IDER usage */
1145 if (igc_check_reset_block(hw) < 0)
1147 "PHY reset is blocked due to SOL/IDER session.");
1149 PMD_INIT_LOG(DEBUG, "port_id %d vendorID=0x%x deviceID=0x%x",
1150 dev->data->port_id, pci_dev->id.vendor_id,
1151 pci_dev->id.device_id);
1153 rte_intr_callback_register(&pci_dev->intr_handle,
1154 eth_igc_interrupt_handler, (void *)dev);
1156 /* enable uio/vfio intr/eventfd mapping */
1157 rte_intr_enable(&pci_dev->intr_handle);
1159 /* enable support intr */
1160 igc_intr_other_enable(dev);
1162 /* initiate queue status */
1163 for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1164 igc->txq_stats_map[i] = -1;
1165 igc->rxq_stats_map[i] = -1;
1170 igc_hw_control_release(hw);
1175 eth_igc_dev_uninit(__rte_unused struct rte_eth_dev *eth_dev)
1177 PMD_INIT_FUNC_TRACE();
1179 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1182 eth_igc_close(eth_dev);
1187 eth_igc_reset(struct rte_eth_dev *dev)
1191 PMD_INIT_FUNC_TRACE();
1193 ret = eth_igc_dev_uninit(dev);
1197 return eth_igc_dev_init(dev);
1201 eth_igc_promiscuous_enable(struct rte_eth_dev *dev)
1203 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1206 rctl = IGC_READ_REG(hw, IGC_RCTL);
1207 rctl |= (IGC_RCTL_UPE | IGC_RCTL_MPE);
1208 IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1213 eth_igc_promiscuous_disable(struct rte_eth_dev *dev)
1215 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1218 rctl = IGC_READ_REG(hw, IGC_RCTL);
1219 rctl &= (~IGC_RCTL_UPE);
1220 if (dev->data->all_multicast == 1)
1221 rctl |= IGC_RCTL_MPE;
1223 rctl &= (~IGC_RCTL_MPE);
1224 IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1229 eth_igc_allmulticast_enable(struct rte_eth_dev *dev)
1231 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1234 rctl = IGC_READ_REG(hw, IGC_RCTL);
1235 rctl |= IGC_RCTL_MPE;
1236 IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1241 eth_igc_allmulticast_disable(struct rte_eth_dev *dev)
1243 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1246 if (dev->data->promiscuous == 1)
1247 return 0; /* must remain in all_multicast mode */
1249 rctl = IGC_READ_REG(hw, IGC_RCTL);
1250 rctl &= (~IGC_RCTL_MPE);
1251 IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1256 eth_igc_fw_version_get(struct rte_eth_dev *dev, char *fw_version,
1259 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1260 struct igc_fw_version fw;
1263 igc_get_fw_version(hw, &fw);
1265 /* if option rom is valid, display its version too */
1267 ret = snprintf(fw_version, fw_size,
1268 "%d.%d, 0x%08x, %d.%d.%d",
1269 fw.eep_major, fw.eep_minor, fw.etrack_id,
1270 fw.or_major, fw.or_build, fw.or_patch);
1273 if (fw.etrack_id != 0X0000) {
1274 ret = snprintf(fw_version, fw_size,
1276 fw.eep_major, fw.eep_minor,
1279 ret = snprintf(fw_version, fw_size,
1281 fw.eep_major, fw.eep_minor,
1286 ret += 1; /* add the size of '\0' */
1287 if (fw_size < (u32)ret)
1294 eth_igc_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1296 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1298 dev_info->min_rx_bufsize = 256; /* See BSIZE field of RCTL register. */
1299 dev_info->max_rx_pktlen = MAX_RX_JUMBO_FRAME_SIZE;
1300 dev_info->max_mac_addrs = hw->mac.rar_entry_count;
1301 dev_info->rx_offload_capa = IGC_RX_OFFLOAD_ALL;
1302 dev_info->tx_offload_capa = IGC_TX_OFFLOAD_ALL;
1304 dev_info->max_rx_queues = IGC_QUEUE_PAIRS_NUM;
1305 dev_info->max_tx_queues = IGC_QUEUE_PAIRS_NUM;
1306 dev_info->max_vmdq_pools = 0;
1308 dev_info->hash_key_size = IGC_HKEY_MAX_INDEX * sizeof(uint32_t);
1309 dev_info->reta_size = ETH_RSS_RETA_SIZE_128;
1310 dev_info->flow_type_rss_offloads = IGC_RSS_OFFLOAD_ALL;
1312 dev_info->default_rxconf = (struct rte_eth_rxconf) {
1314 .pthresh = IGC_DEFAULT_RX_PTHRESH,
1315 .hthresh = IGC_DEFAULT_RX_HTHRESH,
1316 .wthresh = IGC_DEFAULT_RX_WTHRESH,
1318 .rx_free_thresh = IGC_DEFAULT_RX_FREE_THRESH,
1323 dev_info->default_txconf = (struct rte_eth_txconf) {
1325 .pthresh = IGC_DEFAULT_TX_PTHRESH,
1326 .hthresh = IGC_DEFAULT_TX_HTHRESH,
1327 .wthresh = IGC_DEFAULT_TX_WTHRESH,
1332 dev_info->rx_desc_lim = rx_desc_lim;
1333 dev_info->tx_desc_lim = tx_desc_lim;
1335 dev_info->speed_capa = ETH_LINK_SPEED_10M_HD | ETH_LINK_SPEED_10M |
1336 ETH_LINK_SPEED_100M_HD | ETH_LINK_SPEED_100M |
1337 ETH_LINK_SPEED_1G | ETH_LINK_SPEED_2_5G;
1339 dev_info->max_mtu = dev_info->max_rx_pktlen - IGC_ETH_OVERHEAD;
1340 dev_info->min_mtu = RTE_ETHER_MIN_MTU;
1345 eth_igc_led_on(struct rte_eth_dev *dev)
1347 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1349 return igc_led_on(hw) == IGC_SUCCESS ? 0 : -ENOTSUP;
1353 eth_igc_led_off(struct rte_eth_dev *dev)
1355 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1357 return igc_led_off(hw) == IGC_SUCCESS ? 0 : -ENOTSUP;
1360 static const uint32_t *
1361 eth_igc_supported_ptypes_get(__rte_unused struct rte_eth_dev *dev)
1363 static const uint32_t ptypes[] = {
1364 /* refers to rx_desc_pkt_info_to_pkt_type() */
1367 RTE_PTYPE_L3_IPV4_EXT,
1369 RTE_PTYPE_L3_IPV6_EXT,
1373 RTE_PTYPE_TUNNEL_IP,
1374 RTE_PTYPE_INNER_L3_IPV6,
1375 RTE_PTYPE_INNER_L3_IPV6_EXT,
1376 RTE_PTYPE_INNER_L4_TCP,
1377 RTE_PTYPE_INNER_L4_UDP,
1385 eth_igc_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
1387 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1388 uint32_t frame_size = mtu + IGC_ETH_OVERHEAD;
1391 /* if extend vlan has been enabled */
1392 if (IGC_READ_REG(hw, IGC_CTRL_EXT) & IGC_CTRL_EXT_EXT_VLAN)
1393 frame_size += VLAN_TAG_SIZE;
1395 /* check that mtu is within the allowed range */
1396 if (mtu < RTE_ETHER_MIN_MTU ||
1397 frame_size > MAX_RX_JUMBO_FRAME_SIZE)
1401 * refuse mtu that requires the support of scattered packets when
1402 * this feature has not been enabled before.
1404 if (!dev->data->scattered_rx &&
1405 frame_size > dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)
1408 rctl = IGC_READ_REG(hw, IGC_RCTL);
1410 /* switch to jumbo mode if needed */
1411 if (mtu > RTE_ETHER_MTU) {
1412 dev->data->dev_conf.rxmode.offloads |=
1413 DEV_RX_OFFLOAD_JUMBO_FRAME;
1414 rctl |= IGC_RCTL_LPE;
1416 dev->data->dev_conf.rxmode.offloads &=
1417 ~DEV_RX_OFFLOAD_JUMBO_FRAME;
1418 rctl &= ~IGC_RCTL_LPE;
1420 IGC_WRITE_REG(hw, IGC_RCTL, rctl);
1422 /* update max frame size */
1423 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
1425 IGC_WRITE_REG(hw, IGC_RLPML,
1426 dev->data->dev_conf.rxmode.max_rx_pkt_len);
1432 eth_igc_rar_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
1433 uint32_t index, uint32_t pool)
1435 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1437 igc_rar_set(hw, mac_addr->addr_bytes, index);
1443 eth_igc_rar_clear(struct rte_eth_dev *dev, uint32_t index)
1445 uint8_t addr[RTE_ETHER_ADDR_LEN];
1446 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1448 memset(addr, 0, sizeof(addr));
1449 igc_rar_set(hw, addr, index);
1453 eth_igc_default_mac_addr_set(struct rte_eth_dev *dev,
1454 struct rte_ether_addr *addr)
1456 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1457 igc_rar_set(hw, addr->addr_bytes, 0);
1462 eth_igc_set_mc_addr_list(struct rte_eth_dev *dev,
1463 struct rte_ether_addr *mc_addr_set,
1464 uint32_t nb_mc_addr)
1466 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1467 igc_update_mc_addr_list(hw, (u8 *)mc_addr_set, nb_mc_addr);
1472 * Read hardware registers
1475 igc_read_stats_registers(struct igc_hw *hw, struct igc_hw_stats *stats)
1479 uint64_t old_gprc = stats->gprc;
1480 uint64_t old_gptc = stats->gptc;
1481 uint64_t old_tpr = stats->tpr;
1482 uint64_t old_tpt = stats->tpt;
1483 uint64_t old_rpthc = stats->rpthc;
1484 uint64_t old_hgptc = stats->hgptc;
1486 stats->crcerrs += IGC_READ_REG(hw, IGC_CRCERRS);
1487 stats->algnerrc += IGC_READ_REG(hw, IGC_ALGNERRC);
1488 stats->rxerrc += IGC_READ_REG(hw, IGC_RXERRC);
1489 stats->mpc += IGC_READ_REG(hw, IGC_MPC);
1490 stats->scc += IGC_READ_REG(hw, IGC_SCC);
1491 stats->ecol += IGC_READ_REG(hw, IGC_ECOL);
1493 stats->mcc += IGC_READ_REG(hw, IGC_MCC);
1494 stats->latecol += IGC_READ_REG(hw, IGC_LATECOL);
1495 stats->colc += IGC_READ_REG(hw, IGC_COLC);
1497 stats->dc += IGC_READ_REG(hw, IGC_DC);
1498 stats->tncrs += IGC_READ_REG(hw, IGC_TNCRS);
1499 stats->htdpmc += IGC_READ_REG(hw, IGC_HTDPMC);
1500 stats->rlec += IGC_READ_REG(hw, IGC_RLEC);
1501 stats->xonrxc += IGC_READ_REG(hw, IGC_XONRXC);
1502 stats->xontxc += IGC_READ_REG(hw, IGC_XONTXC);
1505 * For watchdog management we need to know if we have been
1506 * paused during the last interval, so capture that here.
1508 pause_frames = IGC_READ_REG(hw, IGC_XOFFRXC);
1509 stats->xoffrxc += pause_frames;
1510 stats->xofftxc += IGC_READ_REG(hw, IGC_XOFFTXC);
1511 stats->fcruc += IGC_READ_REG(hw, IGC_FCRUC);
1512 stats->prc64 += IGC_READ_REG(hw, IGC_PRC64);
1513 stats->prc127 += IGC_READ_REG(hw, IGC_PRC127);
1514 stats->prc255 += IGC_READ_REG(hw, IGC_PRC255);
1515 stats->prc511 += IGC_READ_REG(hw, IGC_PRC511);
1516 stats->prc1023 += IGC_READ_REG(hw, IGC_PRC1023);
1517 stats->prc1522 += IGC_READ_REG(hw, IGC_PRC1522);
1518 stats->gprc += IGC_READ_REG(hw, IGC_GPRC);
1519 stats->bprc += IGC_READ_REG(hw, IGC_BPRC);
1520 stats->mprc += IGC_READ_REG(hw, IGC_MPRC);
1521 stats->gptc += IGC_READ_REG(hw, IGC_GPTC);
1523 /* For the 64-bit byte counters the low dword must be read first. */
1524 /* Both registers clear on the read of the high dword */
1526 /* Workaround CRC bytes included in size, take away 4 bytes/packet */
1527 stats->gorc += IGC_READ_REG(hw, IGC_GORCL);
1528 stats->gorc += ((uint64_t)IGC_READ_REG(hw, IGC_GORCH) << 32);
1529 stats->gorc -= (stats->gprc - old_gprc) * RTE_ETHER_CRC_LEN;
1530 stats->gotc += IGC_READ_REG(hw, IGC_GOTCL);
1531 stats->gotc += ((uint64_t)IGC_READ_REG(hw, IGC_GOTCH) << 32);
1532 stats->gotc -= (stats->gptc - old_gptc) * RTE_ETHER_CRC_LEN;
1534 stats->rnbc += IGC_READ_REG(hw, IGC_RNBC);
1535 stats->ruc += IGC_READ_REG(hw, IGC_RUC);
1536 stats->rfc += IGC_READ_REG(hw, IGC_RFC);
1537 stats->roc += IGC_READ_REG(hw, IGC_ROC);
1538 stats->rjc += IGC_READ_REG(hw, IGC_RJC);
1540 stats->mgprc += IGC_READ_REG(hw, IGC_MGTPRC);
1541 stats->mgpdc += IGC_READ_REG(hw, IGC_MGTPDC);
1542 stats->mgptc += IGC_READ_REG(hw, IGC_MGTPTC);
1543 stats->b2ospc += IGC_READ_REG(hw, IGC_B2OSPC);
1544 stats->b2ogprc += IGC_READ_REG(hw, IGC_B2OGPRC);
1545 stats->o2bgptc += IGC_READ_REG(hw, IGC_O2BGPTC);
1546 stats->o2bspc += IGC_READ_REG(hw, IGC_O2BSPC);
1548 stats->tpr += IGC_READ_REG(hw, IGC_TPR);
1549 stats->tpt += IGC_READ_REG(hw, IGC_TPT);
1551 stats->tor += IGC_READ_REG(hw, IGC_TORL);
1552 stats->tor += ((uint64_t)IGC_READ_REG(hw, IGC_TORH) << 32);
1553 stats->tor -= (stats->tpr - old_tpr) * RTE_ETHER_CRC_LEN;
1554 stats->tot += IGC_READ_REG(hw, IGC_TOTL);
1555 stats->tot += ((uint64_t)IGC_READ_REG(hw, IGC_TOTH) << 32);
1556 stats->tot -= (stats->tpt - old_tpt) * RTE_ETHER_CRC_LEN;
1558 stats->ptc64 += IGC_READ_REG(hw, IGC_PTC64);
1559 stats->ptc127 += IGC_READ_REG(hw, IGC_PTC127);
1560 stats->ptc255 += IGC_READ_REG(hw, IGC_PTC255);
1561 stats->ptc511 += IGC_READ_REG(hw, IGC_PTC511);
1562 stats->ptc1023 += IGC_READ_REG(hw, IGC_PTC1023);
1563 stats->ptc1522 += IGC_READ_REG(hw, IGC_PTC1522);
1564 stats->mptc += IGC_READ_REG(hw, IGC_MPTC);
1565 stats->bptc += IGC_READ_REG(hw, IGC_BPTC);
1566 stats->tsctc += IGC_READ_REG(hw, IGC_TSCTC);
1568 stats->iac += IGC_READ_REG(hw, IGC_IAC);
1569 stats->rpthc += IGC_READ_REG(hw, IGC_RPTHC);
1570 stats->hgptc += IGC_READ_REG(hw, IGC_HGPTC);
1571 stats->icrxdmtc += IGC_READ_REG(hw, IGC_ICRXDMTC);
1573 /* Host to Card Statistics */
1574 stats->hgorc += IGC_READ_REG(hw, IGC_HGORCL);
1575 stats->hgorc += ((uint64_t)IGC_READ_REG(hw, IGC_HGORCH) << 32);
1576 stats->hgorc -= (stats->rpthc - old_rpthc) * RTE_ETHER_CRC_LEN;
1577 stats->hgotc += IGC_READ_REG(hw, IGC_HGOTCL);
1578 stats->hgotc += ((uint64_t)IGC_READ_REG(hw, IGC_HGOTCH) << 32);
1579 stats->hgotc -= (stats->hgptc - old_hgptc) * RTE_ETHER_CRC_LEN;
1580 stats->lenerrs += IGC_READ_REG(hw, IGC_LENERRS);
1584 * Write 0 to all queue status registers
1587 igc_reset_queue_stats_register(struct igc_hw *hw)
1591 for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1592 IGC_WRITE_REG(hw, IGC_PQGPRC(i), 0);
1593 IGC_WRITE_REG(hw, IGC_PQGPTC(i), 0);
1594 IGC_WRITE_REG(hw, IGC_PQGORC(i), 0);
1595 IGC_WRITE_REG(hw, IGC_PQGOTC(i), 0);
1596 IGC_WRITE_REG(hw, IGC_PQMPRC(i), 0);
1597 IGC_WRITE_REG(hw, IGC_RQDPC(i), 0);
1598 IGC_WRITE_REG(hw, IGC_TQDPC(i), 0);
1603 * Read all hardware queue status registers
1606 igc_read_queue_stats_register(struct rte_eth_dev *dev)
1608 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1609 struct igc_hw_queue_stats *queue_stats =
1610 IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1614 * This register is not cleared on read. Furthermore, the register wraps
1615 * around back to 0x00000000 on the next increment when reaching a value
1616 * of 0xFFFFFFFF and then continues normal count operation.
1618 for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1626 * Read the register first, if the value is smaller than that
1627 * previous read, that mean the register has been overflowed,
1628 * then we add the high 4 bytes by 1 and replace the low 4
1629 * bytes by the new value.
1631 tmp = IGC_READ_REG(hw, IGC_PQGPRC(i));
1632 value.ddword = queue_stats->pqgprc[i];
1633 if (value.dword[U32_0_IN_U64] > tmp)
1634 value.dword[U32_1_IN_U64]++;
1635 value.dword[U32_0_IN_U64] = tmp;
1636 queue_stats->pqgprc[i] = value.ddword;
1638 tmp = IGC_READ_REG(hw, IGC_PQGPTC(i));
1639 value.ddword = queue_stats->pqgptc[i];
1640 if (value.dword[U32_0_IN_U64] > tmp)
1641 value.dword[U32_1_IN_U64]++;
1642 value.dword[U32_0_IN_U64] = tmp;
1643 queue_stats->pqgptc[i] = value.ddword;
1645 tmp = IGC_READ_REG(hw, IGC_PQGORC(i));
1646 value.ddword = queue_stats->pqgorc[i];
1647 if (value.dword[U32_0_IN_U64] > tmp)
1648 value.dword[U32_1_IN_U64]++;
1649 value.dword[U32_0_IN_U64] = tmp;
1650 queue_stats->pqgorc[i] = value.ddword;
1652 tmp = IGC_READ_REG(hw, IGC_PQGOTC(i));
1653 value.ddword = queue_stats->pqgotc[i];
1654 if (value.dword[U32_0_IN_U64] > tmp)
1655 value.dword[U32_1_IN_U64]++;
1656 value.dword[U32_0_IN_U64] = tmp;
1657 queue_stats->pqgotc[i] = value.ddword;
1659 tmp = IGC_READ_REG(hw, IGC_PQMPRC(i));
1660 value.ddword = queue_stats->pqmprc[i];
1661 if (value.dword[U32_0_IN_U64] > tmp)
1662 value.dword[U32_1_IN_U64]++;
1663 value.dword[U32_0_IN_U64] = tmp;
1664 queue_stats->pqmprc[i] = value.ddword;
1666 tmp = IGC_READ_REG(hw, IGC_RQDPC(i));
1667 value.ddword = queue_stats->rqdpc[i];
1668 if (value.dword[U32_0_IN_U64] > tmp)
1669 value.dword[U32_1_IN_U64]++;
1670 value.dword[U32_0_IN_U64] = tmp;
1671 queue_stats->rqdpc[i] = value.ddword;
1673 tmp = IGC_READ_REG(hw, IGC_TQDPC(i));
1674 value.ddword = queue_stats->tqdpc[i];
1675 if (value.dword[U32_0_IN_U64] > tmp)
1676 value.dword[U32_1_IN_U64]++;
1677 value.dword[U32_0_IN_U64] = tmp;
1678 queue_stats->tqdpc[i] = value.ddword;
1683 eth_igc_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *rte_stats)
1685 struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
1686 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1687 struct igc_hw_stats *stats = IGC_DEV_PRIVATE_STATS(dev);
1688 struct igc_hw_queue_stats *queue_stats =
1689 IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1693 * Cancel status handler since it will read the queue status registers
1695 rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
1697 /* Read status register */
1698 igc_read_queue_stats_register(dev);
1699 igc_read_stats_registers(hw, stats);
1701 if (rte_stats == NULL) {
1702 /* Restart queue status handler */
1703 rte_eal_alarm_set(IGC_ALARM_INTERVAL,
1704 igc_update_queue_stats_handler, dev);
1709 rte_stats->imissed = stats->mpc;
1710 rte_stats->ierrors = stats->crcerrs +
1711 stats->rlec + stats->ruc + stats->roc +
1712 stats->rxerrc + stats->algnerrc;
1715 rte_stats->oerrors = stats->ecol + stats->latecol;
1717 rte_stats->ipackets = stats->gprc;
1718 rte_stats->opackets = stats->gptc;
1719 rte_stats->ibytes = stats->gorc;
1720 rte_stats->obytes = stats->gotc;
1722 /* Get per-queue statuses */
1723 for (i = 0; i < IGC_QUEUE_PAIRS_NUM; i++) {
1724 /* GET TX queue statuses */
1725 int map_id = igc->txq_stats_map[i];
1727 rte_stats->q_opackets[map_id] += queue_stats->pqgptc[i];
1728 rte_stats->q_obytes[map_id] += queue_stats->pqgotc[i];
1730 /* Get RX queue statuses */
1731 map_id = igc->rxq_stats_map[i];
1733 rte_stats->q_ipackets[map_id] += queue_stats->pqgprc[i];
1734 rte_stats->q_ibytes[map_id] += queue_stats->pqgorc[i];
1735 rte_stats->q_errors[map_id] += queue_stats->rqdpc[i];
1739 /* Restart queue status handler */
1740 rte_eal_alarm_set(IGC_ALARM_INTERVAL,
1741 igc_update_queue_stats_handler, dev);
1746 eth_igc_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1749 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1750 struct igc_hw_stats *hw_stats =
1751 IGC_DEV_PRIVATE_STATS(dev);
1754 igc_read_stats_registers(hw, hw_stats);
1756 if (n < IGC_NB_XSTATS)
1757 return IGC_NB_XSTATS;
1759 /* If this is a reset xstats is NULL, and we have cleared the
1760 * registers by reading them.
1765 /* Extended stats */
1766 for (i = 0; i < IGC_NB_XSTATS; i++) {
1768 xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
1769 rte_igc_stats_strings[i].offset);
1772 return IGC_NB_XSTATS;
1776 eth_igc_xstats_reset(struct rte_eth_dev *dev)
1778 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1779 struct igc_hw_stats *hw_stats = IGC_DEV_PRIVATE_STATS(dev);
1780 struct igc_hw_queue_stats *queue_stats =
1781 IGC_DEV_PRIVATE_QUEUE_STATS(dev);
1783 /* Cancel queue status handler for avoid conflict */
1784 rte_eal_alarm_cancel(igc_update_queue_stats_handler, dev);
1786 /* HW registers are cleared on read */
1787 igc_reset_queue_stats_register(hw);
1788 igc_read_stats_registers(hw, hw_stats);
1790 /* Reset software totals */
1791 memset(hw_stats, 0, sizeof(*hw_stats));
1792 memset(queue_stats, 0, sizeof(*queue_stats));
1794 /* Restart the queue status handler */
1795 rte_eal_alarm_set(IGC_ALARM_INTERVAL, igc_update_queue_stats_handler,
1802 eth_igc_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1803 struct rte_eth_xstat_name *xstats_names, unsigned int size)
1807 if (xstats_names == NULL)
1808 return IGC_NB_XSTATS;
1810 if (size < IGC_NB_XSTATS) {
1811 PMD_DRV_LOG(ERR, "not enough buffers!");
1812 return IGC_NB_XSTATS;
1815 for (i = 0; i < IGC_NB_XSTATS; i++)
1816 strlcpy(xstats_names[i].name, rte_igc_stats_strings[i].name,
1817 sizeof(xstats_names[i].name));
1819 return IGC_NB_XSTATS;
1823 eth_igc_xstats_get_names_by_id(struct rte_eth_dev *dev,
1824 struct rte_eth_xstat_name *xstats_names, const uint64_t *ids,
1830 return eth_igc_xstats_get_names(dev, xstats_names, limit);
1832 for (i = 0; i < limit; i++) {
1833 if (ids[i] >= IGC_NB_XSTATS) {
1834 PMD_DRV_LOG(ERR, "id value isn't valid");
1837 strlcpy(xstats_names[i].name,
1838 rte_igc_stats_strings[ids[i]].name,
1839 sizeof(xstats_names[i].name));
1845 eth_igc_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids,
1846 uint64_t *values, unsigned int n)
1848 struct igc_hw *hw = IGC_DEV_PRIVATE_HW(dev);
1849 struct igc_hw_stats *hw_stats = IGC_DEV_PRIVATE_STATS(dev);
1852 igc_read_stats_registers(hw, hw_stats);
1855 if (n < IGC_NB_XSTATS)
1856 return IGC_NB_XSTATS;
1858 /* If this is a reset xstats is NULL, and we have cleared the
1859 * registers by reading them.
1864 /* Extended stats */
1865 for (i = 0; i < IGC_NB_XSTATS; i++)
1866 values[i] = *(uint64_t *)(((char *)hw_stats) +
1867 rte_igc_stats_strings[i].offset);
1869 return IGC_NB_XSTATS;
1872 for (i = 0; i < n; i++) {
1873 if (ids[i] >= IGC_NB_XSTATS) {
1874 PMD_DRV_LOG(ERR, "id value isn't valid");
1877 values[i] = *(uint64_t *)(((char *)hw_stats) +
1878 rte_igc_stats_strings[ids[i]].offset);
1885 eth_igc_queue_stats_mapping_set(struct rte_eth_dev *dev,
1886 uint16_t queue_id, uint8_t stat_idx, uint8_t is_rx)
1888 struct igc_adapter *igc = IGC_DEV_PRIVATE(dev);
1890 /* check queue id is valid */
1891 if (queue_id >= IGC_QUEUE_PAIRS_NUM) {
1892 PMD_DRV_LOG(ERR, "queue id(%u) error, max is %u",
1893 queue_id, IGC_QUEUE_PAIRS_NUM - 1);
1897 /* store the mapping status id */
1899 igc->rxq_stats_map[queue_id] = stat_idx;
1901 igc->txq_stats_map[queue_id] = stat_idx;
1907 eth_igc_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1908 struct rte_pci_device *pci_dev)
1910 PMD_INIT_FUNC_TRACE();
1911 return rte_eth_dev_pci_generic_probe(pci_dev,
1912 sizeof(struct igc_adapter), eth_igc_dev_init);
1916 eth_igc_pci_remove(struct rte_pci_device *pci_dev)
1918 PMD_INIT_FUNC_TRACE();
1919 return rte_eth_dev_pci_generic_remove(pci_dev, eth_igc_dev_uninit);
1922 static struct rte_pci_driver rte_igc_pmd = {
1923 .id_table = pci_id_igc_map,
1924 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
1925 .probe = eth_igc_pci_probe,
1926 .remove = eth_igc_pci_remove,
1929 RTE_PMD_REGISTER_PCI(net_igc, rte_igc_pmd);
1930 RTE_PMD_REGISTER_PCI_TABLE(net_igc, pci_id_igc_map);
1931 RTE_PMD_REGISTER_KMOD_DEP(net_igc, "* igb_uio | uio_pci_generic | vfio-pci");