4 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <rte_ethdev.h>
35 #include <rte_malloc.h>
36 #include <rte_memzone.h>
37 #include <rte_string_fns.h>
39 #include <rte_spinlock.h>
40 #include <rte_kvargs.h>
43 #include "base/fm10k_api.h"
45 /* Default delay to acquire mailbox lock */
46 #define FM10K_MBXLOCK_DELAY_US 20
47 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
49 #define MAIN_VSI_POOL_NUMBER 0
51 /* Max try times to acquire switch status */
52 #define MAX_QUERY_SWITCH_STATE_TIMES 10
53 /* Wait interval to get switch status */
54 #define WAIT_SWITCH_MSG_US 100000
55 /* Number of chars per uint32 type */
56 #define CHARS_PER_UINT32 (sizeof(uint32_t))
57 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
59 /* default 1:1 map from queue ID to interrupt vector ID */
60 #define Q2V(dev, queue_id) (dev->pci_dev->intr_handle.intr_vec[queue_id])
62 /* First 64 Logical ports for PF/VMDQ, second 64 for Flow director */
63 #define MAX_LPORT_NUM 128
64 #define GLORT_FD_Q_BASE 0x40
65 #define GLORT_PF_MASK 0xFFC0
66 #define GLORT_FD_MASK GLORT_PF_MASK
67 #define GLORT_FD_INDEX GLORT_FD_Q_BASE
69 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
70 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
71 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
72 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
73 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
74 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
76 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
77 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
78 const u8 *mac, bool add, uint32_t pool);
79 static void fm10k_tx_queue_release(void *queue);
80 static void fm10k_rx_queue_release(void *queue);
81 static void fm10k_set_rx_function(struct rte_eth_dev *dev);
82 static void fm10k_set_tx_function(struct rte_eth_dev *dev);
83 static int fm10k_check_ftag(struct rte_devargs *devargs);
85 struct fm10k_xstats_name_off {
86 char name[RTE_ETH_XSTATS_NAME_SIZE];
90 struct fm10k_xstats_name_off fm10k_hw_stats_strings[] = {
91 {"completion_timeout_count", offsetof(struct fm10k_hw_stats, timeout)},
92 {"unsupported_requests_count", offsetof(struct fm10k_hw_stats, ur)},
93 {"completer_abort_count", offsetof(struct fm10k_hw_stats, ca)},
94 {"unsupported_message_count", offsetof(struct fm10k_hw_stats, um)},
95 {"checksum_error_count", offsetof(struct fm10k_hw_stats, xec)},
96 {"vlan_dropped", offsetof(struct fm10k_hw_stats, vlan_drop)},
97 {"loopback_dropped", offsetof(struct fm10k_hw_stats, loopback_drop)},
98 {"rx_mbuf_allocation_errors", offsetof(struct fm10k_hw_stats,
102 #define FM10K_NB_HW_XSTATS (sizeof(fm10k_hw_stats_strings) / \
103 sizeof(fm10k_hw_stats_strings[0]))
105 struct fm10k_xstats_name_off fm10k_hw_stats_rx_q_strings[] = {
106 {"packets", offsetof(struct fm10k_hw_stats_q, rx_packets)},
107 {"bytes", offsetof(struct fm10k_hw_stats_q, rx_bytes)},
108 {"dropped", offsetof(struct fm10k_hw_stats_q, rx_drops)},
111 #define FM10K_NB_RX_Q_XSTATS (sizeof(fm10k_hw_stats_rx_q_strings) / \
112 sizeof(fm10k_hw_stats_rx_q_strings[0]))
114 struct fm10k_xstats_name_off fm10k_hw_stats_tx_q_strings[] = {
115 {"packets", offsetof(struct fm10k_hw_stats_q, tx_packets)},
116 {"bytes", offsetof(struct fm10k_hw_stats_q, tx_bytes)},
119 #define FM10K_NB_TX_Q_XSTATS (sizeof(fm10k_hw_stats_tx_q_strings) / \
120 sizeof(fm10k_hw_stats_tx_q_strings[0]))
122 #define FM10K_NB_XSTATS (FM10K_NB_HW_XSTATS + FM10K_MAX_QUEUES_PF * \
123 (FM10K_NB_RX_Q_XSTATS + FM10K_NB_TX_Q_XSTATS))
125 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
128 fm10k_mbx_initlock(struct fm10k_hw *hw)
130 rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
134 fm10k_mbx_lock(struct fm10k_hw *hw)
136 while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
137 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
141 fm10k_mbx_unlock(struct fm10k_hw *hw)
143 rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
146 /* Stubs needed for linkage when vPMD is disabled */
147 int __attribute__((weak))
148 fm10k_rx_vec_condition_check(__rte_unused struct rte_eth_dev *dev)
153 uint16_t __attribute__((weak))
155 __rte_unused void *rx_queue,
156 __rte_unused struct rte_mbuf **rx_pkts,
157 __rte_unused uint16_t nb_pkts)
162 uint16_t __attribute__((weak))
163 fm10k_recv_scattered_pkts_vec(
164 __rte_unused void *rx_queue,
165 __rte_unused struct rte_mbuf **rx_pkts,
166 __rte_unused uint16_t nb_pkts)
171 int __attribute__((weak))
172 fm10k_rxq_vec_setup(__rte_unused struct fm10k_rx_queue *rxq)
178 void __attribute__((weak))
179 fm10k_rx_queue_release_mbufs_vec(
180 __rte_unused struct fm10k_rx_queue *rxq)
185 void __attribute__((weak))
186 fm10k_txq_vec_setup(__rte_unused struct fm10k_tx_queue *txq)
191 int __attribute__((weak))
192 fm10k_tx_vec_condition_check(__rte_unused struct fm10k_tx_queue *txq)
197 uint16_t __attribute__((weak))
198 fm10k_xmit_pkts_vec(__rte_unused void *tx_queue,
199 __rte_unused struct rte_mbuf **tx_pkts,
200 __rte_unused uint16_t nb_pkts)
206 * reset queue to initial state, allocate software buffers used when starting
208 * return 0 on success
209 * return -ENOMEM if buffers cannot be allocated
210 * return -EINVAL if buffers do not satisfy alignment condition
213 rx_queue_reset(struct fm10k_rx_queue *q)
215 static const union fm10k_rx_desc zero = {{0} };
218 PMD_INIT_FUNC_TRACE();
220 diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
224 for (i = 0; i < q->nb_desc; ++i) {
225 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
226 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
227 rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
231 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
232 q->hw_ring[i].q.pkt_addr = dma_addr;
233 q->hw_ring[i].q.hdr_addr = dma_addr;
236 /* initialize extra software ring entries. Space for these extra
237 * entries is always allocated.
239 memset(&q->fake_mbuf, 0x0, sizeof(q->fake_mbuf));
240 for (i = 0; i < q->nb_fake_desc; ++i) {
241 q->sw_ring[q->nb_desc + i] = &q->fake_mbuf;
242 q->hw_ring[q->nb_desc + i] = zero;
247 q->next_trigger = q->alloc_thresh - 1;
248 FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
249 q->rxrearm_start = 0;
256 * clean queue, descriptor rings, free software buffers used when stopping
260 rx_queue_clean(struct fm10k_rx_queue *q)
262 union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
264 PMD_INIT_FUNC_TRACE();
266 /* zero descriptor rings */
267 for (i = 0; i < q->nb_desc; ++i)
268 q->hw_ring[i] = zero;
270 /* zero faked descriptors */
271 for (i = 0; i < q->nb_fake_desc; ++i)
272 q->hw_ring[q->nb_desc + i] = zero;
274 /* vPMD driver has a different way of releasing mbufs. */
275 if (q->rx_using_sse) {
276 fm10k_rx_queue_release_mbufs_vec(q);
280 /* free software buffers */
281 for (i = 0; i < q->nb_desc; ++i) {
283 rte_pktmbuf_free_seg(q->sw_ring[i]);
284 q->sw_ring[i] = NULL;
290 * free all queue memory used when releasing the queue (i.e. configure)
293 rx_queue_free(struct fm10k_rx_queue *q)
295 PMD_INIT_FUNC_TRACE();
297 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
300 rte_free(q->sw_ring);
309 * disable RX queue, wait unitl HW finished necessary flush operation
312 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
316 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
317 FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
318 reg & ~FM10K_RXQCTL_ENABLE);
320 /* Wait 100us at most */
321 for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
323 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
324 if (!(reg & FM10K_RXQCTL_ENABLE))
328 if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
335 * reset queue to initial state, allocate software buffers used when starting
339 tx_queue_reset(struct fm10k_tx_queue *q)
341 PMD_INIT_FUNC_TRACE();
345 q->nb_free = q->nb_desc - 1;
346 fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
347 FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
351 * clean queue, descriptor rings, free software buffers used when stopping
355 tx_queue_clean(struct fm10k_tx_queue *q)
357 struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
359 PMD_INIT_FUNC_TRACE();
361 /* zero descriptor rings */
362 for (i = 0; i < q->nb_desc; ++i)
363 q->hw_ring[i] = zero;
365 /* free software buffers */
366 for (i = 0; i < q->nb_desc; ++i) {
368 rte_pktmbuf_free_seg(q->sw_ring[i]);
369 q->sw_ring[i] = NULL;
375 * free all queue memory used when releasing the queue (i.e. configure)
378 tx_queue_free(struct fm10k_tx_queue *q)
380 PMD_INIT_FUNC_TRACE();
382 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
384 if (q->rs_tracker.list) {
385 rte_free(q->rs_tracker.list);
386 q->rs_tracker.list = NULL;
389 rte_free(q->sw_ring);
398 * disable TX queue, wait unitl HW finished necessary flush operation
401 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
405 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
406 FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
407 reg & ~FM10K_TXDCTL_ENABLE);
409 /* Wait 100us at most */
410 for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
412 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
413 if (!(reg & FM10K_TXDCTL_ENABLE))
417 if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
424 fm10k_check_mq_mode(struct rte_eth_dev *dev)
426 enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
427 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
428 struct rte_eth_vmdq_rx_conf *vmdq_conf;
429 uint16_t nb_rx_q = dev->data->nb_rx_queues;
431 vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
433 if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
434 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
438 if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
441 if (hw->mac.type == fm10k_mac_vf) {
442 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
446 /* Check VMDQ queue pool number */
447 if (vmdq_conf->nb_queue_pools >
448 sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
449 vmdq_conf->nb_queue_pools > nb_rx_q) {
450 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
451 vmdq_conf->nb_queue_pools);
458 static const struct fm10k_txq_ops def_txq_ops = {
459 .reset = tx_queue_reset,
463 fm10k_dev_configure(struct rte_eth_dev *dev)
467 PMD_INIT_FUNC_TRACE();
469 if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
470 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
471 /* multipe queue mode checking */
472 ret = fm10k_check_mq_mode(dev);
474 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
482 /* fls = find last set bit = 32 minus the number of leading zeros */
484 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
488 fm10k_dev_vmdq_rx_configure(struct rte_eth_dev *dev)
490 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
491 struct rte_eth_vmdq_rx_conf *vmdq_conf;
494 vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
496 for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
497 if (!vmdq_conf->pool_map[i].pools)
500 fm10k_update_vlan(hw, vmdq_conf->pool_map[i].vlan_id, 0, true);
501 fm10k_mbx_unlock(hw);
506 fm10k_dev_pf_main_vsi_reset(struct rte_eth_dev *dev)
508 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
510 /* Add default mac address */
511 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
512 MAIN_VSI_POOL_NUMBER);
516 fm10k_dev_rss_configure(struct rte_eth_dev *dev)
518 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
519 struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
520 uint32_t mrqc, *key, i, reta, j;
523 #define RSS_KEY_SIZE 40
524 static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
525 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
526 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
527 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
528 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
529 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
532 if (dev->data->nb_rx_queues == 1 ||
533 dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
534 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
537 /* random key is rss_intel_key (default) or user provided (rss_key) */
538 if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
539 key = (uint32_t *)rss_intel_key;
541 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
543 /* Now fill our hash function seeds, 4 bytes at a time */
544 for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
545 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
548 * Fill in redirection table
549 * The byte-swap is needed because NIC registers are in
550 * little-endian order.
553 for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
554 if (j == dev->data->nb_rx_queues)
556 reta = (reta << CHAR_BIT) | j;
558 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
563 * Generate RSS hash based on packet types, TCP/UDP
564 * port numbers and/or IPv4/v6 src and dst addresses
566 hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
568 mrqc |= (hf & ETH_RSS_IPV4) ? FM10K_MRQC_IPV4 : 0;
569 mrqc |= (hf & ETH_RSS_IPV6) ? FM10K_MRQC_IPV6 : 0;
570 mrqc |= (hf & ETH_RSS_IPV6_EX) ? FM10K_MRQC_IPV6 : 0;
571 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP) ? FM10K_MRQC_TCP_IPV4 : 0;
572 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP) ? FM10K_MRQC_TCP_IPV6 : 0;
573 mrqc |= (hf & ETH_RSS_IPV6_TCP_EX) ? FM10K_MRQC_TCP_IPV6 : 0;
574 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP) ? FM10K_MRQC_UDP_IPV4 : 0;
575 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP) ? FM10K_MRQC_UDP_IPV6 : 0;
576 mrqc |= (hf & ETH_RSS_IPV6_UDP_EX) ? FM10K_MRQC_UDP_IPV6 : 0;
579 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
584 FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
588 fm10k_dev_logic_port_update(struct rte_eth_dev *dev, uint16_t nb_lport_new)
590 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
593 for (i = 0; i < nb_lport_new; i++) {
594 /* Set unicast mode by default. App can change
595 * to other mode in other API func.
598 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map + i,
599 FM10K_XCAST_MODE_NONE);
600 fm10k_mbx_unlock(hw);
605 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
607 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
608 struct rte_eth_vmdq_rx_conf *vmdq_conf;
609 struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
610 struct fm10k_macvlan_filter_info *macvlan;
611 uint16_t nb_queue_pools = 0; /* pool number in configuration */
612 uint16_t nb_lport_new;
614 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
615 vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
617 fm10k_dev_rss_configure(dev);
619 /* only PF supports VMDQ */
620 if (hw->mac.type != fm10k_mac_pf)
623 if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG)
624 nb_queue_pools = vmdq_conf->nb_queue_pools;
626 /* no pool number change, no need to update logic port and VLAN/MAC */
627 if (macvlan->nb_queue_pools == nb_queue_pools)
630 nb_lport_new = nb_queue_pools ? nb_queue_pools : 1;
631 fm10k_dev_logic_port_update(dev, nb_lport_new);
633 /* reset MAC/VLAN as it's based on VMDQ or PF main VSI */
634 memset(dev->data->mac_addrs, 0,
635 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM);
636 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
637 &dev->data->mac_addrs[0]);
638 memset(macvlan, 0, sizeof(*macvlan));
639 macvlan->nb_queue_pools = nb_queue_pools;
642 fm10k_dev_vmdq_rx_configure(dev);
644 fm10k_dev_pf_main_vsi_reset(dev);
648 fm10k_dev_tx_init(struct rte_eth_dev *dev)
650 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
652 struct fm10k_tx_queue *txq;
656 /* Disable TXINT to avoid possible interrupt */
657 for (i = 0; i < hw->mac.max_queues; i++)
658 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
659 3 << FM10K_TXINT_TIMER_SHIFT);
662 for (i = 0; i < dev->data->nb_tx_queues; ++i) {
663 txq = dev->data->tx_queues[i];
664 base_addr = txq->hw_ring_phys_addr;
665 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
667 /* disable queue to avoid issues while updating state */
668 ret = tx_queue_disable(hw, i);
670 PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
673 /* Enable use of FTAG bit in TX descriptor, PFVTCTL
674 * register is read-only for VF.
676 if (fm10k_check_ftag(dev->pci_dev->devargs)) {
677 if (hw->mac.type == fm10k_mac_pf) {
678 FM10K_WRITE_REG(hw, FM10K_PFVTCTL(i),
679 FM10K_PFVTCTL_FTAG_DESC_ENABLE);
680 PMD_INIT_LOG(DEBUG, "FTAG mode is enabled");
682 PMD_INIT_LOG(ERR, "VF FTAG is not supported.");
687 /* set location and size for descriptor ring */
688 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
689 base_addr & UINT64_LOWER_32BITS_MASK);
690 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
691 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
692 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
694 /* assign default SGLORT for each TX queue */
695 FM10K_WRITE_REG(hw, FM10K_TX_SGLORT(i), hw->mac.dglort_map);
698 /* set up vector or scalar TX function as appropriate */
699 fm10k_set_tx_function(dev);
705 fm10k_dev_rx_init(struct rte_eth_dev *dev)
707 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
708 struct fm10k_macvlan_filter_info *macvlan;
709 struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
711 struct fm10k_rx_queue *rxq;
714 uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
715 uint32_t logic_port = hw->mac.dglort_map;
717 uint16_t queue_stride = 0;
719 /* enable RXINT for interrupt mode */
721 if (rte_intr_dp_is_en(intr_handle)) {
722 for (; i < dev->data->nb_rx_queues; i++) {
723 FM10K_WRITE_REG(hw, FM10K_RXINT(i), Q2V(dev, i));
724 if (hw->mac.type == fm10k_mac_pf)
725 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
727 FM10K_ITR_MASK_CLEAR);
729 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
731 FM10K_ITR_MASK_CLEAR);
734 /* Disable other RXINT to avoid possible interrupt */
735 for (; i < hw->mac.max_queues; i++)
736 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
737 3 << FM10K_RXINT_TIMER_SHIFT);
739 /* Setup RX queues */
740 for (i = 0; i < dev->data->nb_rx_queues; ++i) {
741 rxq = dev->data->rx_queues[i];
742 base_addr = rxq->hw_ring_phys_addr;
743 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
745 /* disable queue to avoid issues while updating state */
746 ret = rx_queue_disable(hw, i);
748 PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
752 /* Setup the Base and Length of the Rx Descriptor Ring */
753 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
754 base_addr & UINT64_LOWER_32BITS_MASK);
755 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
756 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
757 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
759 /* Configure the Rx buffer size for one buff without split */
760 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
761 RTE_PKTMBUF_HEADROOM);
762 /* As RX buffer is aligned to 512B within mbuf, some bytes are
763 * reserved for this purpose, and the worst case could be 511B.
764 * But SRR reg assumes all buffers have the same size. In order
765 * to fill the gap, we'll have to consider the worst case and
766 * assume 512B is reserved. If we don't do so, it's possible
767 * for HW to overwrite data to next mbuf.
769 buf_size -= FM10K_RX_DATABUF_ALIGN;
771 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
772 (buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT) |
773 FM10K_SRRCTL_LOOPBACK_SUPPRESS);
775 /* It adds dual VLAN length for supporting dual VLAN */
776 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
777 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
778 dev->data->dev_conf.rxmode.enable_scatter) {
780 dev->data->scattered_rx = 1;
781 reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
782 reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
783 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
786 /* Enable drop on empty, it's RO for VF */
787 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
788 rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
790 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
791 FM10K_WRITE_FLUSH(hw);
794 /* Configure VMDQ/RSS if applicable */
795 fm10k_dev_mq_rx_configure(dev);
797 /* Decide the best RX function */
798 fm10k_set_rx_function(dev);
800 /* update RX_SGLORT for loopback suppress*/
801 if (hw->mac.type != fm10k_mac_pf)
803 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
804 if (macvlan->nb_queue_pools)
805 queue_stride = dev->data->nb_rx_queues / macvlan->nb_queue_pools;
806 for (i = 0; i < dev->data->nb_rx_queues; ++i) {
807 if (i && queue_stride && !(i % queue_stride))
809 FM10K_WRITE_REG(hw, FM10K_RX_SGLORT(i), logic_port);
816 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
818 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
821 struct fm10k_rx_queue *rxq;
823 PMD_INIT_FUNC_TRACE();
825 if (rx_queue_id < dev->data->nb_rx_queues) {
826 rxq = dev->data->rx_queues[rx_queue_id];
827 err = rx_queue_reset(rxq);
828 if (err == -ENOMEM) {
829 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
831 } else if (err == -EINVAL) {
832 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
837 /* Setup the HW Rx Head and Tail Descriptor Pointers
838 * Note: this must be done AFTER the queue is enabled on real
839 * hardware, but BEFORE the queue is enabled when using the
840 * emulation platform. Do it in both places for now and remove
841 * this comment and the following two register writes when the
842 * emulation platform is no longer being used.
844 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
845 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
847 /* Set PF ownership flag for PF devices */
848 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
849 if (hw->mac.type == fm10k_mac_pf)
850 reg |= FM10K_RXQCTL_PF;
851 reg |= FM10K_RXQCTL_ENABLE;
852 /* enable RX queue */
853 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
854 FM10K_WRITE_FLUSH(hw);
856 /* Setup the HW Rx Head and Tail Descriptor Pointers
857 * Note: this must be done AFTER the queue is enabled
859 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
860 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
861 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
868 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
870 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
872 PMD_INIT_FUNC_TRACE();
874 if (rx_queue_id < dev->data->nb_rx_queues) {
875 /* Disable RX queue */
876 rx_queue_disable(hw, rx_queue_id);
878 /* Free mbuf and clean HW ring */
879 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
880 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
887 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
889 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
890 /** @todo - this should be defined in the shared code */
891 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY 0x00010000
892 uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
895 PMD_INIT_FUNC_TRACE();
897 if (tx_queue_id < dev->data->nb_tx_queues) {
898 struct fm10k_tx_queue *q = dev->data->tx_queues[tx_queue_id];
902 /* reset head and tail pointers */
903 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
904 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
906 /* enable TX queue */
907 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
908 FM10K_TXDCTL_ENABLE | txdctl);
909 FM10K_WRITE_FLUSH(hw);
910 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
918 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
920 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
922 PMD_INIT_FUNC_TRACE();
924 if (tx_queue_id < dev->data->nb_tx_queues) {
925 tx_queue_disable(hw, tx_queue_id);
926 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
927 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
933 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
935 return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
936 != FM10K_DGLORTMAP_NONE);
940 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
942 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
945 PMD_INIT_FUNC_TRACE();
947 /* Return if it didn't acquire valid glort range */
948 if (!fm10k_glort_valid(hw))
952 status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
953 FM10K_XCAST_MODE_PROMISC);
954 fm10k_mbx_unlock(hw);
956 if (status != FM10K_SUCCESS)
957 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
961 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
963 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
967 PMD_INIT_FUNC_TRACE();
969 /* Return if it didn't acquire valid glort range */
970 if (!fm10k_glort_valid(hw))
973 if (dev->data->all_multicast == 1)
974 mode = FM10K_XCAST_MODE_ALLMULTI;
976 mode = FM10K_XCAST_MODE_NONE;
979 status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
981 fm10k_mbx_unlock(hw);
983 if (status != FM10K_SUCCESS)
984 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
988 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
990 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
993 PMD_INIT_FUNC_TRACE();
995 /* Return if it didn't acquire valid glort range */
996 if (!fm10k_glort_valid(hw))
999 /* If promiscuous mode is enabled, it doesn't make sense to enable
1000 * allmulticast and disable promiscuous since fm10k only can select
1003 if (dev->data->promiscuous) {
1004 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
1005 "needn't enable allmulticast");
1010 status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1011 FM10K_XCAST_MODE_ALLMULTI);
1012 fm10k_mbx_unlock(hw);
1014 if (status != FM10K_SUCCESS)
1015 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
1019 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
1021 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1024 PMD_INIT_FUNC_TRACE();
1026 /* Return if it didn't acquire valid glort range */
1027 if (!fm10k_glort_valid(hw))
1030 if (dev->data->promiscuous) {
1031 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
1032 "since promisc mode is enabled");
1037 /* Change mode to unicast mode */
1038 status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1039 FM10K_XCAST_MODE_NONE);
1040 fm10k_mbx_unlock(hw);
1042 if (status != FM10K_SUCCESS)
1043 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
1047 fm10k_dev_dglort_map_configure(struct rte_eth_dev *dev)
1049 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1050 uint32_t dglortdec, pool_len, rss_len, i, dglortmask;
1051 uint16_t nb_queue_pools;
1052 struct fm10k_macvlan_filter_info *macvlan;
1054 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1055 nb_queue_pools = macvlan->nb_queue_pools;
1056 pool_len = nb_queue_pools ? fls(nb_queue_pools - 1) : 0;
1057 rss_len = fls(dev->data->nb_rx_queues - 1) - pool_len;
1059 /* GLORT 0x0-0x3F are used by PF and VMDQ, 0x40-0x7F used by FD */
1060 dglortdec = (rss_len << FM10K_DGLORTDEC_RSSLENGTH_SHIFT) | pool_len;
1061 dglortmask = (GLORT_PF_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1063 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), dglortmask);
1064 /* Configure VMDQ/RSS DGlort Decoder */
1065 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0), dglortdec);
1067 /* Flow Director configurations, only queue number is valid. */
1068 dglortdec = fls(dev->data->nb_rx_queues - 1);
1069 dglortmask = (GLORT_FD_MASK << FM10K_DGLORTMAP_MASK_SHIFT) |
1070 (hw->mac.dglort_map + GLORT_FD_Q_BASE);
1071 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(1), dglortmask);
1072 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(1), dglortdec);
1074 /* Invalidate all other GLORT entries */
1075 for (i = 2; i < FM10K_DGLORT_COUNT; i++)
1076 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
1077 FM10K_DGLORTMAP_NONE);
1080 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
1082 fm10k_dev_start(struct rte_eth_dev *dev)
1084 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1087 PMD_INIT_FUNC_TRACE();
1089 /* stop, init, then start the hw */
1090 diag = fm10k_stop_hw(hw);
1091 if (diag != FM10K_SUCCESS) {
1092 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
1096 diag = fm10k_init_hw(hw);
1097 if (diag != FM10K_SUCCESS) {
1098 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1102 diag = fm10k_start_hw(hw);
1103 if (diag != FM10K_SUCCESS) {
1104 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
1108 diag = fm10k_dev_tx_init(dev);
1110 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
1114 if (fm10k_dev_rxq_interrupt_setup(dev))
1117 diag = fm10k_dev_rx_init(dev);
1119 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
1123 if (hw->mac.type == fm10k_mac_pf)
1124 fm10k_dev_dglort_map_configure(dev);
1126 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1127 struct fm10k_rx_queue *rxq;
1128 rxq = dev->data->rx_queues[i];
1130 if (rxq->rx_deferred_start)
1132 diag = fm10k_dev_rx_queue_start(dev, i);
1135 for (j = 0; j < i; ++j)
1136 rx_queue_clean(dev->data->rx_queues[j]);
1141 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1142 struct fm10k_tx_queue *txq;
1143 txq = dev->data->tx_queues[i];
1145 if (txq->tx_deferred_start)
1147 diag = fm10k_dev_tx_queue_start(dev, i);
1150 for (j = 0; j < i; ++j)
1151 tx_queue_clean(dev->data->tx_queues[j]);
1152 for (j = 0; j < dev->data->nb_rx_queues; ++j)
1153 rx_queue_clean(dev->data->rx_queues[j]);
1158 /* Update default vlan when not in VMDQ mode */
1159 if (!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_VMDQ_FLAG))
1160 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
1166 fm10k_dev_stop(struct rte_eth_dev *dev)
1168 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1169 struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1172 PMD_INIT_FUNC_TRACE();
1174 if (dev->data->tx_queues)
1175 for (i = 0; i < dev->data->nb_tx_queues; i++)
1176 fm10k_dev_tx_queue_stop(dev, i);
1178 if (dev->data->rx_queues)
1179 for (i = 0; i < dev->data->nb_rx_queues; i++)
1180 fm10k_dev_rx_queue_stop(dev, i);
1182 /* Disable datapath event */
1183 if (rte_intr_dp_is_en(intr_handle)) {
1184 for (i = 0; i < dev->data->nb_rx_queues; i++) {
1185 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
1186 3 << FM10K_RXINT_TIMER_SHIFT);
1187 if (hw->mac.type == fm10k_mac_pf)
1188 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, i)),
1189 FM10K_ITR_MASK_SET);
1191 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, i)),
1192 FM10K_ITR_MASK_SET);
1195 /* Clean datapath event and queue/vec mapping */
1196 rte_intr_efd_disable(intr_handle);
1197 rte_free(intr_handle->intr_vec);
1198 intr_handle->intr_vec = NULL;
1202 fm10k_dev_queue_release(struct rte_eth_dev *dev)
1206 PMD_INIT_FUNC_TRACE();
1208 if (dev->data->tx_queues) {
1209 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1210 struct fm10k_tx_queue *txq = dev->data->tx_queues[i];
1216 if (dev->data->rx_queues) {
1217 for (i = 0; i < dev->data->nb_rx_queues; i++)
1218 fm10k_rx_queue_release(dev->data->rx_queues[i]);
1223 fm10k_dev_close(struct rte_eth_dev *dev)
1225 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1227 PMD_INIT_FUNC_TRACE();
1230 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
1231 MAX_LPORT_NUM, false);
1232 fm10k_mbx_unlock(hw);
1234 /* Stop mailbox service first */
1235 fm10k_close_mbx_service(hw);
1236 fm10k_dev_stop(dev);
1237 fm10k_dev_queue_release(dev);
1242 fm10k_link_update(struct rte_eth_dev *dev,
1243 __rte_unused int wait_to_complete)
1245 PMD_INIT_FUNC_TRACE();
1247 /* The host-interface link is always up. The speed is ~50Gbps per Gen3
1248 * x8 PCIe interface. For now, we leave the speed undefined since there
1249 * is no 50Gbps Ethernet. */
1250 dev->data->dev_link.link_speed = 0;
1251 dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1252 dev->data->dev_link.link_status = 1;
1258 fm10k_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstats *xstats,
1261 struct fm10k_hw_stats *hw_stats =
1262 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1263 unsigned i, q, count = 0;
1265 if (n < FM10K_NB_XSTATS)
1266 return FM10K_NB_XSTATS;
1269 for (i = 0; i < FM10K_NB_HW_XSTATS; i++) {
1270 snprintf(xstats[count].name, sizeof(xstats[count].name),
1271 "%s", fm10k_hw_stats_strings[count].name);
1272 xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
1273 fm10k_hw_stats_strings[count].offset);
1277 /* PF queue stats */
1278 for (q = 0; q < FM10K_MAX_QUEUES_PF; q++) {
1279 for (i = 0; i < FM10K_NB_RX_Q_XSTATS; i++) {
1280 snprintf(xstats[count].name, sizeof(xstats[count].name),
1282 fm10k_hw_stats_rx_q_strings[i].name);
1283 xstats[count].value =
1284 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1285 fm10k_hw_stats_rx_q_strings[i].offset);
1288 for (i = 0; i < FM10K_NB_TX_Q_XSTATS; i++) {
1289 snprintf(xstats[count].name, sizeof(xstats[count].name),
1291 fm10k_hw_stats_tx_q_strings[i].name);
1292 xstats[count].value =
1293 *(uint64_t *)(((char *)&hw_stats->q[q]) +
1294 fm10k_hw_stats_tx_q_strings[i].offset);
1299 return FM10K_NB_XSTATS;
1303 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1305 uint64_t ipackets, opackets, ibytes, obytes;
1306 struct fm10k_hw *hw =
1307 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1308 struct fm10k_hw_stats *hw_stats =
1309 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1312 PMD_INIT_FUNC_TRACE();
1314 fm10k_update_hw_stats(hw, hw_stats);
1316 ipackets = opackets = ibytes = obytes = 0;
1317 for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1318 (i < hw->mac.max_queues); ++i) {
1319 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
1320 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
1321 stats->q_ibytes[i] = hw_stats->q[i].rx_bytes.count;
1322 stats->q_obytes[i] = hw_stats->q[i].tx_bytes.count;
1323 ipackets += stats->q_ipackets[i];
1324 opackets += stats->q_opackets[i];
1325 ibytes += stats->q_ibytes[i];
1326 obytes += stats->q_obytes[i];
1328 stats->ipackets = ipackets;
1329 stats->opackets = opackets;
1330 stats->ibytes = ibytes;
1331 stats->obytes = obytes;
1335 fm10k_stats_reset(struct rte_eth_dev *dev)
1337 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1338 struct fm10k_hw_stats *hw_stats =
1339 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
1341 PMD_INIT_FUNC_TRACE();
1343 memset(hw_stats, 0, sizeof(*hw_stats));
1344 fm10k_rebind_hw_stats(hw, hw_stats);
1348 fm10k_dev_infos_get(struct rte_eth_dev *dev,
1349 struct rte_eth_dev_info *dev_info)
1351 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1353 PMD_INIT_FUNC_TRACE();
1355 dev_info->min_rx_bufsize = FM10K_MIN_RX_BUF_SIZE;
1356 dev_info->max_rx_pktlen = FM10K_MAX_PKT_SIZE;
1357 dev_info->max_rx_queues = hw->mac.max_queues;
1358 dev_info->max_tx_queues = hw->mac.max_queues;
1359 dev_info->max_mac_addrs = FM10K_MAX_MACADDR_NUM;
1360 dev_info->max_hash_mac_addrs = 0;
1361 dev_info->max_vfs = dev->pci_dev->max_vfs;
1362 dev_info->vmdq_pool_base = 0;
1363 dev_info->vmdq_queue_base = 0;
1364 dev_info->max_vmdq_pools = ETH_32_POOLS;
1365 dev_info->vmdq_queue_num = FM10K_MAX_QUEUES_PF;
1366 dev_info->rx_offload_capa =
1367 DEV_RX_OFFLOAD_VLAN_STRIP |
1368 DEV_RX_OFFLOAD_IPV4_CKSUM |
1369 DEV_RX_OFFLOAD_UDP_CKSUM |
1370 DEV_RX_OFFLOAD_TCP_CKSUM;
1371 dev_info->tx_offload_capa =
1372 DEV_TX_OFFLOAD_VLAN_INSERT |
1373 DEV_TX_OFFLOAD_IPV4_CKSUM |
1374 DEV_TX_OFFLOAD_UDP_CKSUM |
1375 DEV_TX_OFFLOAD_TCP_CKSUM |
1376 DEV_TX_OFFLOAD_TCP_TSO;
1378 dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
1379 dev_info->reta_size = FM10K_MAX_RSS_INDICES;
1381 dev_info->default_rxconf = (struct rte_eth_rxconf) {
1383 .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1384 .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1385 .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1387 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1391 dev_info->default_txconf = (struct rte_eth_txconf) {
1393 .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1394 .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1395 .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1397 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1398 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1399 .txq_flags = FM10K_SIMPLE_TX_FLAG,
1402 dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
1403 .nb_max = FM10K_MAX_RX_DESC,
1404 .nb_min = FM10K_MIN_RX_DESC,
1405 .nb_align = FM10K_MULT_RX_DESC,
1408 dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
1409 .nb_max = FM10K_MAX_TX_DESC,
1410 .nb_min = FM10K_MIN_TX_DESC,
1411 .nb_align = FM10K_MULT_TX_DESC,
1416 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1419 uint16_t mac_num = 0;
1420 uint32_t vid_idx, vid_bit, mac_index;
1421 struct fm10k_hw *hw;
1422 struct fm10k_macvlan_filter_info *macvlan;
1423 struct rte_eth_dev_data *data = dev->data;
1425 hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1426 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1428 if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1429 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1433 if (vlan_id > ETH_VLAN_ID_MAX) {
1434 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1438 vid_idx = FM10K_VFTA_IDX(vlan_id);
1439 vid_bit = FM10K_VFTA_BIT(vlan_id);
1440 /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1441 if (on && (macvlan->vfta[vid_idx] & vid_bit))
1443 /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1444 if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1445 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1446 "in the VLAN filter table");
1451 result = fm10k_update_vlan(hw, vlan_id, 0, on);
1452 fm10k_mbx_unlock(hw);
1453 if (result != FM10K_SUCCESS) {
1454 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1458 for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1459 (result == FM10K_SUCCESS); mac_index++) {
1460 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1462 if (mac_num > macvlan->mac_num - 1) {
1463 PMD_INIT_LOG(ERR, "MAC address number "
1468 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1469 data->mac_addrs[mac_index].addr_bytes,
1471 fm10k_mbx_unlock(hw);
1474 if (result != FM10K_SUCCESS) {
1475 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1480 macvlan->vlan_num++;
1481 macvlan->vfta[vid_idx] |= vid_bit;
1483 macvlan->vlan_num--;
1484 macvlan->vfta[vid_idx] &= ~vid_bit;
1490 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1492 if (mask & ETH_VLAN_STRIP_MASK) {
1493 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1494 PMD_INIT_LOG(ERR, "VLAN stripping is "
1495 "always on in fm10k");
1498 if (mask & ETH_VLAN_EXTEND_MASK) {
1499 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1500 PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1501 "supported in fm10k");
1504 if (mask & ETH_VLAN_FILTER_MASK) {
1505 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1506 PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1510 /* Add/Remove a MAC address, and update filters to main VSI */
1511 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1512 const u8 *mac, bool add, uint32_t pool)
1514 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1515 struct fm10k_macvlan_filter_info *macvlan;
1518 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1520 if (pool != MAIN_VSI_POOL_NUMBER) {
1521 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1522 "mac to pool %u", pool);
1525 for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1526 if (!macvlan->vfta[j])
1528 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1529 if (!(macvlan->vfta[j] & (1 << k)))
1531 if (i + 1 > macvlan->vlan_num) {
1532 PMD_INIT_LOG(ERR, "vlan number not match");
1536 fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1537 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1538 fm10k_mbx_unlock(hw);
1544 /* Add/Remove a MAC address, and update filters to VMDQ */
1545 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1546 const u8 *mac, bool add, uint32_t pool)
1548 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1549 struct fm10k_macvlan_filter_info *macvlan;
1550 struct rte_eth_vmdq_rx_conf *vmdq_conf;
1553 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1554 vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1556 if (pool > macvlan->nb_queue_pools) {
1557 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1559 pool, macvlan->nb_queue_pools);
1562 for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1563 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1566 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1567 vmdq_conf->pool_map[i].vlan_id, add, 0);
1568 fm10k_mbx_unlock(hw);
1572 /* Add/Remove a MAC address, and update filters */
1573 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1574 const u8 *mac, bool add, uint32_t pool)
1576 struct fm10k_macvlan_filter_info *macvlan;
1578 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1580 if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1581 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1583 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1591 /* Add a MAC address, and update filters */
1593 fm10k_macaddr_add(struct rte_eth_dev *dev,
1594 struct ether_addr *mac_addr,
1598 struct fm10k_macvlan_filter_info *macvlan;
1600 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1601 fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1602 macvlan->mac_vmdq_id[index] = pool;
1605 /* Remove a MAC address, and update filters */
1607 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1609 struct rte_eth_dev_data *data = dev->data;
1610 struct fm10k_macvlan_filter_info *macvlan;
1612 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1613 fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1614 FALSE, macvlan->mac_vmdq_id[index]);
1615 macvlan->mac_vmdq_id[index] = 0;
1619 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1621 if ((request < min) || (request > max) || ((request % mult) != 0))
1629 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1631 if ((request < min) || (request > max) || ((div % request) != 0))
1638 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1640 uint16_t rx_free_thresh;
1642 if (conf->rx_free_thresh == 0)
1643 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1645 rx_free_thresh = conf->rx_free_thresh;
1647 /* make sure the requested threshold satisfies the constraints */
1648 if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1649 FM10K_RX_FREE_THRESH_MAX(q),
1650 FM10K_RX_FREE_THRESH_DIV(q),
1652 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1653 "less than or equal to %u, "
1654 "greater than or equal to %u, "
1655 "and a divisor of %u",
1656 rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1657 FM10K_RX_FREE_THRESH_MIN(q),
1658 FM10K_RX_FREE_THRESH_DIV(q));
1662 q->alloc_thresh = rx_free_thresh;
1663 q->drop_en = conf->rx_drop_en;
1664 q->rx_deferred_start = conf->rx_deferred_start;
1670 * Hardware requires specific alignment for Rx packet buffers. At
1671 * least one of the following two conditions must be satisfied.
1672 * 1. Address is 512B aligned
1673 * 2. Address is 8B aligned and buffer does not cross 4K boundary.
1675 * As such, the driver may need to adjust the DMA address within the
1676 * buffer by up to 512B.
1678 * return 1 if the element size is valid, otherwise return 0.
1681 mempool_element_size_valid(struct rte_mempool *mp)
1685 /* elt_size includes mbuf header and headroom */
1686 min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1687 RTE_PKTMBUF_HEADROOM;
1689 /* account for up to 512B of alignment */
1690 min_size -= FM10K_RX_DATABUF_ALIGN;
1692 /* sanity check for overflow */
1693 if (min_size > mp->elt_size)
1701 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1702 uint16_t nb_desc, unsigned int socket_id,
1703 const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1705 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1706 struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
1707 struct fm10k_rx_queue *q;
1708 const struct rte_memzone *mz;
1710 PMD_INIT_FUNC_TRACE();
1712 /* make sure the mempool element size can account for alignment. */
1713 if (!mempool_element_size_valid(mp)) {
1714 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1718 /* make sure a valid number of descriptors have been requested */
1719 if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1720 FM10K_MULT_RX_DESC, nb_desc)) {
1721 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1722 "less than or equal to %"PRIu32", "
1723 "greater than or equal to %u, "
1724 "and a multiple of %u",
1725 nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1726 FM10K_MULT_RX_DESC);
1731 * if this queue existed already, free the associated memory. The
1732 * queue cannot be reused in case we need to allocate memory on
1733 * different socket than was previously used.
1735 if (dev->data->rx_queues[queue_id] != NULL) {
1736 rx_queue_free(dev->data->rx_queues[queue_id]);
1737 dev->data->rx_queues[queue_id] = NULL;
1740 /* allocate memory for the queue structure */
1741 q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1744 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1750 q->nb_desc = nb_desc;
1751 q->nb_fake_desc = FM10K_MULT_RX_DESC;
1752 q->port_id = dev->data->port_id;
1753 q->queue_id = queue_id;
1754 q->tail_ptr = (volatile uint32_t *)
1755 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1756 if (handle_rxconf(q, conf))
1759 /* allocate memory for the software ring */
1760 q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1761 (nb_desc + q->nb_fake_desc) * sizeof(struct rte_mbuf *),
1762 RTE_CACHE_LINE_SIZE, socket_id);
1763 if (q->sw_ring == NULL) {
1764 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1770 * allocate memory for the hardware descriptor ring. A memzone large
1771 * enough to hold the maximum ring size is requested to allow for
1772 * resizing in later calls to the queue setup function.
1774 mz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_id,
1775 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC,
1778 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1779 rte_free(q->sw_ring);
1783 q->hw_ring = mz->addr;
1784 q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1786 /* Check if number of descs satisfied Vector requirement */
1787 if (!rte_is_power_of_2(nb_desc)) {
1788 PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
1789 "preconditions - canceling the feature for "
1790 "the whole port[%d]",
1791 q->queue_id, q->port_id);
1792 dev_info->rx_vec_allowed = false;
1794 fm10k_rxq_vec_setup(q);
1796 dev->data->rx_queues[queue_id] = q;
1801 fm10k_rx_queue_release(void *queue)
1803 PMD_INIT_FUNC_TRACE();
1805 rx_queue_free(queue);
1809 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1811 uint16_t tx_free_thresh;
1812 uint16_t tx_rs_thresh;
1814 /* constraint MACROs require that tx_free_thresh is configured
1815 * before tx_rs_thresh */
1816 if (conf->tx_free_thresh == 0)
1817 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1819 tx_free_thresh = conf->tx_free_thresh;
1821 /* make sure the requested threshold satisfies the constraints */
1822 if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1823 FM10K_TX_FREE_THRESH_MAX(q),
1824 FM10K_TX_FREE_THRESH_DIV(q),
1826 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1827 "less than or equal to %u, "
1828 "greater than or equal to %u, "
1829 "and a divisor of %u",
1830 tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1831 FM10K_TX_FREE_THRESH_MIN(q),
1832 FM10K_TX_FREE_THRESH_DIV(q));
1836 q->free_thresh = tx_free_thresh;
1838 if (conf->tx_rs_thresh == 0)
1839 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1841 tx_rs_thresh = conf->tx_rs_thresh;
1843 q->tx_deferred_start = conf->tx_deferred_start;
1845 /* make sure the requested threshold satisfies the constraints */
1846 if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1847 FM10K_TX_RS_THRESH_MAX(q),
1848 FM10K_TX_RS_THRESH_DIV(q),
1850 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1851 "less than or equal to %u, "
1852 "greater than or equal to %u, "
1853 "and a divisor of %u",
1854 tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1855 FM10K_TX_RS_THRESH_MIN(q),
1856 FM10K_TX_RS_THRESH_DIV(q));
1860 q->rs_thresh = tx_rs_thresh;
1866 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1867 uint16_t nb_desc, unsigned int socket_id,
1868 const struct rte_eth_txconf *conf)
1870 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1871 struct fm10k_tx_queue *q;
1872 const struct rte_memzone *mz;
1874 PMD_INIT_FUNC_TRACE();
1876 /* make sure a valid number of descriptors have been requested */
1877 if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1878 FM10K_MULT_TX_DESC, nb_desc)) {
1879 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1880 "less than or equal to %"PRIu32", "
1881 "greater than or equal to %u, "
1882 "and a multiple of %u",
1883 nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1884 FM10K_MULT_TX_DESC);
1889 * if this queue existed already, free the associated memory. The
1890 * queue cannot be reused in case we need to allocate memory on
1891 * different socket than was previously used.
1893 if (dev->data->tx_queues[queue_id] != NULL) {
1894 struct fm10k_tx_queue *txq = dev->data->tx_queues[queue_id];
1897 dev->data->tx_queues[queue_id] = NULL;
1900 /* allocate memory for the queue structure */
1901 q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1904 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1909 q->nb_desc = nb_desc;
1910 q->port_id = dev->data->port_id;
1911 q->queue_id = queue_id;
1912 q->txq_flags = conf->txq_flags;
1913 q->ops = &def_txq_ops;
1914 q->tail_ptr = (volatile uint32_t *)
1915 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1916 if (handle_txconf(q, conf))
1919 /* allocate memory for the software ring */
1920 q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1921 nb_desc * sizeof(struct rte_mbuf *),
1922 RTE_CACHE_LINE_SIZE, socket_id);
1923 if (q->sw_ring == NULL) {
1924 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1930 * allocate memory for the hardware descriptor ring. A memzone large
1931 * enough to hold the maximum ring size is requested to allow for
1932 * resizing in later calls to the queue setup function.
1934 mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
1935 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC,
1938 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1939 rte_free(q->sw_ring);
1943 q->hw_ring = mz->addr;
1944 q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1947 * allocate memory for the RS bit tracker. Enough slots to hold the
1948 * descriptor index for each RS bit needing to be set are required.
1950 q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1951 ((nb_desc + 1) / q->rs_thresh) *
1953 RTE_CACHE_LINE_SIZE, socket_id);
1954 if (q->rs_tracker.list == NULL) {
1955 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1956 rte_free(q->sw_ring);
1961 dev->data->tx_queues[queue_id] = q;
1966 fm10k_tx_queue_release(void *queue)
1968 struct fm10k_tx_queue *q = queue;
1969 PMD_INIT_FUNC_TRACE();
1975 fm10k_reta_update(struct rte_eth_dev *dev,
1976 struct rte_eth_rss_reta_entry64 *reta_conf,
1979 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1980 uint16_t i, j, idx, shift;
1984 PMD_INIT_FUNC_TRACE();
1986 if (reta_size > FM10K_MAX_RSS_INDICES) {
1987 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1988 "(%d) doesn't match the number hardware can supported "
1989 "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1994 * Update Redirection Table RETA[n], n=0..31. The redirection table has
1995 * 128-entries in 32 registers
1997 for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1998 idx = i / RTE_RETA_GROUP_SIZE;
1999 shift = i % RTE_RETA_GROUP_SIZE;
2000 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2001 BIT_MASK_PER_UINT32);
2006 if (mask != BIT_MASK_PER_UINT32)
2007 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2009 for (j = 0; j < CHARS_PER_UINT32; j++) {
2010 if (mask & (0x1 << j)) {
2012 reta &= ~(UINT8_MAX << CHAR_BIT * j);
2013 reta |= reta_conf[idx].reta[shift + j] <<
2017 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
2024 fm10k_reta_query(struct rte_eth_dev *dev,
2025 struct rte_eth_rss_reta_entry64 *reta_conf,
2028 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2029 uint16_t i, j, idx, shift;
2033 PMD_INIT_FUNC_TRACE();
2035 if (reta_size < FM10K_MAX_RSS_INDICES) {
2036 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
2037 "(%d) doesn't match the number hardware can supported "
2038 "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
2043 * Read Redirection Table RETA[n], n=0..31. The redirection table has
2044 * 128-entries in 32 registers
2046 for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
2047 idx = i / RTE_RETA_GROUP_SIZE;
2048 shift = i % RTE_RETA_GROUP_SIZE;
2049 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
2050 BIT_MASK_PER_UINT32);
2054 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
2055 for (j = 0; j < CHARS_PER_UINT32; j++) {
2056 if (mask & (0x1 << j))
2057 reta_conf[idx].reta[shift + j] = ((reta >>
2058 CHAR_BIT * j) & UINT8_MAX);
2066 fm10k_rss_hash_update(struct rte_eth_dev *dev,
2067 struct rte_eth_rss_conf *rss_conf)
2069 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2070 uint32_t *key = (uint32_t *)rss_conf->rss_key;
2072 uint64_t hf = rss_conf->rss_hf;
2075 PMD_INIT_FUNC_TRACE();
2077 if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2078 FM10K_RSSRK_ENTRIES_PER_REG)
2085 mrqc |= (hf & ETH_RSS_IPV4) ? FM10K_MRQC_IPV4 : 0;
2086 mrqc |= (hf & ETH_RSS_IPV6) ? FM10K_MRQC_IPV6 : 0;
2087 mrqc |= (hf & ETH_RSS_IPV6_EX) ? FM10K_MRQC_IPV6 : 0;
2088 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP) ? FM10K_MRQC_TCP_IPV4 : 0;
2089 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP) ? FM10K_MRQC_TCP_IPV6 : 0;
2090 mrqc |= (hf & ETH_RSS_IPV6_TCP_EX) ? FM10K_MRQC_TCP_IPV6 : 0;
2091 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP) ? FM10K_MRQC_UDP_IPV4 : 0;
2092 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP) ? FM10K_MRQC_UDP_IPV6 : 0;
2093 mrqc |= (hf & ETH_RSS_IPV6_UDP_EX) ? FM10K_MRQC_UDP_IPV6 : 0;
2095 /* If the mapping doesn't fit any supported, return */
2100 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2101 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
2103 FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
2109 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
2110 struct rte_eth_rss_conf *rss_conf)
2112 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2113 uint32_t *key = (uint32_t *)rss_conf->rss_key;
2118 PMD_INIT_FUNC_TRACE();
2120 if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
2121 FM10K_RSSRK_ENTRIES_PER_REG)
2125 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
2126 key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
2128 mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
2130 hf |= (mrqc & FM10K_MRQC_IPV4) ? ETH_RSS_IPV4 : 0;
2131 hf |= (mrqc & FM10K_MRQC_IPV6) ? ETH_RSS_IPV6 : 0;
2132 hf |= (mrqc & FM10K_MRQC_IPV6) ? ETH_RSS_IPV6_EX : 0;
2133 hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP : 0;
2134 hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP : 0;
2135 hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX : 0;
2136 hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP : 0;
2137 hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP : 0;
2138 hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX : 0;
2140 rss_conf->rss_hf = hf;
2146 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
2148 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2149 uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2151 /* Bind all local non-queue interrupt to vector 0 */
2152 int_map |= FM10K_MISC_VEC_ID;
2154 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2155 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2156 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2157 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2158 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2159 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2161 /* Enable misc causes */
2162 FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
2163 FM10K_EIMR_ENABLE(THI_FAULT) |
2164 FM10K_EIMR_ENABLE(FUM_FAULT) |
2165 FM10K_EIMR_ENABLE(MAILBOX) |
2166 FM10K_EIMR_ENABLE(SWITCHREADY) |
2167 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
2168 FM10K_EIMR_ENABLE(SRAMERROR) |
2169 FM10K_EIMR_ENABLE(VFLR));
2172 FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2173 FM10K_ITR_MASK_CLEAR);
2174 FM10K_WRITE_FLUSH(hw);
2178 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
2180 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2181 uint32_t int_map = FM10K_INT_MAP_DISABLE;
2183 int_map |= FM10K_MISC_VEC_ID;
2185 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_mailbox), int_map);
2186 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), int_map);
2187 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), int_map);
2188 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_switch_event), int_map);
2189 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_sram), int_map);
2190 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_vflr), int_map);
2192 /* Disable misc causes */
2193 FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
2194 FM10K_EIMR_DISABLE(THI_FAULT) |
2195 FM10K_EIMR_DISABLE(FUM_FAULT) |
2196 FM10K_EIMR_DISABLE(MAILBOX) |
2197 FM10K_EIMR_DISABLE(SWITCHREADY) |
2198 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
2199 FM10K_EIMR_DISABLE(SRAMERROR) |
2200 FM10K_EIMR_DISABLE(VFLR));
2203 FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
2204 FM10K_WRITE_FLUSH(hw);
2208 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
2210 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2211 uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
2213 /* Bind all local non-queue interrupt to vector 0 */
2214 int_map |= FM10K_MISC_VEC_ID;
2216 /* Only INT 0 available, other 15 are reserved. */
2217 FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2220 FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2221 FM10K_ITR_MASK_CLEAR);
2222 FM10K_WRITE_FLUSH(hw);
2226 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
2228 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2229 uint32_t int_map = FM10K_INT_MAP_DISABLE;
2231 int_map |= FM10K_MISC_VEC_ID;
2233 /* Only INT 0 available, other 15 are reserved. */
2234 FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
2237 FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
2238 FM10K_WRITE_FLUSH(hw);
2242 fm10k_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
2244 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2247 if (hw->mac.type == fm10k_mac_pf)
2248 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
2249 FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2251 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
2252 FM10K_ITR_AUTOMASK | FM10K_ITR_MASK_CLEAR);
2253 rte_intr_enable(&dev->pci_dev->intr_handle);
2258 fm10k_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
2260 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2263 if (hw->mac.type == fm10k_mac_pf)
2264 FM10K_WRITE_REG(hw, FM10K_ITR(Q2V(dev, queue_id)),
2265 FM10K_ITR_MASK_SET);
2267 FM10K_WRITE_REG(hw, FM10K_VFITR(Q2V(dev, queue_id)),
2268 FM10K_ITR_MASK_SET);
2273 fm10k_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
2275 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2276 struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2277 uint32_t intr_vector, vec;
2281 /* fm10k needs one separate interrupt for mailbox,
2282 * so only drivers which support multiple interrupt vectors
2283 * e.g. vfio-pci can work for fm10k interrupt mode
2285 if (!rte_intr_cap_multiple(intr_handle) ||
2286 dev->data->dev_conf.intr_conf.rxq == 0)
2289 intr_vector = dev->data->nb_rx_queues;
2291 /* disable interrupt first */
2292 rte_intr_disable(&dev->pci_dev->intr_handle);
2293 if (hw->mac.type == fm10k_mac_pf)
2294 fm10k_dev_disable_intr_pf(dev);
2296 fm10k_dev_disable_intr_vf(dev);
2298 if (rte_intr_efd_enable(intr_handle, intr_vector)) {
2299 PMD_INIT_LOG(ERR, "Failed to init event fd");
2303 if (rte_intr_dp_is_en(intr_handle) && !result) {
2304 intr_handle->intr_vec = rte_zmalloc("intr_vec",
2305 dev->data->nb_rx_queues * sizeof(int), 0);
2306 if (intr_handle->intr_vec) {
2307 for (queue_id = 0, vec = FM10K_RX_VEC_START;
2308 queue_id < dev->data->nb_rx_queues;
2310 intr_handle->intr_vec[queue_id] = vec;
2311 if (vec < intr_handle->nb_efd - 1
2312 + FM10K_RX_VEC_START)
2316 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2317 " intr_vec", dev->data->nb_rx_queues);
2318 rte_intr_efd_disable(intr_handle);
2323 if (hw->mac.type == fm10k_mac_pf)
2324 fm10k_dev_enable_intr_pf(dev);
2326 fm10k_dev_enable_intr_vf(dev);
2327 rte_intr_enable(&dev->pci_dev->intr_handle);
2328 hw->mac.ops.update_int_moderator(hw);
2333 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
2335 struct fm10k_fault fault;
2337 const char *estr = "Unknown error";
2339 /* Process PCA fault */
2340 if (eicr & FM10K_EICR_PCA_FAULT) {
2341 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
2344 switch (fault.type) {
2346 estr = "PCA_NO_FAULT"; break;
2347 case PCA_UNMAPPED_ADDR:
2348 estr = "PCA_UNMAPPED_ADDR"; break;
2349 case PCA_BAD_QACCESS_PF:
2350 estr = "PCA_BAD_QACCESS_PF"; break;
2351 case PCA_BAD_QACCESS_VF:
2352 estr = "PCA_BAD_QACCESS_VF"; break;
2353 case PCA_MALICIOUS_REQ:
2354 estr = "PCA_MALICIOUS_REQ"; break;
2355 case PCA_POISONED_TLP:
2356 estr = "PCA_POISONED_TLP"; break;
2358 estr = "PCA_TLP_ABORT"; break;
2362 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2363 estr, fault.func ? "VF" : "PF", fault.func,
2364 fault.address, fault.specinfo);
2367 /* Process THI fault */
2368 if (eicr & FM10K_EICR_THI_FAULT) {
2369 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
2372 switch (fault.type) {
2374 estr = "THI_NO_FAULT"; break;
2375 case THI_MAL_DIS_Q_FAULT:
2376 estr = "THI_MAL_DIS_Q_FAULT"; break;
2380 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2381 estr, fault.func ? "VF" : "PF", fault.func,
2382 fault.address, fault.specinfo);
2385 /* Process FUM fault */
2386 if (eicr & FM10K_EICR_FUM_FAULT) {
2387 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
2390 switch (fault.type) {
2392 estr = "FUM_NO_FAULT"; break;
2393 case FUM_UNMAPPED_ADDR:
2394 estr = "FUM_UNMAPPED_ADDR"; break;
2395 case FUM_POISONED_TLP:
2396 estr = "FUM_POISONED_TLP"; break;
2397 case FUM_BAD_VF_QACCESS:
2398 estr = "FUM_BAD_VF_QACCESS"; break;
2399 case FUM_ADD_DECODE_ERR:
2400 estr = "FUM_ADD_DECODE_ERR"; break;
2402 estr = "FUM_RO_ERROR"; break;
2403 case FUM_QPRC_CRC_ERROR:
2404 estr = "FUM_QPRC_CRC_ERROR"; break;
2405 case FUM_CSR_TIMEOUT:
2406 estr = "FUM_CSR_TIMEOUT"; break;
2407 case FUM_INVALID_TYPE:
2408 estr = "FUM_INVALID_TYPE"; break;
2409 case FUM_INVALID_LENGTH:
2410 estr = "FUM_INVALID_LENGTH"; break;
2411 case FUM_INVALID_BE:
2412 estr = "FUM_INVALID_BE"; break;
2413 case FUM_INVALID_ALIGN:
2414 estr = "FUM_INVALID_ALIGN"; break;
2418 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
2419 estr, fault.func ? "VF" : "PF", fault.func,
2420 fault.address, fault.specinfo);
2425 PMD_INIT_LOG(ERR, "Failed to handle fault event.");
2430 * PF interrupt handler triggered by NIC for handling specific interrupt.
2433 * Pointer to interrupt handle.
2435 * The address of parameter (struct rte_eth_dev *) regsitered before.
2441 fm10k_dev_interrupt_handler_pf(
2442 __rte_unused struct rte_intr_handle *handle,
2445 struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2446 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2447 uint32_t cause, status;
2449 if (hw->mac.type != fm10k_mac_pf)
2452 cause = FM10K_READ_REG(hw, FM10K_EICR);
2454 /* Handle PCI fault cases */
2455 if (cause & FM10K_EICR_FAULT_MASK) {
2456 PMD_INIT_LOG(ERR, "INT: find fault!");
2457 fm10k_dev_handle_fault(hw, cause);
2460 /* Handle switch up/down */
2461 if (cause & FM10K_EICR_SWITCHNOTREADY)
2462 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
2464 if (cause & FM10K_EICR_SWITCHREADY)
2465 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2467 /* Handle mailbox message */
2469 hw->mbx.ops.process(hw, &hw->mbx);
2470 fm10k_mbx_unlock(hw);
2472 /* Handle SRAM error */
2473 if (cause & FM10K_EICR_SRAMERROR) {
2474 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2476 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2477 /* Write to clear pending bits */
2478 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2480 /* Todo: print out error message after shared code updates */
2483 /* Clear these 3 events if having any */
2484 cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2485 FM10K_EICR_SWITCHREADY;
2487 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2489 /* Re-enable interrupt from device side */
2490 FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2491 FM10K_ITR_MASK_CLEAR);
2492 /* Re-enable interrupt from host side */
2493 rte_intr_enable(&(dev->pci_dev->intr_handle));
2497 * VF interrupt handler triggered by NIC for handling specific interrupt.
2500 * Pointer to interrupt handle.
2502 * The address of parameter (struct rte_eth_dev *) regsitered before.
2508 fm10k_dev_interrupt_handler_vf(
2509 __rte_unused struct rte_intr_handle *handle,
2512 struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2513 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2515 if (hw->mac.type != fm10k_mac_vf)
2518 /* Handle mailbox message if lock is acquired */
2520 hw->mbx.ops.process(hw, &hw->mbx);
2521 fm10k_mbx_unlock(hw);
2523 /* Re-enable interrupt from device side */
2524 FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2525 FM10K_ITR_MASK_CLEAR);
2526 /* Re-enable interrupt from host side */
2527 rte_intr_enable(&(dev->pci_dev->intr_handle));
2530 /* Mailbox message handler in VF */
2531 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2532 FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2533 FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2534 FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2535 FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2539 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2543 /* Initialize mailbox lock */
2544 fm10k_mbx_initlock(hw);
2546 /* Replace default message handler with new ones */
2547 if (hw->mac.type == fm10k_mac_vf)
2548 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2551 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2555 /* Connect to SM for PF device or PF for VF device */
2556 return hw->mbx.ops.connect(hw, &hw->mbx);
2560 fm10k_close_mbx_service(struct fm10k_hw *hw)
2562 /* Disconnect from SM for PF device or PF for VF device */
2563 hw->mbx.ops.disconnect(hw, &hw->mbx);
2566 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2567 .dev_configure = fm10k_dev_configure,
2568 .dev_start = fm10k_dev_start,
2569 .dev_stop = fm10k_dev_stop,
2570 .dev_close = fm10k_dev_close,
2571 .promiscuous_enable = fm10k_dev_promiscuous_enable,
2572 .promiscuous_disable = fm10k_dev_promiscuous_disable,
2573 .allmulticast_enable = fm10k_dev_allmulticast_enable,
2574 .allmulticast_disable = fm10k_dev_allmulticast_disable,
2575 .stats_get = fm10k_stats_get,
2576 .xstats_get = fm10k_xstats_get,
2577 .stats_reset = fm10k_stats_reset,
2578 .xstats_reset = fm10k_stats_reset,
2579 .link_update = fm10k_link_update,
2580 .dev_infos_get = fm10k_dev_infos_get,
2581 .vlan_filter_set = fm10k_vlan_filter_set,
2582 .vlan_offload_set = fm10k_vlan_offload_set,
2583 .mac_addr_add = fm10k_macaddr_add,
2584 .mac_addr_remove = fm10k_macaddr_remove,
2585 .rx_queue_start = fm10k_dev_rx_queue_start,
2586 .rx_queue_stop = fm10k_dev_rx_queue_stop,
2587 .tx_queue_start = fm10k_dev_tx_queue_start,
2588 .tx_queue_stop = fm10k_dev_tx_queue_stop,
2589 .rx_queue_setup = fm10k_rx_queue_setup,
2590 .rx_queue_release = fm10k_rx_queue_release,
2591 .tx_queue_setup = fm10k_tx_queue_setup,
2592 .tx_queue_release = fm10k_tx_queue_release,
2593 .rx_descriptor_done = fm10k_dev_rx_descriptor_done,
2594 .rx_queue_intr_enable = fm10k_dev_rx_queue_intr_enable,
2595 .rx_queue_intr_disable = fm10k_dev_rx_queue_intr_disable,
2596 .reta_update = fm10k_reta_update,
2597 .reta_query = fm10k_reta_query,
2598 .rss_hash_update = fm10k_rss_hash_update,
2599 .rss_hash_conf_get = fm10k_rss_hash_conf_get,
2602 static int ftag_check_handler(__rte_unused const char *key,
2603 const char *value, __rte_unused void *opaque)
2605 if (strcmp(value, "1"))
2612 fm10k_check_ftag(struct rte_devargs *devargs)
2614 struct rte_kvargs *kvlist;
2615 const char *ftag_key = "enable_ftag";
2617 if (devargs == NULL)
2620 kvlist = rte_kvargs_parse(devargs->args, NULL);
2624 if (!rte_kvargs_count(kvlist, ftag_key)) {
2625 rte_kvargs_free(kvlist);
2628 /* FTAG is enabled when there's key-value pair: enable_ftag=1 */
2629 if (rte_kvargs_process(kvlist, ftag_key,
2630 ftag_check_handler, NULL) < 0) {
2631 rte_kvargs_free(kvlist);
2634 rte_kvargs_free(kvlist);
2639 static void __attribute__((cold))
2640 fm10k_set_tx_function(struct rte_eth_dev *dev)
2642 struct fm10k_tx_queue *txq;
2645 uint16_t tx_ftag_en = 0;
2647 if (fm10k_check_ftag(dev->pci_dev->devargs))
2650 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2651 txq = dev->data->tx_queues[i];
2652 txq->tx_ftag_en = tx_ftag_en;
2653 /* Check if Vector Tx is satisfied */
2654 if (fm10k_tx_vec_condition_check(txq)) {
2661 PMD_INIT_LOG(DEBUG, "Use vector Tx func");
2662 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2663 txq = dev->data->tx_queues[i];
2664 fm10k_txq_vec_setup(txq);
2666 dev->tx_pkt_burst = fm10k_xmit_pkts_vec;
2668 dev->tx_pkt_burst = fm10k_xmit_pkts;
2669 PMD_INIT_LOG(DEBUG, "Use regular Tx func");
2673 static void __attribute__((cold))
2674 fm10k_set_rx_function(struct rte_eth_dev *dev)
2676 struct fm10k_dev_info *dev_info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2677 uint16_t i, rx_using_sse;
2678 uint16_t rx_ftag_en = 0;
2680 if (fm10k_check_ftag(dev->pci_dev->devargs))
2683 /* In order to allow Vector Rx there are a few configuration
2684 * conditions to be met.
2686 if (!fm10k_rx_vec_condition_check(dev) &&
2687 dev_info->rx_vec_allowed && !rx_ftag_en) {
2688 if (dev->data->scattered_rx)
2689 dev->rx_pkt_burst = fm10k_recv_scattered_pkts_vec;
2691 dev->rx_pkt_burst = fm10k_recv_pkts_vec;
2692 } else if (dev->data->scattered_rx)
2693 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
2695 dev->rx_pkt_burst = fm10k_recv_pkts;
2698 (dev->rx_pkt_burst == fm10k_recv_scattered_pkts_vec ||
2699 dev->rx_pkt_burst == fm10k_recv_pkts_vec);
2702 PMD_INIT_LOG(DEBUG, "Use vector Rx func");
2704 PMD_INIT_LOG(DEBUG, "Use regular Rx func");
2706 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2707 struct fm10k_rx_queue *rxq = dev->data->rx_queues[i];
2709 rxq->rx_using_sse = rx_using_sse;
2710 rxq->rx_ftag_en = rx_ftag_en;
2715 fm10k_params_init(struct rte_eth_dev *dev)
2717 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2718 struct fm10k_dev_info *info = FM10K_DEV_PRIVATE_TO_INFO(dev);
2720 /* Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2721 * there is no way to get link status without reading BAR4. Until this
2722 * works, assume we have maximum bandwidth.
2723 * @todo - fix bus info
2725 hw->bus_caps.speed = fm10k_bus_speed_8000;
2726 hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2727 hw->bus_caps.payload = fm10k_bus_payload_512;
2728 hw->bus.speed = fm10k_bus_speed_8000;
2729 hw->bus.width = fm10k_bus_width_pcie_x8;
2730 hw->bus.payload = fm10k_bus_payload_256;
2732 info->rx_vec_allowed = true;
2736 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2738 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2740 struct fm10k_macvlan_filter_info *macvlan;
2742 PMD_INIT_FUNC_TRACE();
2744 dev->dev_ops = &fm10k_eth_dev_ops;
2745 dev->rx_pkt_burst = &fm10k_recv_pkts;
2746 dev->tx_pkt_burst = &fm10k_xmit_pkts;
2748 /* only initialize in the primary process */
2749 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2752 rte_eth_copy_pci_info(dev, dev->pci_dev);
2754 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2755 memset(macvlan, 0, sizeof(*macvlan));
2756 /* Vendor and Device ID need to be set before init of shared code */
2757 memset(hw, 0, sizeof(*hw));
2758 hw->device_id = dev->pci_dev->id.device_id;
2759 hw->vendor_id = dev->pci_dev->id.vendor_id;
2760 hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2761 hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2762 hw->revision_id = 0;
2763 hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2764 if (hw->hw_addr == NULL) {
2765 PMD_INIT_LOG(ERR, "Bad mem resource."
2766 " Try to blacklist unused devices.");
2770 /* Store fm10k_adapter pointer */
2771 hw->back = dev->data->dev_private;
2773 /* Initialize the shared code */
2774 diag = fm10k_init_shared_code(hw);
2775 if (diag != FM10K_SUCCESS) {
2776 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2780 /* Initialize parameters */
2781 fm10k_params_init(dev);
2783 /* Initialize the hw */
2784 diag = fm10k_init_hw(hw);
2785 if (diag != FM10K_SUCCESS) {
2786 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2790 /* Initialize MAC address(es) */
2791 dev->data->mac_addrs = rte_zmalloc("fm10k",
2792 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2793 if (dev->data->mac_addrs == NULL) {
2794 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2798 diag = fm10k_read_mac_addr(hw);
2800 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2801 &dev->data->mac_addrs[0]);
2803 if (diag != FM10K_SUCCESS ||
2804 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2806 /* Generate a random addr */
2807 eth_random_addr(hw->mac.addr);
2808 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2809 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2810 &dev->data->mac_addrs[0]);
2813 /* Reset the hw statistics */
2814 fm10k_stats_reset(dev);
2817 diag = fm10k_reset_hw(hw);
2818 if (diag != FM10K_SUCCESS) {
2819 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2823 /* Setup mailbox service */
2824 diag = fm10k_setup_mbx_service(hw);
2825 if (diag != FM10K_SUCCESS) {
2826 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2830 /*PF/VF has different interrupt handling mechanism */
2831 if (hw->mac.type == fm10k_mac_pf) {
2832 /* register callback func to eal lib */
2833 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2834 fm10k_dev_interrupt_handler_pf, (void *)dev);
2836 /* enable MISC interrupt */
2837 fm10k_dev_enable_intr_pf(dev);
2839 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2840 fm10k_dev_interrupt_handler_vf, (void *)dev);
2842 fm10k_dev_enable_intr_vf(dev);
2845 /* Enable intr after callback registered */
2846 rte_intr_enable(&(dev->pci_dev->intr_handle));
2848 hw->mac.ops.update_int_moderator(hw);
2850 /* Make sure Switch Manager is ready before going forward. */
2851 if (hw->mac.type == fm10k_mac_pf) {
2852 int switch_ready = 0;
2854 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2856 hw->mac.ops.get_host_state(hw, &switch_ready);
2857 fm10k_mbx_unlock(hw);
2860 /* Delay some time to acquire async LPORT_MAP info. */
2861 rte_delay_us(WAIT_SWITCH_MSG_US);
2864 if (switch_ready == 0) {
2865 PMD_INIT_LOG(ERR, "switch is not ready");
2871 * Below function will trigger operations on mailbox, acquire lock to
2872 * avoid race condition from interrupt handler. Operations on mailbox
2873 * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2874 * will handle and generate an interrupt to our side. Then, FIFO in
2875 * mailbox will be touched.
2878 /* Enable port first */
2879 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
2882 /* Set unicast mode by default. App can change to other mode in other
2885 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2886 FM10K_XCAST_MODE_NONE);
2888 fm10k_mbx_unlock(hw);
2890 /* Make sure default VID is ready before going forward. */
2891 if (hw->mac.type == fm10k_mac_pf) {
2892 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2893 if (hw->mac.default_vid)
2895 /* Delay some time to acquire async port VLAN info. */
2896 rte_delay_us(WAIT_SWITCH_MSG_US);
2899 if (!hw->mac.default_vid) {
2900 PMD_INIT_LOG(ERR, "default VID is not ready");
2905 /* Add default mac address */
2906 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2907 MAIN_VSI_POOL_NUMBER);
2913 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2915 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2917 PMD_INIT_FUNC_TRACE();
2919 /* only uninitialize in the primary process */
2920 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2923 /* safe to close dev here */
2924 fm10k_dev_close(dev);
2926 dev->dev_ops = NULL;
2927 dev->rx_pkt_burst = NULL;
2928 dev->tx_pkt_burst = NULL;
2930 /* disable uio/vfio intr */
2931 rte_intr_disable(&(dev->pci_dev->intr_handle));
2933 /*PF/VF has different interrupt handling mechanism */
2934 if (hw->mac.type == fm10k_mac_pf) {
2935 /* disable interrupt */
2936 fm10k_dev_disable_intr_pf(dev);
2938 /* unregister callback func to eal lib */
2939 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2940 fm10k_dev_interrupt_handler_pf, (void *)dev);
2942 /* disable interrupt */
2943 fm10k_dev_disable_intr_vf(dev);
2945 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2946 fm10k_dev_interrupt_handler_vf, (void *)dev);
2949 /* free mac memory */
2950 if (dev->data->mac_addrs) {
2951 rte_free(dev->data->mac_addrs);
2952 dev->data->mac_addrs = NULL;
2955 memset(hw, 0, sizeof(*hw));
2961 * The set of PCI devices this driver supports. This driver will enable both PF
2962 * and SRIOV-VF devices.
2964 static const struct rte_pci_id pci_id_fm10k_map[] = {
2965 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2966 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2967 #include "rte_pci_dev_ids.h"
2968 { .vendor_id = 0, /* sentinel */ },
2971 static struct eth_driver rte_pmd_fm10k = {
2973 .name = "rte_pmd_fm10k",
2974 .id_table = pci_id_fm10k_map,
2975 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
2976 RTE_PCI_DRV_DETACHABLE,
2978 .eth_dev_init = eth_fm10k_dev_init,
2979 .eth_dev_uninit = eth_fm10k_dev_uninit,
2980 .dev_private_size = sizeof(struct fm10k_adapter),
2984 * Driver initialization routine.
2985 * Invoked once at EAL init time.
2986 * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2989 rte_pmd_fm10k_init(__rte_unused const char *name,
2990 __rte_unused const char *params)
2992 PMD_INIT_FUNC_TRACE();
2993 rte_eth_driver_register(&rte_pmd_fm10k);
2997 static struct rte_driver rte_fm10k_driver = {
2999 .init = rte_pmd_fm10k_init,
3002 PMD_REGISTER_DRIVER(rte_fm10k_driver);