4 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <rte_ethdev.h>
35 #include <rte_malloc.h>
36 #include <rte_memzone.h>
37 #include <rte_string_fns.h>
39 #include <rte_spinlock.h>
42 #include "base/fm10k_api.h"
44 #define FM10K_RX_BUFF_ALIGN 512
45 /* Default delay to acquire mailbox lock */
46 #define FM10K_MBXLOCK_DELAY_US 20
47 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
49 /* Number of chars per uint32 type */
50 #define CHARS_PER_UINT32 (sizeof(uint32_t))
51 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
53 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
56 fm10k_mbx_initlock(struct fm10k_hw *hw)
58 rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
62 fm10k_mbx_lock(struct fm10k_hw *hw)
64 while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
65 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
69 fm10k_mbx_unlock(struct fm10k_hw *hw)
71 rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
75 * reset queue to initial state, allocate software buffers used when starting
78 * return -ENOMEM if buffers cannot be allocated
79 * return -EINVAL if buffers do not satisfy alignment condition
82 rx_queue_reset(struct fm10k_rx_queue *q)
86 PMD_INIT_FUNC_TRACE();
88 diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
92 for (i = 0; i < q->nb_desc; ++i) {
93 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
94 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
95 rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
99 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
100 q->hw_ring[i].q.pkt_addr = dma_addr;
101 q->hw_ring[i].q.hdr_addr = dma_addr;
106 q->next_trigger = q->alloc_thresh - 1;
107 FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
112 * clean queue, descriptor rings, free software buffers used when stopping
116 rx_queue_clean(struct fm10k_rx_queue *q)
118 union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
120 PMD_INIT_FUNC_TRACE();
122 /* zero descriptor rings */
123 for (i = 0; i < q->nb_desc; ++i)
124 q->hw_ring[i] = zero;
126 /* free software buffers */
127 for (i = 0; i < q->nb_desc; ++i) {
129 rte_pktmbuf_free_seg(q->sw_ring[i]);
130 q->sw_ring[i] = NULL;
136 * free all queue memory used when releasing the queue (i.e. configure)
139 rx_queue_free(struct fm10k_rx_queue *q)
141 PMD_INIT_FUNC_TRACE();
143 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
146 rte_free(q->sw_ring);
155 * disable RX queue, wait unitl HW finished necessary flush operation
158 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
162 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
163 FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
164 reg & ~FM10K_RXQCTL_ENABLE);
166 /* Wait 100us at most */
167 for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
169 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(i));
170 if (!(reg & FM10K_RXQCTL_ENABLE))
174 if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
181 * reset queue to initial state, allocate software buffers used when starting
185 tx_queue_reset(struct fm10k_tx_queue *q)
187 PMD_INIT_FUNC_TRACE();
191 q->nb_free = q->nb_desc - 1;
192 q->free_trigger = q->nb_free - q->free_thresh;
193 fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
194 FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
198 * clean queue, descriptor rings, free software buffers used when stopping
202 tx_queue_clean(struct fm10k_tx_queue *q)
204 struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
206 PMD_INIT_FUNC_TRACE();
208 /* zero descriptor rings */
209 for (i = 0; i < q->nb_desc; ++i)
210 q->hw_ring[i] = zero;
212 /* free software buffers */
213 for (i = 0; i < q->nb_desc; ++i) {
215 rte_pktmbuf_free_seg(q->sw_ring[i]);
216 q->sw_ring[i] = NULL;
222 * free all queue memory used when releasing the queue (i.e. configure)
225 tx_queue_free(struct fm10k_tx_queue *q)
227 PMD_INIT_FUNC_TRACE();
229 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
231 if (q->rs_tracker.list) {
232 rte_free(q->rs_tracker.list);
233 q->rs_tracker.list = NULL;
236 rte_free(q->sw_ring);
245 * disable TX queue, wait unitl HW finished necessary flush operation
248 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
252 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
253 FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
254 reg & ~FM10K_TXDCTL_ENABLE);
256 /* Wait 100us at most */
257 for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
259 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(i));
260 if (!(reg & FM10K_TXDCTL_ENABLE))
264 if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
271 fm10k_dev_configure(struct rte_eth_dev *dev)
273 PMD_INIT_FUNC_TRACE();
275 if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
276 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
282 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
284 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
285 struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
286 uint32_t mrqc, *key, i, reta, j;
289 #define RSS_KEY_SIZE 40
290 static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
291 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
292 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
293 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
294 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
295 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
298 if (dev->data->nb_rx_queues == 1 ||
299 dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
300 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
303 /* random key is rss_intel_key (default) or user provided (rss_key) */
304 if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
305 key = (uint32_t *)rss_intel_key;
307 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
309 /* Now fill our hash function seeds, 4 bytes at a time */
310 for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
311 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
314 * Fill in redirection table
315 * The byte-swap is needed because NIC registers are in
316 * little-endian order.
319 for (i = 0, j = 0; i < FM10K_RETA_SIZE; i++, j++) {
320 if (j == dev->data->nb_rx_queues)
322 reta = (reta << CHAR_BIT) | j;
324 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
329 * Generate RSS hash based on packet types, TCP/UDP
330 * port numbers and/or IPv4/v6 src and dst addresses
332 hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
334 mrqc |= (hf & ETH_RSS_IPV4) ? FM10K_MRQC_IPV4 : 0;
335 mrqc |= (hf & ETH_RSS_IPV6) ? FM10K_MRQC_IPV6 : 0;
336 mrqc |= (hf & ETH_RSS_IPV6_EX) ? FM10K_MRQC_IPV6 : 0;
337 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP) ? FM10K_MRQC_TCP_IPV4 : 0;
338 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP) ? FM10K_MRQC_TCP_IPV6 : 0;
339 mrqc |= (hf & ETH_RSS_IPV6_TCP_EX) ? FM10K_MRQC_TCP_IPV6 : 0;
340 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP) ? FM10K_MRQC_UDP_IPV4 : 0;
341 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP) ? FM10K_MRQC_UDP_IPV6 : 0;
342 mrqc |= (hf & ETH_RSS_IPV6_UDP_EX) ? FM10K_MRQC_UDP_IPV6 : 0;
345 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
350 FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
354 fm10k_dev_tx_init(struct rte_eth_dev *dev)
356 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
358 struct fm10k_tx_queue *txq;
362 /* Disable TXINT to avoid possible interrupt */
363 for (i = 0; i < hw->mac.max_queues; i++)
364 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
365 3 << FM10K_TXINT_TIMER_SHIFT);
368 for (i = 0; i < dev->data->nb_tx_queues; ++i) {
369 txq = dev->data->tx_queues[i];
370 base_addr = txq->hw_ring_phys_addr;
371 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
373 /* disable queue to avoid issues while updating state */
374 ret = tx_queue_disable(hw, i);
376 PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
380 /* set location and size for descriptor ring */
381 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
382 base_addr & UINT64_LOWER_32BITS_MASK);
383 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
384 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
385 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
391 fm10k_dev_rx_init(struct rte_eth_dev *dev)
393 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
395 struct fm10k_rx_queue *rxq;
398 uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
400 struct rte_pktmbuf_pool_private *mbp_priv;
402 /* Disable RXINT to avoid possible interrupt */
403 for (i = 0; i < hw->mac.max_queues; i++)
404 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
405 3 << FM10K_RXINT_TIMER_SHIFT);
407 /* Setup RX queues */
408 for (i = 0; i < dev->data->nb_rx_queues; ++i) {
409 rxq = dev->data->rx_queues[i];
410 base_addr = rxq->hw_ring_phys_addr;
411 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
413 /* disable queue to avoid issues while updating state */
414 ret = rx_queue_disable(hw, i);
416 PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
420 /* Setup the Base and Length of the Rx Descriptor Ring */
421 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
422 base_addr & UINT64_LOWER_32BITS_MASK);
423 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
424 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
425 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
427 /* Configure the Rx buffer size for one buff without split */
428 mbp_priv = rte_mempool_get_priv(rxq->mp);
429 buf_size = (uint16_t) (mbp_priv->mbuf_data_room_size -
430 RTE_PKTMBUF_HEADROOM);
431 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
432 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
434 /* It adds dual VLAN length for supporting dual VLAN */
435 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
436 2 * FM10K_VLAN_TAG_SIZE) > buf_size){
437 dev->data->scattered_rx = 1;
438 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
441 /* Enable drop on empty, it's RO for VF */
442 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
443 rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
445 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
446 FM10K_WRITE_FLUSH(hw);
449 if (dev->data->dev_conf.rxmode.enable_scatter) {
450 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
451 dev->data->scattered_rx = 1;
454 /* Configure RSS if applicable */
455 fm10k_dev_mq_rx_configure(dev);
460 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
462 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
465 struct fm10k_rx_queue *rxq;
467 PMD_INIT_FUNC_TRACE();
469 if (rx_queue_id < dev->data->nb_rx_queues) {
470 rxq = dev->data->rx_queues[rx_queue_id];
471 err = rx_queue_reset(rxq);
472 if (err == -ENOMEM) {
473 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
475 } else if (err == -EINVAL) {
476 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
481 /* Setup the HW Rx Head and Tail Descriptor Pointers
482 * Note: this must be done AFTER the queue is enabled on real
483 * hardware, but BEFORE the queue is enabled when using the
484 * emulation platform. Do it in both places for now and remove
485 * this comment and the following two register writes when the
486 * emulation platform is no longer being used.
488 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
489 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
491 /* Set PF ownership flag for PF devices */
492 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
493 if (hw->mac.type == fm10k_mac_pf)
494 reg |= FM10K_RXQCTL_PF;
495 reg |= FM10K_RXQCTL_ENABLE;
496 /* enable RX queue */
497 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
498 FM10K_WRITE_FLUSH(hw);
500 /* Setup the HW Rx Head and Tail Descriptor Pointers
501 * Note: this must be done AFTER the queue is enabled
503 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
504 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
511 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
513 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
515 PMD_INIT_FUNC_TRACE();
517 if (rx_queue_id < dev->data->nb_rx_queues) {
518 /* Disable RX queue */
519 rx_queue_disable(hw, rx_queue_id);
521 /* Free mbuf and clean HW ring */
522 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
529 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
531 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
532 /** @todo - this should be defined in the shared code */
533 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY 0x00010000
534 uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
537 PMD_INIT_FUNC_TRACE();
539 if (tx_queue_id < dev->data->nb_tx_queues) {
540 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
542 /* reset head and tail pointers */
543 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
544 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
546 /* enable TX queue */
547 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
548 FM10K_TXDCTL_ENABLE | txdctl);
549 FM10K_WRITE_FLUSH(hw);
557 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
559 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
561 PMD_INIT_FUNC_TRACE();
563 if (tx_queue_id < dev->data->nb_tx_queues) {
564 tx_queue_disable(hw, tx_queue_id);
565 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
571 /* fls = find last set bit = 32 minus the number of leading zeros */
573 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
575 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
577 fm10k_dev_start(struct rte_eth_dev *dev)
579 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
582 PMD_INIT_FUNC_TRACE();
584 /* stop, init, then start the hw */
585 diag = fm10k_stop_hw(hw);
586 if (diag != FM10K_SUCCESS) {
587 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
591 diag = fm10k_init_hw(hw);
592 if (diag != FM10K_SUCCESS) {
593 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
597 diag = fm10k_start_hw(hw);
598 if (diag != FM10K_SUCCESS) {
599 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
603 diag = fm10k_dev_tx_init(dev);
605 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
609 diag = fm10k_dev_rx_init(dev);
611 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
615 if (hw->mac.type == fm10k_mac_pf) {
616 /* Establish only VSI 0 as valid */
617 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
619 /* Configure RSS bits used in RETA table */
620 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0),
621 fls(dev->data->nb_rx_queues - 1) <<
622 FM10K_DGLORTDEC_RSSLENGTH_SHIFT);
624 /* Invalidate all other GLORT entries */
625 for (i = 1; i < FM10K_DGLORT_COUNT; i++)
626 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
627 FM10K_DGLORTMAP_NONE);
630 for (i = 0; i < dev->data->nb_rx_queues; i++) {
631 struct fm10k_rx_queue *rxq;
632 rxq = dev->data->rx_queues[i];
634 if (rxq->rx_deferred_start)
636 diag = fm10k_dev_rx_queue_start(dev, i);
639 for (j = 0; j < i; ++j)
640 rx_queue_clean(dev->data->rx_queues[j]);
645 for (i = 0; i < dev->data->nb_tx_queues; i++) {
646 struct fm10k_tx_queue *txq;
647 txq = dev->data->tx_queues[i];
649 if (txq->tx_deferred_start)
651 diag = fm10k_dev_tx_queue_start(dev, i);
654 for (j = 0; j < dev->data->nb_rx_queues; ++j)
655 rx_queue_clean(dev->data->rx_queues[j]);
664 fm10k_dev_stop(struct rte_eth_dev *dev)
668 PMD_INIT_FUNC_TRACE();
670 for (i = 0; i < dev->data->nb_tx_queues; i++)
671 fm10k_dev_tx_queue_stop(dev, i);
673 for (i = 0; i < dev->data->nb_rx_queues; i++)
674 fm10k_dev_rx_queue_stop(dev, i);
678 fm10k_dev_close(struct rte_eth_dev *dev)
680 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
682 PMD_INIT_FUNC_TRACE();
684 /* Stop mailbox service first */
685 fm10k_close_mbx_service(hw);
691 fm10k_link_update(struct rte_eth_dev *dev,
692 __rte_unused int wait_to_complete)
694 PMD_INIT_FUNC_TRACE();
696 /* The host-interface link is always up. The speed is ~50Gbps per Gen3
697 * x8 PCIe interface. For now, we leave the speed undefined since there
698 * is no 50Gbps Ethernet. */
699 dev->data->dev_link.link_speed = 0;
700 dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
701 dev->data->dev_link.link_status = 1;
707 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
709 uint64_t ipackets, opackets, ibytes, obytes;
710 struct fm10k_hw *hw =
711 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
712 struct fm10k_hw_stats *hw_stats =
713 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
716 PMD_INIT_FUNC_TRACE();
718 fm10k_update_hw_stats(hw, hw_stats);
720 ipackets = opackets = ibytes = obytes = 0;
721 for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
722 (i < FM10K_MAX_QUEUES_PF); ++i) {
723 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
724 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
725 stats->q_ibytes[i] = hw_stats->q[i].rx_bytes.count;
726 stats->q_obytes[i] = hw_stats->q[i].tx_bytes.count;
727 ipackets += stats->q_ipackets[i];
728 opackets += stats->q_opackets[i];
729 ibytes += stats->q_ibytes[i];
730 obytes += stats->q_obytes[i];
732 stats->ipackets = ipackets;
733 stats->opackets = opackets;
734 stats->ibytes = ibytes;
735 stats->obytes = obytes;
739 fm10k_stats_reset(struct rte_eth_dev *dev)
741 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
742 struct fm10k_hw_stats *hw_stats =
743 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
745 PMD_INIT_FUNC_TRACE();
747 memset(hw_stats, 0, sizeof(*hw_stats));
748 fm10k_rebind_hw_stats(hw, hw_stats);
752 fm10k_dev_infos_get(struct rte_eth_dev *dev,
753 struct rte_eth_dev_info *dev_info)
755 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
757 PMD_INIT_FUNC_TRACE();
759 dev_info->min_rx_bufsize = FM10K_MIN_RX_BUF_SIZE;
760 dev_info->max_rx_pktlen = FM10K_MAX_PKT_SIZE;
761 dev_info->max_rx_queues = hw->mac.max_queues;
762 dev_info->max_tx_queues = hw->mac.max_queues;
763 dev_info->max_mac_addrs = 1;
764 dev_info->max_hash_mac_addrs = 0;
765 dev_info->max_vfs = FM10K_MAX_VF_NUM;
766 dev_info->max_vmdq_pools = ETH_64_POOLS;
767 dev_info->rx_offload_capa =
768 DEV_RX_OFFLOAD_IPV4_CKSUM |
769 DEV_RX_OFFLOAD_UDP_CKSUM |
770 DEV_RX_OFFLOAD_TCP_CKSUM;
771 dev_info->tx_offload_capa = 0;
772 dev_info->reta_size = FM10K_MAX_RSS_INDICES;
774 dev_info->default_rxconf = (struct rte_eth_rxconf) {
776 .pthresh = FM10K_DEFAULT_RX_PTHRESH,
777 .hthresh = FM10K_DEFAULT_RX_HTHRESH,
778 .wthresh = FM10K_DEFAULT_RX_WTHRESH,
780 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
784 dev_info->default_txconf = (struct rte_eth_txconf) {
786 .pthresh = FM10K_DEFAULT_TX_PTHRESH,
787 .hthresh = FM10K_DEFAULT_TX_HTHRESH,
788 .wthresh = FM10K_DEFAULT_TX_WTHRESH,
790 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
791 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
792 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
793 ETH_TXQ_FLAGS_NOOFFLOADS,
799 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
801 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
803 PMD_INIT_FUNC_TRACE();
805 /* @todo - add support for the VF */
806 if (hw->mac.type != fm10k_mac_pf)
809 return fm10k_update_vlan(hw, vlan_id, 0, on);
813 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
815 if ((request < min) || (request > max) || ((request % mult) != 0))
822 * Create a memzone for hardware descriptor rings. Malloc cannot be used since
823 * the physical address is required. If the memzone is already created, then
824 * this function returns a pointer to the existing memzone.
826 static inline const struct rte_memzone *
827 allocate_hw_ring(const char *driver_name, const char *ring_name,
828 uint8_t port_id, uint16_t queue_id, int socket_id,
829 uint32_t size, uint32_t align)
831 char name[RTE_MEMZONE_NAMESIZE];
832 const struct rte_memzone *mz;
834 snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
835 driver_name, ring_name, port_id, queue_id, socket_id);
837 /* return the memzone if it already exists */
838 mz = rte_memzone_lookup(name);
842 #ifdef RTE_LIBRTE_XEN_DOM0
843 return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
846 return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
851 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
853 if ((request < min) || (request > max) || ((div % request) != 0))
860 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
862 uint16_t rx_free_thresh;
864 if (conf->rx_free_thresh == 0)
865 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
867 rx_free_thresh = conf->rx_free_thresh;
869 /* make sure the requested threshold satisfies the constraints */
870 if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
871 FM10K_RX_FREE_THRESH_MAX(q),
872 FM10K_RX_FREE_THRESH_DIV(q),
874 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
875 "less than or equal to %u, "
876 "greater than or equal to %u, "
877 "and a divisor of %u",
878 rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
879 FM10K_RX_FREE_THRESH_MIN(q),
880 FM10K_RX_FREE_THRESH_DIV(q));
884 q->alloc_thresh = rx_free_thresh;
885 q->drop_en = conf->rx_drop_en;
886 q->rx_deferred_start = conf->rx_deferred_start;
892 * Hardware requires specific alignment for Rx packet buffers. At
893 * least one of the following two conditions must be satisfied.
894 * 1. Address is 512B aligned
895 * 2. Address is 8B aligned and buffer does not cross 4K boundary.
897 * As such, the driver may need to adjust the DMA address within the
898 * buffer by up to 512B. The mempool element size is checked here
899 * to make sure a maximally sized Ethernet frame can still be wholly
900 * contained within the buffer after 512B alignment.
902 * return 1 if the element size is valid, otherwise return 0.
905 mempool_element_size_valid(struct rte_mempool *mp)
909 /* elt_size includes mbuf header and headroom */
910 min_size = mp->elt_size - sizeof(struct rte_mbuf) -
911 RTE_PKTMBUF_HEADROOM;
913 /* account for up to 512B of alignment */
914 min_size -= FM10K_RX_BUFF_ALIGN;
916 /* sanity check for overflow */
917 if (min_size > mp->elt_size)
920 if (min_size < ETHER_MAX_VLAN_FRAME_LEN)
928 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
929 uint16_t nb_desc, unsigned int socket_id,
930 const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
932 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
933 struct fm10k_rx_queue *q;
934 const struct rte_memzone *mz;
936 PMD_INIT_FUNC_TRACE();
938 /* make sure the mempool element size can account for alignment. */
939 if (!mempool_element_size_valid(mp)) {
940 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
944 /* make sure a valid number of descriptors have been requested */
945 if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
946 FM10K_MULT_RX_DESC, nb_desc)) {
947 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
948 "less than or equal to %"PRIu32", "
949 "greater than or equal to %u, "
950 "and a multiple of %u",
951 nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
957 * if this queue existed already, free the associated memory. The
958 * queue cannot be reused in case we need to allocate memory on
959 * different socket than was previously used.
961 if (dev->data->rx_queues[queue_id] != NULL) {
962 rx_queue_free(dev->data->rx_queues[queue_id]);
963 dev->data->rx_queues[queue_id] = NULL;
966 /* allocate memory for the queue structure */
967 q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
970 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
976 q->nb_desc = nb_desc;
977 q->port_id = dev->data->port_id;
978 q->queue_id = queue_id;
979 q->tail_ptr = (volatile uint32_t *)
980 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
981 if (handle_rxconf(q, conf))
984 /* allocate memory for the software ring */
985 q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
986 nb_desc * sizeof(struct rte_mbuf *),
987 RTE_CACHE_LINE_SIZE, socket_id);
988 if (q->sw_ring == NULL) {
989 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
995 * allocate memory for the hardware descriptor ring. A memzone large
996 * enough to hold the maximum ring size is requested to allow for
997 * resizing in later calls to the queue setup function.
999 mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1000 dev->data->port_id, queue_id, socket_id,
1001 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1003 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1004 rte_free(q->sw_ring);
1008 q->hw_ring = mz->addr;
1009 q->hw_ring_phys_addr = mz->phys_addr;
1011 dev->data->rx_queues[queue_id] = q;
1016 fm10k_rx_queue_release(void *queue)
1018 PMD_INIT_FUNC_TRACE();
1020 rx_queue_free(queue);
1024 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1026 uint16_t tx_free_thresh;
1027 uint16_t tx_rs_thresh;
1029 /* constraint MACROs require that tx_free_thresh is configured
1030 * before tx_rs_thresh */
1031 if (conf->tx_free_thresh == 0)
1032 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1034 tx_free_thresh = conf->tx_free_thresh;
1036 /* make sure the requested threshold satisfies the constraints */
1037 if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1038 FM10K_TX_FREE_THRESH_MAX(q),
1039 FM10K_TX_FREE_THRESH_DIV(q),
1041 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1042 "less than or equal to %u, "
1043 "greater than or equal to %u, "
1044 "and a divisor of %u",
1045 tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1046 FM10K_TX_FREE_THRESH_MIN(q),
1047 FM10K_TX_FREE_THRESH_DIV(q));
1051 q->free_thresh = tx_free_thresh;
1053 if (conf->tx_rs_thresh == 0)
1054 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1056 tx_rs_thresh = conf->tx_rs_thresh;
1058 q->tx_deferred_start = conf->tx_deferred_start;
1060 /* make sure the requested threshold satisfies the constraints */
1061 if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1062 FM10K_TX_RS_THRESH_MAX(q),
1063 FM10K_TX_RS_THRESH_DIV(q),
1065 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1066 "less than or equal to %u, "
1067 "greater than or equal to %u, "
1068 "and a divisor of %u",
1069 tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1070 FM10K_TX_RS_THRESH_MIN(q),
1071 FM10K_TX_RS_THRESH_DIV(q));
1075 q->rs_thresh = tx_rs_thresh;
1081 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1082 uint16_t nb_desc, unsigned int socket_id,
1083 const struct rte_eth_txconf *conf)
1085 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1086 struct fm10k_tx_queue *q;
1087 const struct rte_memzone *mz;
1089 PMD_INIT_FUNC_TRACE();
1091 /* make sure a valid number of descriptors have been requested */
1092 if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1093 FM10K_MULT_TX_DESC, nb_desc)) {
1094 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1095 "less than or equal to %"PRIu32", "
1096 "greater than or equal to %u, "
1097 "and a multiple of %u",
1098 nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1099 FM10K_MULT_TX_DESC);
1104 * if this queue existed already, free the associated memory. The
1105 * queue cannot be reused in case we need to allocate memory on
1106 * different socket than was previously used.
1108 if (dev->data->tx_queues[queue_id] != NULL) {
1109 tx_queue_free(dev->data->tx_queues[queue_id]);
1110 dev->data->tx_queues[queue_id] = NULL;
1113 /* allocate memory for the queue structure */
1114 q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1117 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1122 q->nb_desc = nb_desc;
1123 q->port_id = dev->data->port_id;
1124 q->queue_id = queue_id;
1125 q->tail_ptr = (volatile uint32_t *)
1126 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1127 if (handle_txconf(q, conf))
1130 /* allocate memory for the software ring */
1131 q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1132 nb_desc * sizeof(struct rte_mbuf *),
1133 RTE_CACHE_LINE_SIZE, socket_id);
1134 if (q->sw_ring == NULL) {
1135 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1141 * allocate memory for the hardware descriptor ring. A memzone large
1142 * enough to hold the maximum ring size is requested to allow for
1143 * resizing in later calls to the queue setup function.
1145 mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1146 dev->data->port_id, queue_id, socket_id,
1147 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1149 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1150 rte_free(q->sw_ring);
1154 q->hw_ring = mz->addr;
1155 q->hw_ring_phys_addr = mz->phys_addr;
1158 * allocate memory for the RS bit tracker. Enough slots to hold the
1159 * descriptor index for each RS bit needing to be set are required.
1161 q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1162 ((nb_desc + 1) / q->rs_thresh) *
1164 RTE_CACHE_LINE_SIZE, socket_id);
1165 if (q->rs_tracker.list == NULL) {
1166 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1167 rte_free(q->sw_ring);
1172 dev->data->tx_queues[queue_id] = q;
1177 fm10k_tx_queue_release(void *queue)
1179 PMD_INIT_FUNC_TRACE();
1181 tx_queue_free(queue);
1185 fm10k_reta_update(struct rte_eth_dev *dev,
1186 struct rte_eth_rss_reta_entry64 *reta_conf,
1189 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1190 uint16_t i, j, idx, shift;
1194 PMD_INIT_FUNC_TRACE();
1196 if (reta_size > FM10K_MAX_RSS_INDICES) {
1197 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1198 "(%d) doesn't match the number hardware can supported "
1199 "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1204 * Update Redirection Table RETA[n], n=0..31. The redirection table has
1205 * 128-entries in 32 registers
1207 for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1208 idx = i / RTE_RETA_GROUP_SIZE;
1209 shift = i % RTE_RETA_GROUP_SIZE;
1210 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1211 BIT_MASK_PER_UINT32);
1216 if (mask != BIT_MASK_PER_UINT32)
1217 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1219 for (j = 0; j < CHARS_PER_UINT32; j++) {
1220 if (mask & (0x1 << j)) {
1222 reta &= ~(UINT8_MAX << CHAR_BIT * j);
1223 reta |= reta_conf[idx].reta[shift + j] <<
1227 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1234 fm10k_reta_query(struct rte_eth_dev *dev,
1235 struct rte_eth_rss_reta_entry64 *reta_conf,
1238 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1239 uint16_t i, j, idx, shift;
1243 PMD_INIT_FUNC_TRACE();
1245 if (reta_size < FM10K_MAX_RSS_INDICES) {
1246 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1247 "(%d) doesn't match the number hardware can supported "
1248 "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1253 * Read Redirection Table RETA[n], n=0..31. The redirection table has
1254 * 128-entries in 32 registers
1256 for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1257 idx = i / RTE_RETA_GROUP_SIZE;
1258 shift = i % RTE_RETA_GROUP_SIZE;
1259 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1260 BIT_MASK_PER_UINT32);
1264 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1265 for (j = 0; j < CHARS_PER_UINT32; j++) {
1266 if (mask & (0x1 << j))
1267 reta_conf[idx].reta[shift + j] = ((reta >>
1268 CHAR_BIT * j) & UINT8_MAX);
1276 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1277 struct rte_eth_rss_conf *rss_conf)
1279 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1280 uint32_t *key = (uint32_t *)rss_conf->rss_key;
1282 uint64_t hf = rss_conf->rss_hf;
1285 PMD_INIT_FUNC_TRACE();
1287 if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1288 FM10K_RSSRK_ENTRIES_PER_REG)
1295 mrqc |= (hf & ETH_RSS_IPV4) ? FM10K_MRQC_IPV4 : 0;
1296 mrqc |= (hf & ETH_RSS_IPV6) ? FM10K_MRQC_IPV6 : 0;
1297 mrqc |= (hf & ETH_RSS_IPV6_EX) ? FM10K_MRQC_IPV6 : 0;
1298 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP) ? FM10K_MRQC_TCP_IPV4 : 0;
1299 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP) ? FM10K_MRQC_TCP_IPV6 : 0;
1300 mrqc |= (hf & ETH_RSS_IPV6_TCP_EX) ? FM10K_MRQC_TCP_IPV6 : 0;
1301 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP) ? FM10K_MRQC_UDP_IPV4 : 0;
1302 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP) ? FM10K_MRQC_UDP_IPV6 : 0;
1303 mrqc |= (hf & ETH_RSS_IPV6_UDP_EX) ? FM10K_MRQC_UDP_IPV6 : 0;
1305 /* If the mapping doesn't fit any supported, return */
1310 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1311 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1313 FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1319 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1320 struct rte_eth_rss_conf *rss_conf)
1322 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1323 uint32_t *key = (uint32_t *)rss_conf->rss_key;
1328 PMD_INIT_FUNC_TRACE();
1330 if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1331 FM10K_RSSRK_ENTRIES_PER_REG)
1335 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1336 key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1338 mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1340 hf |= (mrqc & FM10K_MRQC_IPV4) ? ETH_RSS_IPV4 : 0;
1341 hf |= (mrqc & FM10K_MRQC_IPV6) ? ETH_RSS_IPV6 : 0;
1342 hf |= (mrqc & FM10K_MRQC_IPV6) ? ETH_RSS_IPV6_EX : 0;
1343 hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP : 0;
1344 hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP : 0;
1345 hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX : 0;
1346 hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP : 0;
1347 hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP : 0;
1348 hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX : 0;
1350 rss_conf->rss_hf = hf;
1356 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1358 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1359 uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1361 /* Bind all local non-queue interrupt to vector 0 */
1364 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1365 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1366 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1367 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1368 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1369 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1371 /* Enable misc causes */
1372 FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1373 FM10K_EIMR_ENABLE(THI_FAULT) |
1374 FM10K_EIMR_ENABLE(FUM_FAULT) |
1375 FM10K_EIMR_ENABLE(MAILBOX) |
1376 FM10K_EIMR_ENABLE(SWITCHREADY) |
1377 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1378 FM10K_EIMR_ENABLE(SRAMERROR) |
1379 FM10K_EIMR_ENABLE(VFLR));
1382 FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1383 FM10K_ITR_MASK_CLEAR);
1384 FM10K_WRITE_FLUSH(hw);
1388 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1390 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1391 uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1393 /* Bind all local non-queue interrupt to vector 0 */
1396 /* Only INT 0 available, other 15 are reserved. */
1397 FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1400 FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1401 FM10K_ITR_MASK_CLEAR);
1402 FM10K_WRITE_FLUSH(hw);
1406 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1408 struct fm10k_fault fault;
1410 const char *estr = "Unknown error";
1412 /* Process PCA fault */
1413 if (eicr & FM10K_EIMR_PCA_FAULT) {
1414 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
1417 switch (fault.type) {
1419 estr = "PCA_NO_FAULT"; break;
1420 case PCA_UNMAPPED_ADDR:
1421 estr = "PCA_UNMAPPED_ADDR"; break;
1422 case PCA_BAD_QACCESS_PF:
1423 estr = "PCA_BAD_QACCESS_PF"; break;
1424 case PCA_BAD_QACCESS_VF:
1425 estr = "PCA_BAD_QACCESS_VF"; break;
1426 case PCA_MALICIOUS_REQ:
1427 estr = "PCA_MALICIOUS_REQ"; break;
1428 case PCA_POISONED_TLP:
1429 estr = "PCA_POISONED_TLP"; break;
1431 estr = "PCA_TLP_ABORT"; break;
1435 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1436 estr, fault.func ? "VF" : "PF", fault.func,
1437 fault.address, fault.specinfo);
1440 /* Process THI fault */
1441 if (eicr & FM10K_EIMR_THI_FAULT) {
1442 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
1445 switch (fault.type) {
1447 estr = "THI_NO_FAULT"; break;
1448 case THI_MAL_DIS_Q_FAULT:
1449 estr = "THI_MAL_DIS_Q_FAULT"; break;
1453 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1454 estr, fault.func ? "VF" : "PF", fault.func,
1455 fault.address, fault.specinfo);
1458 /* Process FUM fault */
1459 if (eicr & FM10K_EIMR_FUM_FAULT) {
1460 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
1463 switch (fault.type) {
1465 estr = "FUM_NO_FAULT"; break;
1466 case FUM_UNMAPPED_ADDR:
1467 estr = "FUM_UNMAPPED_ADDR"; break;
1468 case FUM_POISONED_TLP:
1469 estr = "FUM_POISONED_TLP"; break;
1470 case FUM_BAD_VF_QACCESS:
1471 estr = "FUM_BAD_VF_QACCESS"; break;
1472 case FUM_ADD_DECODE_ERR:
1473 estr = "FUM_ADD_DECODE_ERR"; break;
1475 estr = "FUM_RO_ERROR"; break;
1476 case FUM_QPRC_CRC_ERROR:
1477 estr = "FUM_QPRC_CRC_ERROR"; break;
1478 case FUM_CSR_TIMEOUT:
1479 estr = "FUM_CSR_TIMEOUT"; break;
1480 case FUM_INVALID_TYPE:
1481 estr = "FUM_INVALID_TYPE"; break;
1482 case FUM_INVALID_LENGTH:
1483 estr = "FUM_INVALID_LENGTH"; break;
1484 case FUM_INVALID_BE:
1485 estr = "FUM_INVALID_BE"; break;
1486 case FUM_INVALID_ALIGN:
1487 estr = "FUM_INVALID_ALIGN"; break;
1491 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1492 estr, fault.func ? "VF" : "PF", fault.func,
1493 fault.address, fault.specinfo);
1500 PMD_INIT_LOG(ERR, "Failed to handle fault event.");
1505 * PF interrupt handler triggered by NIC for handling specific interrupt.
1508 * Pointer to interrupt handle.
1510 * The address of parameter (struct rte_eth_dev *) regsitered before.
1516 fm10k_dev_interrupt_handler_pf(
1517 __rte_unused struct rte_intr_handle *handle,
1520 struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1521 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1522 uint32_t cause, status;
1524 if (hw->mac.type != fm10k_mac_pf)
1527 cause = FM10K_READ_REG(hw, FM10K_EICR);
1529 /* Handle PCI fault cases */
1530 if (cause & FM10K_EICR_FAULT_MASK) {
1531 PMD_INIT_LOG(ERR, "INT: find fault!");
1532 fm10k_dev_handle_fault(hw, cause);
1535 /* Handle switch up/down */
1536 if (cause & FM10K_EICR_SWITCHNOTREADY)
1537 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
1539 if (cause & FM10K_EICR_SWITCHREADY)
1540 PMD_INIT_LOG(INFO, "INT: Switch is ready");
1542 /* Handle mailbox message */
1544 hw->mbx.ops.process(hw, &hw->mbx);
1545 fm10k_mbx_unlock(hw);
1547 /* Handle SRAM error */
1548 if (cause & FM10K_EICR_SRAMERROR) {
1549 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
1551 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
1552 /* Write to clear pending bits */
1553 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
1555 /* Todo: print out error message after shared code updates */
1558 /* Clear these 3 events if having any */
1559 cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
1560 FM10K_EICR_SWITCHREADY;
1562 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
1564 /* Re-enable interrupt from device side */
1565 FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1566 FM10K_ITR_MASK_CLEAR);
1567 /* Re-enable interrupt from host side */
1568 rte_intr_enable(&(dev->pci_dev->intr_handle));
1572 * VF interrupt handler triggered by NIC for handling specific interrupt.
1575 * Pointer to interrupt handle.
1577 * The address of parameter (struct rte_eth_dev *) regsitered before.
1583 fm10k_dev_interrupt_handler_vf(
1584 __rte_unused struct rte_intr_handle *handle,
1587 struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1588 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1590 if (hw->mac.type != fm10k_mac_vf)
1593 /* Handle mailbox message if lock is acquired */
1595 hw->mbx.ops.process(hw, &hw->mbx);
1596 fm10k_mbx_unlock(hw);
1598 /* Re-enable interrupt from device side */
1599 FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1600 FM10K_ITR_MASK_CLEAR);
1601 /* Re-enable interrupt from host side */
1602 rte_intr_enable(&(dev->pci_dev->intr_handle));
1605 /* Mailbox message handler in VF */
1606 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
1607 FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
1608 FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
1609 FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
1610 FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1613 /* Mailbox message handler in PF */
1614 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
1615 FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
1616 FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
1617 FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
1618 FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
1619 FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
1620 FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
1621 FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
1625 fm10k_setup_mbx_service(struct fm10k_hw *hw)
1629 /* Initialize mailbox lock */
1630 fm10k_mbx_initlock(hw);
1632 /* Replace default message handler with new ones */
1633 if (hw->mac.type == fm10k_mac_pf)
1634 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
1636 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
1639 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
1643 /* Connect to SM for PF device or PF for VF device */
1644 return hw->mbx.ops.connect(hw, &hw->mbx);
1648 fm10k_close_mbx_service(struct fm10k_hw *hw)
1650 /* Disconnect from SM for PF device or PF for VF device */
1651 hw->mbx.ops.disconnect(hw, &hw->mbx);
1654 static const struct eth_dev_ops fm10k_eth_dev_ops = {
1655 .dev_configure = fm10k_dev_configure,
1656 .dev_start = fm10k_dev_start,
1657 .dev_stop = fm10k_dev_stop,
1658 .dev_close = fm10k_dev_close,
1659 .stats_get = fm10k_stats_get,
1660 .stats_reset = fm10k_stats_reset,
1661 .link_update = fm10k_link_update,
1662 .dev_infos_get = fm10k_dev_infos_get,
1663 .vlan_filter_set = fm10k_vlan_filter_set,
1664 .rx_queue_start = fm10k_dev_rx_queue_start,
1665 .rx_queue_stop = fm10k_dev_rx_queue_stop,
1666 .tx_queue_start = fm10k_dev_tx_queue_start,
1667 .tx_queue_stop = fm10k_dev_tx_queue_stop,
1668 .rx_queue_setup = fm10k_rx_queue_setup,
1669 .rx_queue_release = fm10k_rx_queue_release,
1670 .tx_queue_setup = fm10k_tx_queue_setup,
1671 .tx_queue_release = fm10k_tx_queue_release,
1672 .reta_update = fm10k_reta_update,
1673 .reta_query = fm10k_reta_query,
1674 .rss_hash_update = fm10k_rss_hash_update,
1675 .rss_hash_conf_get = fm10k_rss_hash_conf_get,
1679 eth_fm10k_dev_init(struct rte_eth_dev *dev)
1681 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1684 PMD_INIT_FUNC_TRACE();
1686 dev->dev_ops = &fm10k_eth_dev_ops;
1687 dev->rx_pkt_burst = &fm10k_recv_pkts;
1688 dev->tx_pkt_burst = &fm10k_xmit_pkts;
1690 if (dev->data->scattered_rx)
1691 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
1693 /* only initialize in the primary process */
1694 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1697 /* Vendor and Device ID need to be set before init of shared code */
1698 memset(hw, 0, sizeof(*hw));
1699 hw->device_id = dev->pci_dev->id.device_id;
1700 hw->vendor_id = dev->pci_dev->id.vendor_id;
1701 hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
1702 hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
1703 hw->revision_id = 0;
1704 hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
1705 if (hw->hw_addr == NULL) {
1706 PMD_INIT_LOG(ERR, "Bad mem resource."
1707 " Try to blacklist unused devices.");
1711 /* Store fm10k_adapter pointer */
1712 hw->back = dev->data->dev_private;
1714 /* Initialize the shared code */
1715 diag = fm10k_init_shared_code(hw);
1716 if (diag != FM10K_SUCCESS) {
1717 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
1722 * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
1723 * there is no way to get link status without reading BAR4. Until this
1724 * works, assume we have maximum bandwidth.
1725 * @todo - fix bus info
1727 hw->bus_caps.speed = fm10k_bus_speed_8000;
1728 hw->bus_caps.width = fm10k_bus_width_pcie_x8;
1729 hw->bus_caps.payload = fm10k_bus_payload_512;
1730 hw->bus.speed = fm10k_bus_speed_8000;
1731 hw->bus.width = fm10k_bus_width_pcie_x8;
1732 hw->bus.payload = fm10k_bus_payload_256;
1734 /* Initialize the hw */
1735 diag = fm10k_init_hw(hw);
1736 if (diag != FM10K_SUCCESS) {
1737 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
1741 /* Initialize MAC address(es) */
1742 dev->data->mac_addrs = rte_zmalloc("fm10k", ETHER_ADDR_LEN, 0);
1743 if (dev->data->mac_addrs == NULL) {
1744 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
1748 diag = fm10k_read_mac_addr(hw);
1749 if (diag != FM10K_SUCCESS) {
1751 * TODO: remove special handling on VF. Need shared code to
1754 if (hw->mac.type == fm10k_mac_pf) {
1755 PMD_INIT_LOG(ERR, "Read MAC addr failed: %d", diag);
1758 /* Generate a random addr */
1759 eth_random_addr(hw->mac.addr);
1760 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
1764 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
1765 &dev->data->mac_addrs[0]);
1767 /* Reset the hw statistics */
1768 fm10k_stats_reset(dev);
1771 diag = fm10k_reset_hw(hw);
1772 if (diag != FM10K_SUCCESS) {
1773 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
1777 /* Setup mailbox service */
1778 diag = fm10k_setup_mbx_service(hw);
1779 if (diag != FM10K_SUCCESS) {
1780 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
1784 /*PF/VF has different interrupt handling mechanism */
1785 if (hw->mac.type == fm10k_mac_pf) {
1786 /* register callback func to eal lib */
1787 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
1788 fm10k_dev_interrupt_handler_pf, (void *)dev);
1790 /* enable MISC interrupt */
1791 fm10k_dev_enable_intr_pf(dev);
1793 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
1794 fm10k_dev_interrupt_handler_vf, (void *)dev);
1796 fm10k_dev_enable_intr_vf(dev);
1800 * Below function will trigger operations on mailbox, acquire lock to
1801 * avoid race condition from interrupt handler. Operations on mailbox
1802 * FIFO will trigger interrupt to PF/SM, in which interrupt handler
1803 * will handle and generate an interrupt to our side. Then, FIFO in
1804 * mailbox will be touched.
1807 /* Enable port first */
1808 hw->mac.ops.update_lport_state(hw, 0, 0, 1);
1810 /* Update default vlan */
1811 hw->mac.ops.update_vlan(hw, hw->mac.default_vid, 0, true);
1814 * Add default mac/vlan filter. glort is assigned by SM for PF, while is
1815 * unused for VF. PF will assign correct glort for VF.
1817 hw->mac.ops.update_uc_addr(hw, hw->mac.dglort_map, hw->mac.addr,
1818 hw->mac.default_vid, 1, 0);
1820 /* Set unicast mode by default. App can change to other mode in other
1823 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
1824 FM10K_XCAST_MODE_MULTI);
1826 fm10k_mbx_unlock(hw);
1828 /* enable uio intr after callback registered */
1829 rte_intr_enable(&(dev->pci_dev->intr_handle));
1835 * The set of PCI devices this driver supports. This driver will enable both PF
1836 * and SRIOV-VF devices.
1838 static const struct rte_pci_id pci_id_fm10k_map[] = {
1839 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
1840 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
1841 #include "rte_pci_dev_ids.h"
1842 { .vendor_id = 0, /* sentinel */ },
1845 static struct eth_driver rte_pmd_fm10k = {
1847 .name = "rte_pmd_fm10k",
1848 .id_table = pci_id_fm10k_map,
1849 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
1851 .eth_dev_init = eth_fm10k_dev_init,
1852 .dev_private_size = sizeof(struct fm10k_adapter),
1856 * Driver initialization routine.
1857 * Invoked once at EAL init time.
1858 * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
1861 rte_pmd_fm10k_init(__rte_unused const char *name,
1862 __rte_unused const char *params)
1864 PMD_INIT_FUNC_TRACE();
1865 rte_eth_driver_register(&rte_pmd_fm10k);
1869 static struct rte_driver rte_fm10k_driver = {
1871 .init = rte_pmd_fm10k_init,
1874 PMD_REGISTER_DRIVER(rte_fm10k_driver);