4 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <rte_ethdev.h>
35 #include <rte_malloc.h>
36 #include <rte_memzone.h>
37 #include <rte_string_fns.h>
39 #include <rte_spinlock.h>
42 #include "base/fm10k_api.h"
44 /* Default delay to acquire mailbox lock */
45 #define FM10K_MBXLOCK_DELAY_US 20
46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
48 #define MAIN_VSI_POOL_NUMBER 0
50 /* Max try times to acquire switch status */
51 #define MAX_QUERY_SWITCH_STATE_TIMES 10
52 /* Wait interval to get switch status */
53 #define WAIT_SWITCH_MSG_US 100000
54 /* Number of chars per uint32 type */
55 #define CHARS_PER_UINT32 (sizeof(uint32_t))
56 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
58 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
59 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
60 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
61 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
62 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
63 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
65 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
66 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
67 const u8 *mac, bool add, uint32_t pool);
68 static void fm10k_tx_queue_release(void *queue);
69 static void fm10k_rx_queue_release(void *queue);
72 fm10k_mbx_initlock(struct fm10k_hw *hw)
74 rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
78 fm10k_mbx_lock(struct fm10k_hw *hw)
80 while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
81 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
85 fm10k_mbx_unlock(struct fm10k_hw *hw)
87 rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
91 * reset queue to initial state, allocate software buffers used when starting
94 * return -ENOMEM if buffers cannot be allocated
95 * return -EINVAL if buffers do not satisfy alignment condition
98 rx_queue_reset(struct fm10k_rx_queue *q)
102 PMD_INIT_FUNC_TRACE();
104 diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
108 for (i = 0; i < q->nb_desc; ++i) {
109 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
110 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
111 rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
115 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
116 q->hw_ring[i].q.pkt_addr = dma_addr;
117 q->hw_ring[i].q.hdr_addr = dma_addr;
122 q->next_trigger = q->alloc_thresh - 1;
123 FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
128 * clean queue, descriptor rings, free software buffers used when stopping
132 rx_queue_clean(struct fm10k_rx_queue *q)
134 union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
136 PMD_INIT_FUNC_TRACE();
138 /* zero descriptor rings */
139 for (i = 0; i < q->nb_desc; ++i)
140 q->hw_ring[i] = zero;
142 /* free software buffers */
143 for (i = 0; i < q->nb_desc; ++i) {
145 rte_pktmbuf_free_seg(q->sw_ring[i]);
146 q->sw_ring[i] = NULL;
152 * free all queue memory used when releasing the queue (i.e. configure)
155 rx_queue_free(struct fm10k_rx_queue *q)
157 PMD_INIT_FUNC_TRACE();
159 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
162 rte_free(q->sw_ring);
171 * disable RX queue, wait unitl HW finished necessary flush operation
174 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
178 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
179 FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
180 reg & ~FM10K_RXQCTL_ENABLE);
182 /* Wait 100us at most */
183 for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
185 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
186 if (!(reg & FM10K_RXQCTL_ENABLE))
190 if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
197 * reset queue to initial state, allocate software buffers used when starting
201 tx_queue_reset(struct fm10k_tx_queue *q)
203 PMD_INIT_FUNC_TRACE();
207 q->nb_free = q->nb_desc - 1;
208 fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
209 FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
213 * clean queue, descriptor rings, free software buffers used when stopping
217 tx_queue_clean(struct fm10k_tx_queue *q)
219 struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
221 PMD_INIT_FUNC_TRACE();
223 /* zero descriptor rings */
224 for (i = 0; i < q->nb_desc; ++i)
225 q->hw_ring[i] = zero;
227 /* free software buffers */
228 for (i = 0; i < q->nb_desc; ++i) {
230 rte_pktmbuf_free_seg(q->sw_ring[i]);
231 q->sw_ring[i] = NULL;
237 * free all queue memory used when releasing the queue (i.e. configure)
240 tx_queue_free(struct fm10k_tx_queue *q)
242 PMD_INIT_FUNC_TRACE();
244 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
246 if (q->rs_tracker.list) {
247 rte_free(q->rs_tracker.list);
248 q->rs_tracker.list = NULL;
251 rte_free(q->sw_ring);
260 * disable TX queue, wait unitl HW finished necessary flush operation
263 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
267 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
268 FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
269 reg & ~FM10K_TXDCTL_ENABLE);
271 /* Wait 100us at most */
272 for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
274 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
275 if (!(reg & FM10K_TXDCTL_ENABLE))
279 if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
286 fm10k_check_mq_mode(struct rte_eth_dev *dev)
288 enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
289 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
290 struct rte_eth_vmdq_rx_conf *vmdq_conf;
291 uint16_t nb_rx_q = dev->data->nb_rx_queues;
293 vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
295 if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
296 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
300 if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
303 if (hw->mac.type == fm10k_mac_vf) {
304 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
308 /* Check VMDQ queue pool number */
309 if (vmdq_conf->nb_queue_pools >
310 sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
311 vmdq_conf->nb_queue_pools > nb_rx_q) {
312 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
313 vmdq_conf->nb_queue_pools);
321 fm10k_dev_configure(struct rte_eth_dev *dev)
325 PMD_INIT_FUNC_TRACE();
327 if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
328 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
329 /* multipe queue mode checking */
330 ret = fm10k_check_mq_mode(dev);
332 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
341 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
343 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
344 struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
345 uint32_t mrqc, *key, i, reta, j;
348 #define RSS_KEY_SIZE 40
349 static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
350 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
351 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
352 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
353 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
354 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
357 if (dev->data->nb_rx_queues == 1 ||
358 dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
359 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
362 /* random key is rss_intel_key (default) or user provided (rss_key) */
363 if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
364 key = (uint32_t *)rss_intel_key;
366 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
368 /* Now fill our hash function seeds, 4 bytes at a time */
369 for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
370 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
373 * Fill in redirection table
374 * The byte-swap is needed because NIC registers are in
375 * little-endian order.
378 for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
379 if (j == dev->data->nb_rx_queues)
381 reta = (reta << CHAR_BIT) | j;
383 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
388 * Generate RSS hash based on packet types, TCP/UDP
389 * port numbers and/or IPv4/v6 src and dst addresses
391 hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
393 mrqc |= (hf & ETH_RSS_IPV4) ? FM10K_MRQC_IPV4 : 0;
394 mrqc |= (hf & ETH_RSS_IPV6) ? FM10K_MRQC_IPV6 : 0;
395 mrqc |= (hf & ETH_RSS_IPV6_EX) ? FM10K_MRQC_IPV6 : 0;
396 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP) ? FM10K_MRQC_TCP_IPV4 : 0;
397 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP) ? FM10K_MRQC_TCP_IPV6 : 0;
398 mrqc |= (hf & ETH_RSS_IPV6_TCP_EX) ? FM10K_MRQC_TCP_IPV6 : 0;
399 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP) ? FM10K_MRQC_UDP_IPV4 : 0;
400 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP) ? FM10K_MRQC_UDP_IPV6 : 0;
401 mrqc |= (hf & ETH_RSS_IPV6_UDP_EX) ? FM10K_MRQC_UDP_IPV6 : 0;
404 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
409 FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
413 fm10k_dev_tx_init(struct rte_eth_dev *dev)
415 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
417 struct fm10k_tx_queue *txq;
421 /* Disable TXINT to avoid possible interrupt */
422 for (i = 0; i < hw->mac.max_queues; i++)
423 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
424 3 << FM10K_TXINT_TIMER_SHIFT);
427 for (i = 0; i < dev->data->nb_tx_queues; ++i) {
428 txq = dev->data->tx_queues[i];
429 base_addr = txq->hw_ring_phys_addr;
430 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
432 /* disable queue to avoid issues while updating state */
433 ret = tx_queue_disable(hw, i);
435 PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
439 /* set location and size for descriptor ring */
440 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
441 base_addr & UINT64_LOWER_32BITS_MASK);
442 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
443 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
444 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
450 fm10k_dev_rx_init(struct rte_eth_dev *dev)
452 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
454 struct fm10k_rx_queue *rxq;
457 uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
460 /* Disable RXINT to avoid possible interrupt */
461 for (i = 0; i < hw->mac.max_queues; i++)
462 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
463 3 << FM10K_RXINT_TIMER_SHIFT);
465 /* Setup RX queues */
466 for (i = 0; i < dev->data->nb_rx_queues; ++i) {
467 rxq = dev->data->rx_queues[i];
468 base_addr = rxq->hw_ring_phys_addr;
469 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
471 /* disable queue to avoid issues while updating state */
472 ret = rx_queue_disable(hw, i);
474 PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
478 /* Setup the Base and Length of the Rx Descriptor Ring */
479 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
480 base_addr & UINT64_LOWER_32BITS_MASK);
481 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
482 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
483 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
485 /* Configure the Rx buffer size for one buff without split */
486 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
487 RTE_PKTMBUF_HEADROOM);
488 /* As RX buffer is aligned to 512B within mbuf, some bytes are
489 * reserved for this purpose, and the worst case could be 511B.
490 * But SRR reg assumes all buffers have the same size. In order
491 * to fill the gap, we'll have to consider the worst case and
492 * assume 512B is reserved. If we don't do so, it's possible
493 * for HW to overwrite data to next mbuf.
495 buf_size -= FM10K_RX_DATABUF_ALIGN;
497 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
498 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
500 /* It adds dual VLAN length for supporting dual VLAN */
501 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
502 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
503 dev->data->dev_conf.rxmode.enable_scatter) {
505 dev->data->scattered_rx = 1;
506 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
507 reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
508 reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
509 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
512 /* Enable drop on empty, it's RO for VF */
513 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
514 rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
516 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
517 FM10K_WRITE_FLUSH(hw);
520 /* Configure RSS if applicable */
521 fm10k_dev_mq_rx_configure(dev);
526 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
528 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
531 struct fm10k_rx_queue *rxq;
533 PMD_INIT_FUNC_TRACE();
535 if (rx_queue_id < dev->data->nb_rx_queues) {
536 rxq = dev->data->rx_queues[rx_queue_id];
537 err = rx_queue_reset(rxq);
538 if (err == -ENOMEM) {
539 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
541 } else if (err == -EINVAL) {
542 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
547 /* Setup the HW Rx Head and Tail Descriptor Pointers
548 * Note: this must be done AFTER the queue is enabled on real
549 * hardware, but BEFORE the queue is enabled when using the
550 * emulation platform. Do it in both places for now and remove
551 * this comment and the following two register writes when the
552 * emulation platform is no longer being used.
554 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
555 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
557 /* Set PF ownership flag for PF devices */
558 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
559 if (hw->mac.type == fm10k_mac_pf)
560 reg |= FM10K_RXQCTL_PF;
561 reg |= FM10K_RXQCTL_ENABLE;
562 /* enable RX queue */
563 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
564 FM10K_WRITE_FLUSH(hw);
566 /* Setup the HW Rx Head and Tail Descriptor Pointers
567 * Note: this must be done AFTER the queue is enabled
569 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
570 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
577 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
579 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
581 PMD_INIT_FUNC_TRACE();
583 if (rx_queue_id < dev->data->nb_rx_queues) {
584 /* Disable RX queue */
585 rx_queue_disable(hw, rx_queue_id);
587 /* Free mbuf and clean HW ring */
588 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
595 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
597 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
598 /** @todo - this should be defined in the shared code */
599 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY 0x00010000
600 uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
603 PMD_INIT_FUNC_TRACE();
605 if (tx_queue_id < dev->data->nb_tx_queues) {
606 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
608 /* reset head and tail pointers */
609 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
610 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
612 /* enable TX queue */
613 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
614 FM10K_TXDCTL_ENABLE | txdctl);
615 FM10K_WRITE_FLUSH(hw);
623 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
625 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
627 PMD_INIT_FUNC_TRACE();
629 if (tx_queue_id < dev->data->nb_tx_queues) {
630 tx_queue_disable(hw, tx_queue_id);
631 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
637 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
639 return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
640 != FM10K_DGLORTMAP_NONE);
644 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
646 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
649 PMD_INIT_FUNC_TRACE();
651 /* Return if it didn't acquire valid glort range */
652 if (!fm10k_glort_valid(hw))
656 status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
657 FM10K_XCAST_MODE_PROMISC);
658 fm10k_mbx_unlock(hw);
660 if (status != FM10K_SUCCESS)
661 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
665 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
667 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
671 PMD_INIT_FUNC_TRACE();
673 /* Return if it didn't acquire valid glort range */
674 if (!fm10k_glort_valid(hw))
677 if (dev->data->all_multicast == 1)
678 mode = FM10K_XCAST_MODE_ALLMULTI;
680 mode = FM10K_XCAST_MODE_NONE;
683 status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
685 fm10k_mbx_unlock(hw);
687 if (status != FM10K_SUCCESS)
688 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
692 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
694 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
697 PMD_INIT_FUNC_TRACE();
699 /* Return if it didn't acquire valid glort range */
700 if (!fm10k_glort_valid(hw))
703 /* If promiscuous mode is enabled, it doesn't make sense to enable
704 * allmulticast and disable promiscuous since fm10k only can select
707 if (dev->data->promiscuous) {
708 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
709 "needn't enable allmulticast");
714 status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
715 FM10K_XCAST_MODE_ALLMULTI);
716 fm10k_mbx_unlock(hw);
718 if (status != FM10K_SUCCESS)
719 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
723 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
725 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
728 PMD_INIT_FUNC_TRACE();
730 /* Return if it didn't acquire valid glort range */
731 if (!fm10k_glort_valid(hw))
734 if (dev->data->promiscuous) {
735 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
736 "since promisc mode is enabled");
741 /* Change mode to unicast mode */
742 status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
743 FM10K_XCAST_MODE_NONE);
744 fm10k_mbx_unlock(hw);
746 if (status != FM10K_SUCCESS)
747 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
750 /* fls = find last set bit = 32 minus the number of leading zeros */
752 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
754 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
756 fm10k_dev_start(struct rte_eth_dev *dev)
758 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
761 PMD_INIT_FUNC_TRACE();
763 /* stop, init, then start the hw */
764 diag = fm10k_stop_hw(hw);
765 if (diag != FM10K_SUCCESS) {
766 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
770 diag = fm10k_init_hw(hw);
771 if (diag != FM10K_SUCCESS) {
772 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
776 diag = fm10k_start_hw(hw);
777 if (diag != FM10K_SUCCESS) {
778 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
782 diag = fm10k_dev_tx_init(dev);
784 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
788 diag = fm10k_dev_rx_init(dev);
790 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
794 if (hw->mac.type == fm10k_mac_pf) {
795 /* Establish only VSI 0 as valid */
796 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
798 /* Configure RSS bits used in RETA table */
799 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0),
800 fls(dev->data->nb_rx_queues - 1) <<
801 FM10K_DGLORTDEC_RSSLENGTH_SHIFT);
803 /* Invalidate all other GLORT entries */
804 for (i = 1; i < FM10K_DGLORT_COUNT; i++)
805 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
806 FM10K_DGLORTMAP_NONE);
809 for (i = 0; i < dev->data->nb_rx_queues; i++) {
810 struct fm10k_rx_queue *rxq;
811 rxq = dev->data->rx_queues[i];
813 if (rxq->rx_deferred_start)
815 diag = fm10k_dev_rx_queue_start(dev, i);
818 for (j = 0; j < i; ++j)
819 rx_queue_clean(dev->data->rx_queues[j]);
824 for (i = 0; i < dev->data->nb_tx_queues; i++) {
825 struct fm10k_tx_queue *txq;
826 txq = dev->data->tx_queues[i];
828 if (txq->tx_deferred_start)
830 diag = fm10k_dev_tx_queue_start(dev, i);
833 for (j = 0; j < i; ++j)
834 tx_queue_clean(dev->data->tx_queues[j]);
835 for (j = 0; j < dev->data->nb_rx_queues; ++j)
836 rx_queue_clean(dev->data->rx_queues[j]);
841 /* Update default vlan */
842 if (hw->mac.default_vid && hw->mac.default_vid <= ETHER_MAX_VLAN_ID)
843 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
849 fm10k_dev_stop(struct rte_eth_dev *dev)
853 PMD_INIT_FUNC_TRACE();
855 if (dev->data->tx_queues)
856 for (i = 0; i < dev->data->nb_tx_queues; i++)
857 fm10k_dev_tx_queue_stop(dev, i);
859 if (dev->data->rx_queues)
860 for (i = 0; i < dev->data->nb_rx_queues; i++)
861 fm10k_dev_rx_queue_stop(dev, i);
865 fm10k_dev_queue_release(struct rte_eth_dev *dev)
869 PMD_INIT_FUNC_TRACE();
871 if (dev->data->tx_queues) {
872 for (i = 0; i < dev->data->nb_tx_queues; i++)
873 fm10k_tx_queue_release(dev->data->tx_queues[i]);
876 if (dev->data->rx_queues) {
877 for (i = 0; i < dev->data->nb_rx_queues; i++)
878 fm10k_rx_queue_release(dev->data->rx_queues[i]);
883 fm10k_dev_close(struct rte_eth_dev *dev)
885 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
887 struct fm10k_macvlan_filter_info *macvlan;
889 PMD_INIT_FUNC_TRACE();
891 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
892 nb_lport = macvlan->nb_queue_pools ? macvlan->nb_queue_pools : 1;
894 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map,
896 fm10k_mbx_unlock(hw);
898 /* Stop mailbox service first */
899 fm10k_close_mbx_service(hw);
901 fm10k_dev_queue_release(dev);
906 fm10k_link_update(struct rte_eth_dev *dev,
907 __rte_unused int wait_to_complete)
909 PMD_INIT_FUNC_TRACE();
911 /* The host-interface link is always up. The speed is ~50Gbps per Gen3
912 * x8 PCIe interface. For now, we leave the speed undefined since there
913 * is no 50Gbps Ethernet. */
914 dev->data->dev_link.link_speed = 0;
915 dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
916 dev->data->dev_link.link_status = 1;
922 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
924 uint64_t ipackets, opackets, ibytes, obytes;
925 struct fm10k_hw *hw =
926 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
927 struct fm10k_hw_stats *hw_stats =
928 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
931 PMD_INIT_FUNC_TRACE();
933 fm10k_update_hw_stats(hw, hw_stats);
935 ipackets = opackets = ibytes = obytes = 0;
936 for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
937 (i < hw->mac.max_queues); ++i) {
938 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
939 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
940 stats->q_ibytes[i] = hw_stats->q[i].rx_bytes.count;
941 stats->q_obytes[i] = hw_stats->q[i].tx_bytes.count;
942 ipackets += stats->q_ipackets[i];
943 opackets += stats->q_opackets[i];
944 ibytes += stats->q_ibytes[i];
945 obytes += stats->q_obytes[i];
947 stats->ipackets = ipackets;
948 stats->opackets = opackets;
949 stats->ibytes = ibytes;
950 stats->obytes = obytes;
954 fm10k_stats_reset(struct rte_eth_dev *dev)
956 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
957 struct fm10k_hw_stats *hw_stats =
958 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
960 PMD_INIT_FUNC_TRACE();
962 memset(hw_stats, 0, sizeof(*hw_stats));
963 fm10k_rebind_hw_stats(hw, hw_stats);
967 fm10k_dev_infos_get(struct rte_eth_dev *dev,
968 struct rte_eth_dev_info *dev_info)
970 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
972 PMD_INIT_FUNC_TRACE();
974 dev_info->min_rx_bufsize = FM10K_MIN_RX_BUF_SIZE;
975 dev_info->max_rx_pktlen = FM10K_MAX_PKT_SIZE;
976 dev_info->max_rx_queues = hw->mac.max_queues;
977 dev_info->max_tx_queues = hw->mac.max_queues;
978 dev_info->max_mac_addrs = FM10K_MAX_MACADDR_NUM;
979 dev_info->max_hash_mac_addrs = 0;
980 dev_info->max_vfs = dev->pci_dev->max_vfs;
981 dev_info->max_vmdq_pools = ETH_64_POOLS;
982 dev_info->rx_offload_capa =
983 DEV_RX_OFFLOAD_VLAN_STRIP |
984 DEV_RX_OFFLOAD_IPV4_CKSUM |
985 DEV_RX_OFFLOAD_UDP_CKSUM |
986 DEV_RX_OFFLOAD_TCP_CKSUM;
987 dev_info->tx_offload_capa =
988 DEV_TX_OFFLOAD_VLAN_INSERT |
989 DEV_TX_OFFLOAD_IPV4_CKSUM |
990 DEV_TX_OFFLOAD_UDP_CKSUM |
991 DEV_TX_OFFLOAD_TCP_CKSUM |
992 DEV_TX_OFFLOAD_TCP_TSO;
994 dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
995 dev_info->reta_size = FM10K_MAX_RSS_INDICES;
997 dev_info->default_rxconf = (struct rte_eth_rxconf) {
999 .pthresh = FM10K_DEFAULT_RX_PTHRESH,
1000 .hthresh = FM10K_DEFAULT_RX_HTHRESH,
1001 .wthresh = FM10K_DEFAULT_RX_WTHRESH,
1003 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1007 dev_info->default_txconf = (struct rte_eth_txconf) {
1009 .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1010 .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1011 .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1013 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1014 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1015 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
1016 ETH_TXQ_FLAGS_NOOFFLOADS,
1022 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1025 uint16_t mac_num = 0;
1026 uint32_t vid_idx, vid_bit, mac_index;
1027 struct fm10k_hw *hw;
1028 struct fm10k_macvlan_filter_info *macvlan;
1029 struct rte_eth_dev_data *data = dev->data;
1031 hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1032 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1034 if (macvlan->nb_queue_pools > 0) { /* VMDQ mode */
1035 PMD_INIT_LOG(ERR, "Cannot change VLAN filter in VMDQ mode");
1039 if (vlan_id > ETH_VLAN_ID_MAX) {
1040 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1044 vid_idx = FM10K_VFTA_IDX(vlan_id);
1045 vid_bit = FM10K_VFTA_BIT(vlan_id);
1046 /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1047 if (on && (macvlan->vfta[vid_idx] & vid_bit))
1049 /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1050 if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1051 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1052 "in the VLAN filter table");
1057 result = fm10k_update_vlan(hw, vlan_id, 0, on);
1058 fm10k_mbx_unlock(hw);
1059 if (result != FM10K_SUCCESS) {
1060 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1064 for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1065 (result == FM10K_SUCCESS); mac_index++) {
1066 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1068 if (mac_num > macvlan->mac_num - 1) {
1069 PMD_INIT_LOG(ERR, "MAC address number "
1074 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1075 data->mac_addrs[mac_index].addr_bytes,
1077 fm10k_mbx_unlock(hw);
1080 if (result != FM10K_SUCCESS) {
1081 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1086 macvlan->vlan_num++;
1087 macvlan->vfta[vid_idx] |= vid_bit;
1089 macvlan->vlan_num--;
1090 macvlan->vfta[vid_idx] &= ~vid_bit;
1096 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1098 if (mask & ETH_VLAN_STRIP_MASK) {
1099 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1100 PMD_INIT_LOG(ERR, "VLAN stripping is "
1101 "always on in fm10k");
1104 if (mask & ETH_VLAN_EXTEND_MASK) {
1105 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1106 PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1107 "supported in fm10k");
1110 if (mask & ETH_VLAN_FILTER_MASK) {
1111 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1112 PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1116 /* Add/Remove a MAC address, and update filters to main VSI */
1117 static void fm10k_MAC_filter_set_main_vsi(struct rte_eth_dev *dev,
1118 const u8 *mac, bool add, uint32_t pool)
1120 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1121 struct fm10k_macvlan_filter_info *macvlan;
1124 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1126 if (pool != MAIN_VSI_POOL_NUMBER) {
1127 PMD_DRV_LOG(ERR, "VMDQ not enabled, can't set "
1128 "mac to pool %u", pool);
1131 for (i = 0, j = 0; j < FM10K_VFTA_SIZE; j++) {
1132 if (!macvlan->vfta[j])
1134 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1135 if (!(macvlan->vfta[j] & (1 << k)))
1137 if (i + 1 > macvlan->vlan_num) {
1138 PMD_INIT_LOG(ERR, "vlan number not match");
1142 fm10k_update_uc_addr(hw, hw->mac.dglort_map, mac,
1143 j * FM10K_UINT32_BIT_SIZE + k, add, 0);
1144 fm10k_mbx_unlock(hw);
1150 /* Add/Remove a MAC address, and update filters to VMDQ */
1151 static void fm10k_MAC_filter_set_vmdq(struct rte_eth_dev *dev,
1152 const u8 *mac, bool add, uint32_t pool)
1154 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1155 struct fm10k_macvlan_filter_info *macvlan;
1156 struct rte_eth_vmdq_rx_conf *vmdq_conf;
1159 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1160 vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
1162 if (pool > macvlan->nb_queue_pools) {
1163 PMD_DRV_LOG(ERR, "Pool number %u invalid."
1165 pool, macvlan->nb_queue_pools);
1168 for (i = 0; i < vmdq_conf->nb_pool_maps; i++) {
1169 if (!(vmdq_conf->pool_map[i].pools & (1UL << pool)))
1172 fm10k_update_uc_addr(hw, hw->mac.dglort_map + pool, mac,
1173 vmdq_conf->pool_map[i].vlan_id, add, 0);
1174 fm10k_mbx_unlock(hw);
1178 /* Add/Remove a MAC address, and update filters */
1179 static void fm10k_MAC_filter_set(struct rte_eth_dev *dev,
1180 const u8 *mac, bool add, uint32_t pool)
1182 struct fm10k_macvlan_filter_info *macvlan;
1184 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1186 if (macvlan->nb_queue_pools > 0) /* VMDQ mode */
1187 fm10k_MAC_filter_set_vmdq(dev, mac, add, pool);
1189 fm10k_MAC_filter_set_main_vsi(dev, mac, add, pool);
1197 /* Add a MAC address, and update filters */
1199 fm10k_macaddr_add(struct rte_eth_dev *dev,
1200 struct ether_addr *mac_addr,
1204 struct fm10k_macvlan_filter_info *macvlan;
1206 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1207 fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE, pool);
1208 macvlan->mac_vmdq_id[index] = pool;
1211 /* Remove a MAC address, and update filters */
1213 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1215 struct rte_eth_dev_data *data = dev->data;
1216 struct fm10k_macvlan_filter_info *macvlan;
1218 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1219 fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1220 FALSE, macvlan->mac_vmdq_id[index]);
1221 macvlan->mac_vmdq_id[index] = 0;
1225 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1227 if ((request < min) || (request > max) || ((request % mult) != 0))
1234 * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1235 * the physical address is required. If the memzone is already created, then
1236 * this function returns a pointer to the existing memzone.
1238 static inline const struct rte_memzone *
1239 allocate_hw_ring(const char *driver_name, const char *ring_name,
1240 uint8_t port_id, uint16_t queue_id, int socket_id,
1241 uint32_t size, uint32_t align)
1243 char name[RTE_MEMZONE_NAMESIZE];
1244 const struct rte_memzone *mz;
1246 snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1247 driver_name, ring_name, port_id, queue_id, socket_id);
1249 /* return the memzone if it already exists */
1250 mz = rte_memzone_lookup(name);
1254 #ifdef RTE_LIBRTE_XEN_DOM0
1255 return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1258 return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1263 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1265 if ((request < min) || (request > max) || ((div % request) != 0))
1272 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1274 uint16_t rx_free_thresh;
1276 if (conf->rx_free_thresh == 0)
1277 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1279 rx_free_thresh = conf->rx_free_thresh;
1281 /* make sure the requested threshold satisfies the constraints */
1282 if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1283 FM10K_RX_FREE_THRESH_MAX(q),
1284 FM10K_RX_FREE_THRESH_DIV(q),
1286 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1287 "less than or equal to %u, "
1288 "greater than or equal to %u, "
1289 "and a divisor of %u",
1290 rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1291 FM10K_RX_FREE_THRESH_MIN(q),
1292 FM10K_RX_FREE_THRESH_DIV(q));
1296 q->alloc_thresh = rx_free_thresh;
1297 q->drop_en = conf->rx_drop_en;
1298 q->rx_deferred_start = conf->rx_deferred_start;
1304 * Hardware requires specific alignment for Rx packet buffers. At
1305 * least one of the following two conditions must be satisfied.
1306 * 1. Address is 512B aligned
1307 * 2. Address is 8B aligned and buffer does not cross 4K boundary.
1309 * As such, the driver may need to adjust the DMA address within the
1310 * buffer by up to 512B.
1312 * return 1 if the element size is valid, otherwise return 0.
1315 mempool_element_size_valid(struct rte_mempool *mp)
1319 /* elt_size includes mbuf header and headroom */
1320 min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1321 RTE_PKTMBUF_HEADROOM;
1323 /* account for up to 512B of alignment */
1324 min_size -= FM10K_RX_DATABUF_ALIGN;
1326 /* sanity check for overflow */
1327 if (min_size > mp->elt_size)
1335 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1336 uint16_t nb_desc, unsigned int socket_id,
1337 const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1339 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1340 struct fm10k_rx_queue *q;
1341 const struct rte_memzone *mz;
1343 PMD_INIT_FUNC_TRACE();
1345 /* make sure the mempool element size can account for alignment. */
1346 if (!mempool_element_size_valid(mp)) {
1347 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1351 /* make sure a valid number of descriptors have been requested */
1352 if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1353 FM10K_MULT_RX_DESC, nb_desc)) {
1354 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1355 "less than or equal to %"PRIu32", "
1356 "greater than or equal to %u, "
1357 "and a multiple of %u",
1358 nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1359 FM10K_MULT_RX_DESC);
1364 * if this queue existed already, free the associated memory. The
1365 * queue cannot be reused in case we need to allocate memory on
1366 * different socket than was previously used.
1368 if (dev->data->rx_queues[queue_id] != NULL) {
1369 rx_queue_free(dev->data->rx_queues[queue_id]);
1370 dev->data->rx_queues[queue_id] = NULL;
1373 /* allocate memory for the queue structure */
1374 q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1377 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1383 q->nb_desc = nb_desc;
1384 q->port_id = dev->data->port_id;
1385 q->queue_id = queue_id;
1386 q->tail_ptr = (volatile uint32_t *)
1387 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1388 if (handle_rxconf(q, conf))
1391 /* allocate memory for the software ring */
1392 q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1393 nb_desc * sizeof(struct rte_mbuf *),
1394 RTE_CACHE_LINE_SIZE, socket_id);
1395 if (q->sw_ring == NULL) {
1396 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1402 * allocate memory for the hardware descriptor ring. A memzone large
1403 * enough to hold the maximum ring size is requested to allow for
1404 * resizing in later calls to the queue setup function.
1406 mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1407 dev->data->port_id, queue_id, socket_id,
1408 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1410 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1411 rte_free(q->sw_ring);
1415 q->hw_ring = mz->addr;
1416 #ifdef RTE_LIBRTE_XEN_DOM0
1417 q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1419 q->hw_ring_phys_addr = mz->phys_addr;
1422 dev->data->rx_queues[queue_id] = q;
1427 fm10k_rx_queue_release(void *queue)
1429 PMD_INIT_FUNC_TRACE();
1431 rx_queue_free(queue);
1435 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1437 uint16_t tx_free_thresh;
1438 uint16_t tx_rs_thresh;
1440 /* constraint MACROs require that tx_free_thresh is configured
1441 * before tx_rs_thresh */
1442 if (conf->tx_free_thresh == 0)
1443 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1445 tx_free_thresh = conf->tx_free_thresh;
1447 /* make sure the requested threshold satisfies the constraints */
1448 if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1449 FM10K_TX_FREE_THRESH_MAX(q),
1450 FM10K_TX_FREE_THRESH_DIV(q),
1452 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1453 "less than or equal to %u, "
1454 "greater than or equal to %u, "
1455 "and a divisor of %u",
1456 tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1457 FM10K_TX_FREE_THRESH_MIN(q),
1458 FM10K_TX_FREE_THRESH_DIV(q));
1462 q->free_thresh = tx_free_thresh;
1464 if (conf->tx_rs_thresh == 0)
1465 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1467 tx_rs_thresh = conf->tx_rs_thresh;
1469 q->tx_deferred_start = conf->tx_deferred_start;
1471 /* make sure the requested threshold satisfies the constraints */
1472 if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1473 FM10K_TX_RS_THRESH_MAX(q),
1474 FM10K_TX_RS_THRESH_DIV(q),
1476 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1477 "less than or equal to %u, "
1478 "greater than or equal to %u, "
1479 "and a divisor of %u",
1480 tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1481 FM10K_TX_RS_THRESH_MIN(q),
1482 FM10K_TX_RS_THRESH_DIV(q));
1486 q->rs_thresh = tx_rs_thresh;
1492 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1493 uint16_t nb_desc, unsigned int socket_id,
1494 const struct rte_eth_txconf *conf)
1496 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1497 struct fm10k_tx_queue *q;
1498 const struct rte_memzone *mz;
1500 PMD_INIT_FUNC_TRACE();
1502 /* make sure a valid number of descriptors have been requested */
1503 if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1504 FM10K_MULT_TX_DESC, nb_desc)) {
1505 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1506 "less than or equal to %"PRIu32", "
1507 "greater than or equal to %u, "
1508 "and a multiple of %u",
1509 nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1510 FM10K_MULT_TX_DESC);
1515 * if this queue existed already, free the associated memory. The
1516 * queue cannot be reused in case we need to allocate memory on
1517 * different socket than was previously used.
1519 if (dev->data->tx_queues[queue_id] != NULL) {
1520 tx_queue_free(dev->data->tx_queues[queue_id]);
1521 dev->data->tx_queues[queue_id] = NULL;
1524 /* allocate memory for the queue structure */
1525 q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1528 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1533 q->nb_desc = nb_desc;
1534 q->port_id = dev->data->port_id;
1535 q->queue_id = queue_id;
1536 q->tail_ptr = (volatile uint32_t *)
1537 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1538 if (handle_txconf(q, conf))
1541 /* allocate memory for the software ring */
1542 q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1543 nb_desc * sizeof(struct rte_mbuf *),
1544 RTE_CACHE_LINE_SIZE, socket_id);
1545 if (q->sw_ring == NULL) {
1546 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1552 * allocate memory for the hardware descriptor ring. A memzone large
1553 * enough to hold the maximum ring size is requested to allow for
1554 * resizing in later calls to the queue setup function.
1556 mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1557 dev->data->port_id, queue_id, socket_id,
1558 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1560 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1561 rte_free(q->sw_ring);
1565 q->hw_ring = mz->addr;
1566 #ifdef RTE_LIBRTE_XEN_DOM0
1567 q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1569 q->hw_ring_phys_addr = mz->phys_addr;
1573 * allocate memory for the RS bit tracker. Enough slots to hold the
1574 * descriptor index for each RS bit needing to be set are required.
1576 q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1577 ((nb_desc + 1) / q->rs_thresh) *
1579 RTE_CACHE_LINE_SIZE, socket_id);
1580 if (q->rs_tracker.list == NULL) {
1581 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1582 rte_free(q->sw_ring);
1587 dev->data->tx_queues[queue_id] = q;
1592 fm10k_tx_queue_release(void *queue)
1594 PMD_INIT_FUNC_TRACE();
1596 tx_queue_free(queue);
1600 fm10k_reta_update(struct rte_eth_dev *dev,
1601 struct rte_eth_rss_reta_entry64 *reta_conf,
1604 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1605 uint16_t i, j, idx, shift;
1609 PMD_INIT_FUNC_TRACE();
1611 if (reta_size > FM10K_MAX_RSS_INDICES) {
1612 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1613 "(%d) doesn't match the number hardware can supported "
1614 "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1619 * Update Redirection Table RETA[n], n=0..31. The redirection table has
1620 * 128-entries in 32 registers
1622 for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1623 idx = i / RTE_RETA_GROUP_SIZE;
1624 shift = i % RTE_RETA_GROUP_SIZE;
1625 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1626 BIT_MASK_PER_UINT32);
1631 if (mask != BIT_MASK_PER_UINT32)
1632 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1634 for (j = 0; j < CHARS_PER_UINT32; j++) {
1635 if (mask & (0x1 << j)) {
1637 reta &= ~(UINT8_MAX << CHAR_BIT * j);
1638 reta |= reta_conf[idx].reta[shift + j] <<
1642 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1649 fm10k_reta_query(struct rte_eth_dev *dev,
1650 struct rte_eth_rss_reta_entry64 *reta_conf,
1653 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1654 uint16_t i, j, idx, shift;
1658 PMD_INIT_FUNC_TRACE();
1660 if (reta_size < FM10K_MAX_RSS_INDICES) {
1661 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1662 "(%d) doesn't match the number hardware can supported "
1663 "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1668 * Read Redirection Table RETA[n], n=0..31. The redirection table has
1669 * 128-entries in 32 registers
1671 for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1672 idx = i / RTE_RETA_GROUP_SIZE;
1673 shift = i % RTE_RETA_GROUP_SIZE;
1674 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1675 BIT_MASK_PER_UINT32);
1679 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1680 for (j = 0; j < CHARS_PER_UINT32; j++) {
1681 if (mask & (0x1 << j))
1682 reta_conf[idx].reta[shift + j] = ((reta >>
1683 CHAR_BIT * j) & UINT8_MAX);
1691 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1692 struct rte_eth_rss_conf *rss_conf)
1694 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1695 uint32_t *key = (uint32_t *)rss_conf->rss_key;
1697 uint64_t hf = rss_conf->rss_hf;
1700 PMD_INIT_FUNC_TRACE();
1702 if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1703 FM10K_RSSRK_ENTRIES_PER_REG)
1710 mrqc |= (hf & ETH_RSS_IPV4) ? FM10K_MRQC_IPV4 : 0;
1711 mrqc |= (hf & ETH_RSS_IPV6) ? FM10K_MRQC_IPV6 : 0;
1712 mrqc |= (hf & ETH_RSS_IPV6_EX) ? FM10K_MRQC_IPV6 : 0;
1713 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP) ? FM10K_MRQC_TCP_IPV4 : 0;
1714 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP) ? FM10K_MRQC_TCP_IPV6 : 0;
1715 mrqc |= (hf & ETH_RSS_IPV6_TCP_EX) ? FM10K_MRQC_TCP_IPV6 : 0;
1716 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP) ? FM10K_MRQC_UDP_IPV4 : 0;
1717 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP) ? FM10K_MRQC_UDP_IPV6 : 0;
1718 mrqc |= (hf & ETH_RSS_IPV6_UDP_EX) ? FM10K_MRQC_UDP_IPV6 : 0;
1720 /* If the mapping doesn't fit any supported, return */
1725 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1726 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1728 FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1734 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1735 struct rte_eth_rss_conf *rss_conf)
1737 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1738 uint32_t *key = (uint32_t *)rss_conf->rss_key;
1743 PMD_INIT_FUNC_TRACE();
1745 if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1746 FM10K_RSSRK_ENTRIES_PER_REG)
1750 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1751 key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1753 mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1755 hf |= (mrqc & FM10K_MRQC_IPV4) ? ETH_RSS_IPV4 : 0;
1756 hf |= (mrqc & FM10K_MRQC_IPV6) ? ETH_RSS_IPV6 : 0;
1757 hf |= (mrqc & FM10K_MRQC_IPV6) ? ETH_RSS_IPV6_EX : 0;
1758 hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP : 0;
1759 hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP : 0;
1760 hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX : 0;
1761 hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP : 0;
1762 hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP : 0;
1763 hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX : 0;
1765 rss_conf->rss_hf = hf;
1771 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1773 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1774 uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1776 /* Bind all local non-queue interrupt to vector 0 */
1779 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1780 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1781 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1782 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1783 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1784 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1786 /* Enable misc causes */
1787 FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1788 FM10K_EIMR_ENABLE(THI_FAULT) |
1789 FM10K_EIMR_ENABLE(FUM_FAULT) |
1790 FM10K_EIMR_ENABLE(MAILBOX) |
1791 FM10K_EIMR_ENABLE(SWITCHREADY) |
1792 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1793 FM10K_EIMR_ENABLE(SRAMERROR) |
1794 FM10K_EIMR_ENABLE(VFLR));
1797 FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1798 FM10K_ITR_MASK_CLEAR);
1799 FM10K_WRITE_FLUSH(hw);
1803 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
1805 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1806 uint32_t int_map = FM10K_INT_MAP_DISABLE;
1810 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1811 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1812 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1813 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1814 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1815 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1817 /* Disable misc causes */
1818 FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
1819 FM10K_EIMR_DISABLE(THI_FAULT) |
1820 FM10K_EIMR_DISABLE(FUM_FAULT) |
1821 FM10K_EIMR_DISABLE(MAILBOX) |
1822 FM10K_EIMR_DISABLE(SWITCHREADY) |
1823 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
1824 FM10K_EIMR_DISABLE(SRAMERROR) |
1825 FM10K_EIMR_DISABLE(VFLR));
1828 FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
1829 FM10K_WRITE_FLUSH(hw);
1833 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1835 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1836 uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1838 /* Bind all local non-queue interrupt to vector 0 */
1841 /* Only INT 0 available, other 15 are reserved. */
1842 FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1845 FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1846 FM10K_ITR_MASK_CLEAR);
1847 FM10K_WRITE_FLUSH(hw);
1851 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
1853 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1854 uint32_t int_map = FM10K_INT_MAP_DISABLE;
1858 /* Only INT 0 available, other 15 are reserved. */
1859 FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1862 FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
1863 FM10K_WRITE_FLUSH(hw);
1867 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1869 struct fm10k_fault fault;
1871 const char *estr = "Unknown error";
1873 /* Process PCA fault */
1874 if (eicr & FM10K_EICR_PCA_FAULT) {
1875 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
1878 switch (fault.type) {
1880 estr = "PCA_NO_FAULT"; break;
1881 case PCA_UNMAPPED_ADDR:
1882 estr = "PCA_UNMAPPED_ADDR"; break;
1883 case PCA_BAD_QACCESS_PF:
1884 estr = "PCA_BAD_QACCESS_PF"; break;
1885 case PCA_BAD_QACCESS_VF:
1886 estr = "PCA_BAD_QACCESS_VF"; break;
1887 case PCA_MALICIOUS_REQ:
1888 estr = "PCA_MALICIOUS_REQ"; break;
1889 case PCA_POISONED_TLP:
1890 estr = "PCA_POISONED_TLP"; break;
1892 estr = "PCA_TLP_ABORT"; break;
1896 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1897 estr, fault.func ? "VF" : "PF", fault.func,
1898 fault.address, fault.specinfo);
1901 /* Process THI fault */
1902 if (eicr & FM10K_EICR_THI_FAULT) {
1903 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
1906 switch (fault.type) {
1908 estr = "THI_NO_FAULT"; break;
1909 case THI_MAL_DIS_Q_FAULT:
1910 estr = "THI_MAL_DIS_Q_FAULT"; break;
1914 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1915 estr, fault.func ? "VF" : "PF", fault.func,
1916 fault.address, fault.specinfo);
1919 /* Process FUM fault */
1920 if (eicr & FM10K_EICR_FUM_FAULT) {
1921 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
1924 switch (fault.type) {
1926 estr = "FUM_NO_FAULT"; break;
1927 case FUM_UNMAPPED_ADDR:
1928 estr = "FUM_UNMAPPED_ADDR"; break;
1929 case FUM_POISONED_TLP:
1930 estr = "FUM_POISONED_TLP"; break;
1931 case FUM_BAD_VF_QACCESS:
1932 estr = "FUM_BAD_VF_QACCESS"; break;
1933 case FUM_ADD_DECODE_ERR:
1934 estr = "FUM_ADD_DECODE_ERR"; break;
1936 estr = "FUM_RO_ERROR"; break;
1937 case FUM_QPRC_CRC_ERROR:
1938 estr = "FUM_QPRC_CRC_ERROR"; break;
1939 case FUM_CSR_TIMEOUT:
1940 estr = "FUM_CSR_TIMEOUT"; break;
1941 case FUM_INVALID_TYPE:
1942 estr = "FUM_INVALID_TYPE"; break;
1943 case FUM_INVALID_LENGTH:
1944 estr = "FUM_INVALID_LENGTH"; break;
1945 case FUM_INVALID_BE:
1946 estr = "FUM_INVALID_BE"; break;
1947 case FUM_INVALID_ALIGN:
1948 estr = "FUM_INVALID_ALIGN"; break;
1952 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1953 estr, fault.func ? "VF" : "PF", fault.func,
1954 fault.address, fault.specinfo);
1959 PMD_INIT_LOG(ERR, "Failed to handle fault event.");
1964 * PF interrupt handler triggered by NIC for handling specific interrupt.
1967 * Pointer to interrupt handle.
1969 * The address of parameter (struct rte_eth_dev *) regsitered before.
1975 fm10k_dev_interrupt_handler_pf(
1976 __rte_unused struct rte_intr_handle *handle,
1979 struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1980 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1981 uint32_t cause, status;
1983 if (hw->mac.type != fm10k_mac_pf)
1986 cause = FM10K_READ_REG(hw, FM10K_EICR);
1988 /* Handle PCI fault cases */
1989 if (cause & FM10K_EICR_FAULT_MASK) {
1990 PMD_INIT_LOG(ERR, "INT: find fault!");
1991 fm10k_dev_handle_fault(hw, cause);
1994 /* Handle switch up/down */
1995 if (cause & FM10K_EICR_SWITCHNOTREADY)
1996 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
1998 if (cause & FM10K_EICR_SWITCHREADY)
1999 PMD_INIT_LOG(INFO, "INT: Switch is ready");
2001 /* Handle mailbox message */
2003 hw->mbx.ops.process(hw, &hw->mbx);
2004 fm10k_mbx_unlock(hw);
2006 /* Handle SRAM error */
2007 if (cause & FM10K_EICR_SRAMERROR) {
2008 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
2010 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
2011 /* Write to clear pending bits */
2012 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
2014 /* Todo: print out error message after shared code updates */
2017 /* Clear these 3 events if having any */
2018 cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
2019 FM10K_EICR_SWITCHREADY;
2021 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
2023 /* Re-enable interrupt from device side */
2024 FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
2025 FM10K_ITR_MASK_CLEAR);
2026 /* Re-enable interrupt from host side */
2027 rte_intr_enable(&(dev->pci_dev->intr_handle));
2031 * VF interrupt handler triggered by NIC for handling specific interrupt.
2034 * Pointer to interrupt handle.
2036 * The address of parameter (struct rte_eth_dev *) regsitered before.
2042 fm10k_dev_interrupt_handler_vf(
2043 __rte_unused struct rte_intr_handle *handle,
2046 struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2047 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2049 if (hw->mac.type != fm10k_mac_vf)
2052 /* Handle mailbox message if lock is acquired */
2054 hw->mbx.ops.process(hw, &hw->mbx);
2055 fm10k_mbx_unlock(hw);
2057 /* Re-enable interrupt from device side */
2058 FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2059 FM10K_ITR_MASK_CLEAR);
2060 /* Re-enable interrupt from host side */
2061 rte_intr_enable(&(dev->pci_dev->intr_handle));
2064 /* Mailbox message handler in VF */
2065 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2066 FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2067 FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2068 FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2069 FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2072 /* Mailbox message handler in PF */
2073 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
2074 FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
2075 FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
2076 FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
2077 FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
2078 FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
2079 FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
2080 FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2084 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2088 /* Initialize mailbox lock */
2089 fm10k_mbx_initlock(hw);
2091 /* Replace default message handler with new ones */
2092 if (hw->mac.type == fm10k_mac_pf)
2093 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2095 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2098 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2102 /* Connect to SM for PF device or PF for VF device */
2103 return hw->mbx.ops.connect(hw, &hw->mbx);
2107 fm10k_close_mbx_service(struct fm10k_hw *hw)
2109 /* Disconnect from SM for PF device or PF for VF device */
2110 hw->mbx.ops.disconnect(hw, &hw->mbx);
2113 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2114 .dev_configure = fm10k_dev_configure,
2115 .dev_start = fm10k_dev_start,
2116 .dev_stop = fm10k_dev_stop,
2117 .dev_close = fm10k_dev_close,
2118 .promiscuous_enable = fm10k_dev_promiscuous_enable,
2119 .promiscuous_disable = fm10k_dev_promiscuous_disable,
2120 .allmulticast_enable = fm10k_dev_allmulticast_enable,
2121 .allmulticast_disable = fm10k_dev_allmulticast_disable,
2122 .stats_get = fm10k_stats_get,
2123 .stats_reset = fm10k_stats_reset,
2124 .link_update = fm10k_link_update,
2125 .dev_infos_get = fm10k_dev_infos_get,
2126 .vlan_filter_set = fm10k_vlan_filter_set,
2127 .vlan_offload_set = fm10k_vlan_offload_set,
2128 .mac_addr_add = fm10k_macaddr_add,
2129 .mac_addr_remove = fm10k_macaddr_remove,
2130 .rx_queue_start = fm10k_dev_rx_queue_start,
2131 .rx_queue_stop = fm10k_dev_rx_queue_stop,
2132 .tx_queue_start = fm10k_dev_tx_queue_start,
2133 .tx_queue_stop = fm10k_dev_tx_queue_stop,
2134 .rx_queue_setup = fm10k_rx_queue_setup,
2135 .rx_queue_release = fm10k_rx_queue_release,
2136 .tx_queue_setup = fm10k_tx_queue_setup,
2137 .tx_queue_release = fm10k_tx_queue_release,
2138 .reta_update = fm10k_reta_update,
2139 .reta_query = fm10k_reta_query,
2140 .rss_hash_update = fm10k_rss_hash_update,
2141 .rss_hash_conf_get = fm10k_rss_hash_conf_get,
2145 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2147 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2149 struct fm10k_macvlan_filter_info *macvlan;
2151 PMD_INIT_FUNC_TRACE();
2153 dev->dev_ops = &fm10k_eth_dev_ops;
2154 dev->rx_pkt_burst = &fm10k_recv_pkts;
2155 dev->tx_pkt_burst = &fm10k_xmit_pkts;
2157 if (dev->data->scattered_rx)
2158 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
2160 /* only initialize in the primary process */
2161 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2164 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2165 memset(macvlan, 0, sizeof(*macvlan));
2166 /* Vendor and Device ID need to be set before init of shared code */
2167 memset(hw, 0, sizeof(*hw));
2168 hw->device_id = dev->pci_dev->id.device_id;
2169 hw->vendor_id = dev->pci_dev->id.vendor_id;
2170 hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2171 hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2172 hw->revision_id = 0;
2173 hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2174 if (hw->hw_addr == NULL) {
2175 PMD_INIT_LOG(ERR, "Bad mem resource."
2176 " Try to blacklist unused devices.");
2180 /* Store fm10k_adapter pointer */
2181 hw->back = dev->data->dev_private;
2183 /* Initialize the shared code */
2184 diag = fm10k_init_shared_code(hw);
2185 if (diag != FM10K_SUCCESS) {
2186 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2191 * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2192 * there is no way to get link status without reading BAR4. Until this
2193 * works, assume we have maximum bandwidth.
2194 * @todo - fix bus info
2196 hw->bus_caps.speed = fm10k_bus_speed_8000;
2197 hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2198 hw->bus_caps.payload = fm10k_bus_payload_512;
2199 hw->bus.speed = fm10k_bus_speed_8000;
2200 hw->bus.width = fm10k_bus_width_pcie_x8;
2201 hw->bus.payload = fm10k_bus_payload_256;
2203 /* Initialize the hw */
2204 diag = fm10k_init_hw(hw);
2205 if (diag != FM10K_SUCCESS) {
2206 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2210 /* Initialize MAC address(es) */
2211 dev->data->mac_addrs = rte_zmalloc("fm10k",
2212 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2213 if (dev->data->mac_addrs == NULL) {
2214 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2218 diag = fm10k_read_mac_addr(hw);
2220 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2221 &dev->data->mac_addrs[0]);
2223 if (diag != FM10K_SUCCESS ||
2224 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2226 /* Generate a random addr */
2227 eth_random_addr(hw->mac.addr);
2228 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2229 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2230 &dev->data->mac_addrs[0]);
2233 /* Reset the hw statistics */
2234 fm10k_stats_reset(dev);
2237 diag = fm10k_reset_hw(hw);
2238 if (diag != FM10K_SUCCESS) {
2239 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2243 /* Setup mailbox service */
2244 diag = fm10k_setup_mbx_service(hw);
2245 if (diag != FM10K_SUCCESS) {
2246 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2250 /*PF/VF has different interrupt handling mechanism */
2251 if (hw->mac.type == fm10k_mac_pf) {
2252 /* register callback func to eal lib */
2253 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2254 fm10k_dev_interrupt_handler_pf, (void *)dev);
2256 /* enable MISC interrupt */
2257 fm10k_dev_enable_intr_pf(dev);
2259 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2260 fm10k_dev_interrupt_handler_vf, (void *)dev);
2262 fm10k_dev_enable_intr_vf(dev);
2265 /* Enable uio intr after callback registered */
2266 rte_intr_enable(&(dev->pci_dev->intr_handle));
2268 hw->mac.ops.update_int_moderator(hw);
2270 /* Make sure Switch Manager is ready before going forward. */
2271 if (hw->mac.type == fm10k_mac_pf) {
2272 int switch_ready = 0;
2275 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2277 hw->mac.ops.get_host_state(hw, &switch_ready);
2278 fm10k_mbx_unlock(hw);
2281 /* Delay some time to acquire async LPORT_MAP info. */
2282 rte_delay_us(WAIT_SWITCH_MSG_US);
2285 if (switch_ready == 0) {
2286 PMD_INIT_LOG(ERR, "switch is not ready");
2292 * Below function will trigger operations on mailbox, acquire lock to
2293 * avoid race condition from interrupt handler. Operations on mailbox
2294 * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2295 * will handle and generate an interrupt to our side. Then, FIFO in
2296 * mailbox will be touched.
2299 /* Enable port first */
2300 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2302 /* Set unicast mode by default. App can change to other mode in other
2305 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2306 FM10K_XCAST_MODE_NONE);
2308 fm10k_mbx_unlock(hw);
2310 /* Add default mac address */
2311 fm10k_MAC_filter_set(dev, hw->mac.addr, true,
2312 MAIN_VSI_POOL_NUMBER);
2318 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2320 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2322 PMD_INIT_FUNC_TRACE();
2324 /* only uninitialize in the primary process */
2325 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2328 /* safe to close dev here */
2329 fm10k_dev_close(dev);
2331 dev->dev_ops = NULL;
2332 dev->rx_pkt_burst = NULL;
2333 dev->tx_pkt_burst = NULL;
2335 /* disable uio/vfio intr */
2336 rte_intr_disable(&(dev->pci_dev->intr_handle));
2338 /*PF/VF has different interrupt handling mechanism */
2339 if (hw->mac.type == fm10k_mac_pf) {
2340 /* disable interrupt */
2341 fm10k_dev_disable_intr_pf(dev);
2343 /* unregister callback func to eal lib */
2344 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2345 fm10k_dev_interrupt_handler_pf, (void *)dev);
2347 /* disable interrupt */
2348 fm10k_dev_disable_intr_vf(dev);
2350 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2351 fm10k_dev_interrupt_handler_vf, (void *)dev);
2354 /* free mac memory */
2355 if (dev->data->mac_addrs) {
2356 rte_free(dev->data->mac_addrs);
2357 dev->data->mac_addrs = NULL;
2360 memset(hw, 0, sizeof(*hw));
2366 * The set of PCI devices this driver supports. This driver will enable both PF
2367 * and SRIOV-VF devices.
2369 static const struct rte_pci_id pci_id_fm10k_map[] = {
2370 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2371 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2372 #include "rte_pci_dev_ids.h"
2373 { .vendor_id = 0, /* sentinel */ },
2376 static struct eth_driver rte_pmd_fm10k = {
2378 .name = "rte_pmd_fm10k",
2379 .id_table = pci_id_fm10k_map,
2380 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
2382 .eth_dev_init = eth_fm10k_dev_init,
2383 .eth_dev_uninit = eth_fm10k_dev_uninit,
2384 .dev_private_size = sizeof(struct fm10k_adapter),
2388 * Driver initialization routine.
2389 * Invoked once at EAL init time.
2390 * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2393 rte_pmd_fm10k_init(__rte_unused const char *name,
2394 __rte_unused const char *params)
2396 PMD_INIT_FUNC_TRACE();
2397 rte_eth_driver_register(&rte_pmd_fm10k);
2401 static struct rte_driver rte_fm10k_driver = {
2403 .init = rte_pmd_fm10k_init,
2406 PMD_REGISTER_DRIVER(rte_fm10k_driver);