4 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <rte_ethdev.h>
35 #include <rte_malloc.h>
36 #include <rte_memzone.h>
37 #include <rte_string_fns.h>
39 #include <rte_spinlock.h>
42 #include "base/fm10k_api.h"
44 /* Default delay to acquire mailbox lock */
45 #define FM10K_MBXLOCK_DELAY_US 20
46 #define UINT64_LOWER_32BITS_MASK 0x00000000ffffffffULL
48 /* Max try times to acquire switch status */
49 #define MAX_QUERY_SWITCH_STATE_TIMES 10
50 /* Wait interval to get switch status */
51 #define WAIT_SWITCH_MSG_US 100000
52 /* Number of chars per uint32 type */
53 #define CHARS_PER_UINT32 (sizeof(uint32_t))
54 #define BIT_MASK_PER_UINT32 ((1 << CHARS_PER_UINT32) - 1)
56 static void fm10k_close_mbx_service(struct fm10k_hw *hw);
57 static void fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev);
58 static void fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev);
59 static void fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev);
60 static void fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev);
61 static inline int fm10k_glort_valid(struct fm10k_hw *hw);
63 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
65 fm10k_MAC_filter_set(struct rte_eth_dev *dev, const u8 *mac, bool add);
67 fm10k_MACVLAN_remove_all(struct rte_eth_dev *dev);
68 static void fm10k_tx_queue_release(void *queue);
69 static void fm10k_rx_queue_release(void *queue);
72 fm10k_mbx_initlock(struct fm10k_hw *hw)
74 rte_spinlock_init(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
78 fm10k_mbx_lock(struct fm10k_hw *hw)
80 while (!rte_spinlock_trylock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back)))
81 rte_delay_us(FM10K_MBXLOCK_DELAY_US);
85 fm10k_mbx_unlock(struct fm10k_hw *hw)
87 rte_spinlock_unlock(FM10K_DEV_PRIVATE_TO_MBXLOCK(hw->back));
91 * reset queue to initial state, allocate software buffers used when starting
94 * return -ENOMEM if buffers cannot be allocated
95 * return -EINVAL if buffers do not satisfy alignment condition
98 rx_queue_reset(struct fm10k_rx_queue *q)
102 PMD_INIT_FUNC_TRACE();
104 diag = rte_mempool_get_bulk(q->mp, (void **)q->sw_ring, q->nb_desc);
108 for (i = 0; i < q->nb_desc; ++i) {
109 fm10k_pktmbuf_reset(q->sw_ring[i], q->port_id);
110 if (!fm10k_addr_alignment_valid(q->sw_ring[i])) {
111 rte_mempool_put_bulk(q->mp, (void **)q->sw_ring,
115 dma_addr = MBUF_DMA_ADDR_DEFAULT(q->sw_ring[i]);
116 q->hw_ring[i].q.pkt_addr = dma_addr;
117 q->hw_ring[i].q.hdr_addr = dma_addr;
122 q->next_trigger = q->alloc_thresh - 1;
123 FM10K_PCI_REG_WRITE(q->tail_ptr, q->nb_desc - 1);
128 * clean queue, descriptor rings, free software buffers used when stopping
132 rx_queue_clean(struct fm10k_rx_queue *q)
134 union fm10k_rx_desc zero = {.q = {0, 0, 0, 0} };
136 PMD_INIT_FUNC_TRACE();
138 /* zero descriptor rings */
139 for (i = 0; i < q->nb_desc; ++i)
140 q->hw_ring[i] = zero;
142 /* free software buffers */
143 for (i = 0; i < q->nb_desc; ++i) {
145 rte_pktmbuf_free_seg(q->sw_ring[i]);
146 q->sw_ring[i] = NULL;
152 * free all queue memory used when releasing the queue (i.e. configure)
155 rx_queue_free(struct fm10k_rx_queue *q)
157 PMD_INIT_FUNC_TRACE();
159 PMD_INIT_LOG(DEBUG, "Freeing rx queue %p", q);
162 rte_free(q->sw_ring);
171 * disable RX queue, wait unitl HW finished necessary flush operation
174 rx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
178 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
179 FM10K_WRITE_REG(hw, FM10K_RXQCTL(qnum),
180 reg & ~FM10K_RXQCTL_ENABLE);
182 /* Wait 100us at most */
183 for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
185 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(qnum));
186 if (!(reg & FM10K_RXQCTL_ENABLE))
190 if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
197 * reset queue to initial state, allocate software buffers used when starting
201 tx_queue_reset(struct fm10k_tx_queue *q)
203 PMD_INIT_FUNC_TRACE();
207 q->nb_free = q->nb_desc - 1;
208 fifo_reset(&q->rs_tracker, (q->nb_desc + 1) / q->rs_thresh);
209 FM10K_PCI_REG_WRITE(q->tail_ptr, 0);
213 * clean queue, descriptor rings, free software buffers used when stopping
217 tx_queue_clean(struct fm10k_tx_queue *q)
219 struct fm10k_tx_desc zero = {0, 0, 0, 0, 0, 0};
221 PMD_INIT_FUNC_TRACE();
223 /* zero descriptor rings */
224 for (i = 0; i < q->nb_desc; ++i)
225 q->hw_ring[i] = zero;
227 /* free software buffers */
228 for (i = 0; i < q->nb_desc; ++i) {
230 rte_pktmbuf_free_seg(q->sw_ring[i]);
231 q->sw_ring[i] = NULL;
237 * free all queue memory used when releasing the queue (i.e. configure)
240 tx_queue_free(struct fm10k_tx_queue *q)
242 PMD_INIT_FUNC_TRACE();
244 PMD_INIT_LOG(DEBUG, "Freeing tx queue %p", q);
246 if (q->rs_tracker.list) {
247 rte_free(q->rs_tracker.list);
248 q->rs_tracker.list = NULL;
251 rte_free(q->sw_ring);
260 * disable TX queue, wait unitl HW finished necessary flush operation
263 tx_queue_disable(struct fm10k_hw *hw, uint16_t qnum)
267 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
268 FM10K_WRITE_REG(hw, FM10K_TXDCTL(qnum),
269 reg & ~FM10K_TXDCTL_ENABLE);
271 /* Wait 100us at most */
272 for (i = 0; i < FM10K_QUEUE_DISABLE_TIMEOUT; i++) {
274 reg = FM10K_READ_REG(hw, FM10K_TXDCTL(qnum));
275 if (!(reg & FM10K_TXDCTL_ENABLE))
279 if (i == FM10K_QUEUE_DISABLE_TIMEOUT)
286 fm10k_check_mq_mode(struct rte_eth_dev *dev)
288 enum rte_eth_rx_mq_mode rx_mq_mode = dev->data->dev_conf.rxmode.mq_mode;
289 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
290 struct rte_eth_vmdq_rx_conf *vmdq_conf;
291 uint16_t nb_rx_q = dev->data->nb_rx_queues;
293 vmdq_conf = &dev->data->dev_conf.rx_adv_conf.vmdq_rx_conf;
295 if (rx_mq_mode & ETH_MQ_RX_DCB_FLAG) {
296 PMD_INIT_LOG(ERR, "DCB mode is not supported.");
300 if (!(rx_mq_mode & ETH_MQ_RX_VMDQ_FLAG))
303 if (hw->mac.type == fm10k_mac_vf) {
304 PMD_INIT_LOG(ERR, "VMDQ mode is not supported in VF.");
308 /* Check VMDQ queue pool number */
309 if (vmdq_conf->nb_queue_pools >
310 sizeof(vmdq_conf->pool_map[0].pools) * CHAR_BIT ||
311 vmdq_conf->nb_queue_pools > nb_rx_q) {
312 PMD_INIT_LOG(ERR, "Too many of queue pools: %d",
313 vmdq_conf->nb_queue_pools);
321 fm10k_dev_configure(struct rte_eth_dev *dev)
325 PMD_INIT_FUNC_TRACE();
327 if (dev->data->dev_conf.rxmode.hw_strip_crc == 0)
328 PMD_INIT_LOG(WARNING, "fm10k always strip CRC");
329 /* multipe queue mode checking */
330 ret = fm10k_check_mq_mode(dev);
332 PMD_DRV_LOG(ERR, "fm10k_check_mq_mode fails with %d.",
341 fm10k_dev_mq_rx_configure(struct rte_eth_dev *dev)
343 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
344 struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
345 uint32_t mrqc, *key, i, reta, j;
348 #define RSS_KEY_SIZE 40
349 static uint8_t rss_intel_key[RSS_KEY_SIZE] = {
350 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
351 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
352 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
353 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
354 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
357 if (dev->data->nb_rx_queues == 1 ||
358 dev_conf->rxmode.mq_mode != ETH_MQ_RX_RSS ||
359 dev_conf->rx_adv_conf.rss_conf.rss_hf == 0)
362 /* random key is rss_intel_key (default) or user provided (rss_key) */
363 if (dev_conf->rx_adv_conf.rss_conf.rss_key == NULL)
364 key = (uint32_t *)rss_intel_key;
366 key = (uint32_t *)dev_conf->rx_adv_conf.rss_conf.rss_key;
368 /* Now fill our hash function seeds, 4 bytes at a time */
369 for (i = 0; i < RSS_KEY_SIZE / sizeof(*key); ++i)
370 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
373 * Fill in redirection table
374 * The byte-swap is needed because NIC registers are in
375 * little-endian order.
378 for (i = 0, j = 0; i < FM10K_MAX_RSS_INDICES; i++, j++) {
379 if (j == dev->data->nb_rx_queues)
381 reta = (reta << CHAR_BIT) | j;
383 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2),
388 * Generate RSS hash based on packet types, TCP/UDP
389 * port numbers and/or IPv4/v6 src and dst addresses
391 hf = dev_conf->rx_adv_conf.rss_conf.rss_hf;
393 mrqc |= (hf & ETH_RSS_IPV4) ? FM10K_MRQC_IPV4 : 0;
394 mrqc |= (hf & ETH_RSS_IPV6) ? FM10K_MRQC_IPV6 : 0;
395 mrqc |= (hf & ETH_RSS_IPV6_EX) ? FM10K_MRQC_IPV6 : 0;
396 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP) ? FM10K_MRQC_TCP_IPV4 : 0;
397 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP) ? FM10K_MRQC_TCP_IPV6 : 0;
398 mrqc |= (hf & ETH_RSS_IPV6_TCP_EX) ? FM10K_MRQC_TCP_IPV6 : 0;
399 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP) ? FM10K_MRQC_UDP_IPV4 : 0;
400 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP) ? FM10K_MRQC_UDP_IPV6 : 0;
401 mrqc |= (hf & ETH_RSS_IPV6_UDP_EX) ? FM10K_MRQC_UDP_IPV6 : 0;
404 PMD_INIT_LOG(ERR, "Specified RSS mode 0x%"PRIx64"is not"
409 FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
413 fm10k_dev_tx_init(struct rte_eth_dev *dev)
415 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
417 struct fm10k_tx_queue *txq;
421 /* Disable TXINT to avoid possible interrupt */
422 for (i = 0; i < hw->mac.max_queues; i++)
423 FM10K_WRITE_REG(hw, FM10K_TXINT(i),
424 3 << FM10K_TXINT_TIMER_SHIFT);
427 for (i = 0; i < dev->data->nb_tx_queues; ++i) {
428 txq = dev->data->tx_queues[i];
429 base_addr = txq->hw_ring_phys_addr;
430 size = txq->nb_desc * sizeof(struct fm10k_tx_desc);
432 /* disable queue to avoid issues while updating state */
433 ret = tx_queue_disable(hw, i);
435 PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
439 /* set location and size for descriptor ring */
440 FM10K_WRITE_REG(hw, FM10K_TDBAL(i),
441 base_addr & UINT64_LOWER_32BITS_MASK);
442 FM10K_WRITE_REG(hw, FM10K_TDBAH(i),
443 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
444 FM10K_WRITE_REG(hw, FM10K_TDLEN(i), size);
450 fm10k_dev_rx_init(struct rte_eth_dev *dev)
452 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
454 struct fm10k_rx_queue *rxq;
457 uint32_t rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
460 /* Disable RXINT to avoid possible interrupt */
461 for (i = 0; i < hw->mac.max_queues; i++)
462 FM10K_WRITE_REG(hw, FM10K_RXINT(i),
463 3 << FM10K_RXINT_TIMER_SHIFT);
465 /* Setup RX queues */
466 for (i = 0; i < dev->data->nb_rx_queues; ++i) {
467 rxq = dev->data->rx_queues[i];
468 base_addr = rxq->hw_ring_phys_addr;
469 size = rxq->nb_desc * sizeof(union fm10k_rx_desc);
471 /* disable queue to avoid issues while updating state */
472 ret = rx_queue_disable(hw, i);
474 PMD_INIT_LOG(ERR, "failed to disable queue %d", i);
478 /* Setup the Base and Length of the Rx Descriptor Ring */
479 FM10K_WRITE_REG(hw, FM10K_RDBAL(i),
480 base_addr & UINT64_LOWER_32BITS_MASK);
481 FM10K_WRITE_REG(hw, FM10K_RDBAH(i),
482 base_addr >> (CHAR_BIT * sizeof(uint32_t)));
483 FM10K_WRITE_REG(hw, FM10K_RDLEN(i), size);
485 /* Configure the Rx buffer size for one buff without split */
486 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
487 RTE_PKTMBUF_HEADROOM);
488 /* As RX buffer is aligned to 512B within mbuf, some bytes are
489 * reserved for this purpose, and the worst case could be 511B.
490 * But SRR reg assumes all buffers have the same size. In order
491 * to fill the gap, we'll have to consider the worst case and
492 * assume 512B is reserved. If we don't do so, it's possible
493 * for HW to overwrite data to next mbuf.
495 buf_size -= FM10K_RX_DATABUF_ALIGN;
497 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i),
498 buf_size >> FM10K_SRRCTL_BSIZEPKT_SHIFT);
500 /* It adds dual VLAN length for supporting dual VLAN */
501 if ((dev->data->dev_conf.rxmode.max_rx_pkt_len +
502 2 * FM10K_VLAN_TAG_SIZE) > buf_size ||
503 dev->data->dev_conf.rxmode.enable_scatter) {
505 dev->data->scattered_rx = 1;
506 dev->rx_pkt_burst = fm10k_recv_scattered_pkts;
507 reg = FM10K_READ_REG(hw, FM10K_SRRCTL(i));
508 reg |= FM10K_SRRCTL_BUFFER_CHAINING_EN;
509 FM10K_WRITE_REG(hw, FM10K_SRRCTL(i), reg);
512 /* Enable drop on empty, it's RO for VF */
513 if (hw->mac.type == fm10k_mac_pf && rxq->drop_en)
514 rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
516 FM10K_WRITE_REG(hw, FM10K_RXDCTL(i), rxdctl);
517 FM10K_WRITE_FLUSH(hw);
520 /* Configure RSS if applicable */
521 fm10k_dev_mq_rx_configure(dev);
526 fm10k_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
528 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
531 struct fm10k_rx_queue *rxq;
533 PMD_INIT_FUNC_TRACE();
535 if (rx_queue_id < dev->data->nb_rx_queues) {
536 rxq = dev->data->rx_queues[rx_queue_id];
537 err = rx_queue_reset(rxq);
538 if (err == -ENOMEM) {
539 PMD_INIT_LOG(ERR, "Failed to alloc memory : %d", err);
541 } else if (err == -EINVAL) {
542 PMD_INIT_LOG(ERR, "Invalid buffer address alignment :"
547 /* Setup the HW Rx Head and Tail Descriptor Pointers
548 * Note: this must be done AFTER the queue is enabled on real
549 * hardware, but BEFORE the queue is enabled when using the
550 * emulation platform. Do it in both places for now and remove
551 * this comment and the following two register writes when the
552 * emulation platform is no longer being used.
554 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
555 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
557 /* Set PF ownership flag for PF devices */
558 reg = FM10K_READ_REG(hw, FM10K_RXQCTL(rx_queue_id));
559 if (hw->mac.type == fm10k_mac_pf)
560 reg |= FM10K_RXQCTL_PF;
561 reg |= FM10K_RXQCTL_ENABLE;
562 /* enable RX queue */
563 FM10K_WRITE_REG(hw, FM10K_RXQCTL(rx_queue_id), reg);
564 FM10K_WRITE_FLUSH(hw);
566 /* Setup the HW Rx Head and Tail Descriptor Pointers
567 * Note: this must be done AFTER the queue is enabled
569 FM10K_WRITE_REG(hw, FM10K_RDH(rx_queue_id), 0);
570 FM10K_WRITE_REG(hw, FM10K_RDT(rx_queue_id), rxq->nb_desc - 1);
577 fm10k_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
579 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
581 PMD_INIT_FUNC_TRACE();
583 if (rx_queue_id < dev->data->nb_rx_queues) {
584 /* Disable RX queue */
585 rx_queue_disable(hw, rx_queue_id);
587 /* Free mbuf and clean HW ring */
588 rx_queue_clean(dev->data->rx_queues[rx_queue_id]);
595 fm10k_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
597 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
598 /** @todo - this should be defined in the shared code */
599 #define FM10K_TXDCTL_WRITE_BACK_MIN_DELAY 0x00010000
600 uint32_t txdctl = FM10K_TXDCTL_WRITE_BACK_MIN_DELAY;
603 PMD_INIT_FUNC_TRACE();
605 if (tx_queue_id < dev->data->nb_tx_queues) {
606 tx_queue_reset(dev->data->tx_queues[tx_queue_id]);
608 /* reset head and tail pointers */
609 FM10K_WRITE_REG(hw, FM10K_TDH(tx_queue_id), 0);
610 FM10K_WRITE_REG(hw, FM10K_TDT(tx_queue_id), 0);
612 /* enable TX queue */
613 FM10K_WRITE_REG(hw, FM10K_TXDCTL(tx_queue_id),
614 FM10K_TXDCTL_ENABLE | txdctl);
615 FM10K_WRITE_FLUSH(hw);
623 fm10k_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
625 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
627 PMD_INIT_FUNC_TRACE();
629 if (tx_queue_id < dev->data->nb_tx_queues) {
630 tx_queue_disable(hw, tx_queue_id);
631 tx_queue_clean(dev->data->tx_queues[tx_queue_id]);
637 static inline int fm10k_glort_valid(struct fm10k_hw *hw)
639 return ((hw->mac.dglort_map & FM10K_DGLORTMAP_NONE)
640 != FM10K_DGLORTMAP_NONE);
644 fm10k_dev_promiscuous_enable(struct rte_eth_dev *dev)
646 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
649 PMD_INIT_FUNC_TRACE();
651 /* Return if it didn't acquire valid glort range */
652 if (!fm10k_glort_valid(hw))
656 status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
657 FM10K_XCAST_MODE_PROMISC);
658 fm10k_mbx_unlock(hw);
660 if (status != FM10K_SUCCESS)
661 PMD_INIT_LOG(ERR, "Failed to enable promiscuous mode");
665 fm10k_dev_promiscuous_disable(struct rte_eth_dev *dev)
667 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
671 PMD_INIT_FUNC_TRACE();
673 /* Return if it didn't acquire valid glort range */
674 if (!fm10k_glort_valid(hw))
677 if (dev->data->all_multicast == 1)
678 mode = FM10K_XCAST_MODE_ALLMULTI;
680 mode = FM10K_XCAST_MODE_NONE;
683 status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
685 fm10k_mbx_unlock(hw);
687 if (status != FM10K_SUCCESS)
688 PMD_INIT_LOG(ERR, "Failed to disable promiscuous mode");
692 fm10k_dev_allmulticast_enable(struct rte_eth_dev *dev)
694 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
697 PMD_INIT_FUNC_TRACE();
699 /* Return if it didn't acquire valid glort range */
700 if (!fm10k_glort_valid(hw))
703 /* If promiscuous mode is enabled, it doesn't make sense to enable
704 * allmulticast and disable promiscuous since fm10k only can select
707 if (dev->data->promiscuous) {
708 PMD_INIT_LOG(INFO, "Promiscuous mode is enabled, "\
709 "needn't enable allmulticast");
714 status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
715 FM10K_XCAST_MODE_ALLMULTI);
716 fm10k_mbx_unlock(hw);
718 if (status != FM10K_SUCCESS)
719 PMD_INIT_LOG(ERR, "Failed to enable allmulticast mode");
723 fm10k_dev_allmulticast_disable(struct rte_eth_dev *dev)
725 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
728 PMD_INIT_FUNC_TRACE();
730 /* Return if it didn't acquire valid glort range */
731 if (!fm10k_glort_valid(hw))
734 if (dev->data->promiscuous) {
735 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode "\
736 "since promisc mode is enabled");
741 /* Change mode to unicast mode */
742 status = hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
743 FM10K_XCAST_MODE_NONE);
744 fm10k_mbx_unlock(hw);
746 if (status != FM10K_SUCCESS)
747 PMD_INIT_LOG(ERR, "Failed to disable allmulticast mode");
750 /* fls = find last set bit = 32 minus the number of leading zeros */
752 #define fls(x) (((x) == 0) ? 0 : (32 - __builtin_clz((x))))
754 #define BSIZEPKT_ROUNDUP ((1 << FM10K_SRRCTL_BSIZEPKT_SHIFT) - 1)
756 fm10k_dev_start(struct rte_eth_dev *dev)
758 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
761 PMD_INIT_FUNC_TRACE();
763 /* stop, init, then start the hw */
764 diag = fm10k_stop_hw(hw);
765 if (diag != FM10K_SUCCESS) {
766 PMD_INIT_LOG(ERR, "Hardware stop failed: %d", diag);
770 diag = fm10k_init_hw(hw);
771 if (diag != FM10K_SUCCESS) {
772 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
776 diag = fm10k_start_hw(hw);
777 if (diag != FM10K_SUCCESS) {
778 PMD_INIT_LOG(ERR, "Hardware start failed: %d", diag);
782 diag = fm10k_dev_tx_init(dev);
784 PMD_INIT_LOG(ERR, "TX init failed: %d", diag);
788 diag = fm10k_dev_rx_init(dev);
790 PMD_INIT_LOG(ERR, "RX init failed: %d", diag);
794 if (hw->mac.type == fm10k_mac_pf) {
795 /* Establish only VSI 0 as valid */
796 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(0), FM10K_DGLORTMAP_ANY);
798 /* Configure RSS bits used in RETA table */
799 FM10K_WRITE_REG(hw, FM10K_DGLORTDEC(0),
800 fls(dev->data->nb_rx_queues - 1) <<
801 FM10K_DGLORTDEC_RSSLENGTH_SHIFT);
803 /* Invalidate all other GLORT entries */
804 for (i = 1; i < FM10K_DGLORT_COUNT; i++)
805 FM10K_WRITE_REG(hw, FM10K_DGLORTMAP(i),
806 FM10K_DGLORTMAP_NONE);
809 for (i = 0; i < dev->data->nb_rx_queues; i++) {
810 struct fm10k_rx_queue *rxq;
811 rxq = dev->data->rx_queues[i];
813 if (rxq->rx_deferred_start)
815 diag = fm10k_dev_rx_queue_start(dev, i);
818 for (j = 0; j < i; ++j)
819 rx_queue_clean(dev->data->rx_queues[j]);
824 for (i = 0; i < dev->data->nb_tx_queues; i++) {
825 struct fm10k_tx_queue *txq;
826 txq = dev->data->tx_queues[i];
828 if (txq->tx_deferred_start)
830 diag = fm10k_dev_tx_queue_start(dev, i);
833 for (j = 0; j < i; ++j)
834 tx_queue_clean(dev->data->tx_queues[j]);
835 for (j = 0; j < dev->data->nb_rx_queues; ++j)
836 rx_queue_clean(dev->data->rx_queues[j]);
841 /* Update default vlan */
842 if (hw->mac.default_vid && hw->mac.default_vid <= ETHER_MAX_VLAN_ID)
843 fm10k_vlan_filter_set(dev, hw->mac.default_vid, true);
849 fm10k_dev_stop(struct rte_eth_dev *dev)
853 PMD_INIT_FUNC_TRACE();
855 if (dev->data->tx_queues)
856 for (i = 0; i < dev->data->nb_tx_queues; i++)
857 fm10k_dev_tx_queue_stop(dev, i);
859 if (dev->data->rx_queues)
860 for (i = 0; i < dev->data->nb_rx_queues; i++)
861 fm10k_dev_rx_queue_stop(dev, i);
865 fm10k_dev_queue_release(struct rte_eth_dev *dev)
869 PMD_INIT_FUNC_TRACE();
871 if (dev->data->tx_queues) {
872 for (i = 0; i < dev->data->nb_tx_queues; i++)
873 fm10k_tx_queue_release(dev->data->tx_queues[i]);
876 if (dev->data->rx_queues) {
877 for (i = 0; i < dev->data->nb_rx_queues; i++)
878 fm10k_rx_queue_release(dev->data->rx_queues[i]);
883 fm10k_dev_close(struct rte_eth_dev *dev)
885 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
887 PMD_INIT_FUNC_TRACE();
889 fm10k_MACVLAN_remove_all(dev);
891 /* Stop mailbox service first */
892 fm10k_close_mbx_service(hw);
894 fm10k_dev_queue_release(dev);
899 fm10k_link_update(struct rte_eth_dev *dev,
900 __rte_unused int wait_to_complete)
902 PMD_INIT_FUNC_TRACE();
904 /* The host-interface link is always up. The speed is ~50Gbps per Gen3
905 * x8 PCIe interface. For now, we leave the speed undefined since there
906 * is no 50Gbps Ethernet. */
907 dev->data->dev_link.link_speed = 0;
908 dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
909 dev->data->dev_link.link_status = 1;
915 fm10k_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
917 uint64_t ipackets, opackets, ibytes, obytes;
918 struct fm10k_hw *hw =
919 FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
920 struct fm10k_hw_stats *hw_stats =
921 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
924 PMD_INIT_FUNC_TRACE();
926 fm10k_update_hw_stats(hw, hw_stats);
928 ipackets = opackets = ibytes = obytes = 0;
929 for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
930 (i < hw->mac.max_queues); ++i) {
931 stats->q_ipackets[i] = hw_stats->q[i].rx_packets.count;
932 stats->q_opackets[i] = hw_stats->q[i].tx_packets.count;
933 stats->q_ibytes[i] = hw_stats->q[i].rx_bytes.count;
934 stats->q_obytes[i] = hw_stats->q[i].tx_bytes.count;
935 ipackets += stats->q_ipackets[i];
936 opackets += stats->q_opackets[i];
937 ibytes += stats->q_ibytes[i];
938 obytes += stats->q_obytes[i];
940 stats->ipackets = ipackets;
941 stats->opackets = opackets;
942 stats->ibytes = ibytes;
943 stats->obytes = obytes;
947 fm10k_stats_reset(struct rte_eth_dev *dev)
949 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
950 struct fm10k_hw_stats *hw_stats =
951 FM10K_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
953 PMD_INIT_FUNC_TRACE();
955 memset(hw_stats, 0, sizeof(*hw_stats));
956 fm10k_rebind_hw_stats(hw, hw_stats);
960 fm10k_dev_infos_get(struct rte_eth_dev *dev,
961 struct rte_eth_dev_info *dev_info)
963 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
965 PMD_INIT_FUNC_TRACE();
967 dev_info->min_rx_bufsize = FM10K_MIN_RX_BUF_SIZE;
968 dev_info->max_rx_pktlen = FM10K_MAX_PKT_SIZE;
969 dev_info->max_rx_queues = hw->mac.max_queues;
970 dev_info->max_tx_queues = hw->mac.max_queues;
971 dev_info->max_mac_addrs = FM10K_MAX_MACADDR_NUM;
972 dev_info->max_hash_mac_addrs = 0;
973 dev_info->max_vfs = dev->pci_dev->max_vfs;
974 dev_info->max_vmdq_pools = ETH_64_POOLS;
975 dev_info->rx_offload_capa =
976 DEV_RX_OFFLOAD_VLAN_STRIP |
977 DEV_RX_OFFLOAD_IPV4_CKSUM |
978 DEV_RX_OFFLOAD_UDP_CKSUM |
979 DEV_RX_OFFLOAD_TCP_CKSUM;
980 dev_info->tx_offload_capa =
981 DEV_TX_OFFLOAD_VLAN_INSERT |
982 DEV_TX_OFFLOAD_IPV4_CKSUM |
983 DEV_TX_OFFLOAD_UDP_CKSUM |
984 DEV_TX_OFFLOAD_TCP_CKSUM |
985 DEV_TX_OFFLOAD_TCP_TSO;
987 dev_info->hash_key_size = FM10K_RSSRK_SIZE * sizeof(uint32_t);
988 dev_info->reta_size = FM10K_MAX_RSS_INDICES;
990 dev_info->default_rxconf = (struct rte_eth_rxconf) {
992 .pthresh = FM10K_DEFAULT_RX_PTHRESH,
993 .hthresh = FM10K_DEFAULT_RX_HTHRESH,
994 .wthresh = FM10K_DEFAULT_RX_WTHRESH,
996 .rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(0),
1000 dev_info->default_txconf = (struct rte_eth_txconf) {
1002 .pthresh = FM10K_DEFAULT_TX_PTHRESH,
1003 .hthresh = FM10K_DEFAULT_TX_HTHRESH,
1004 .wthresh = FM10K_DEFAULT_TX_WTHRESH,
1006 .tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(0),
1007 .tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(0),
1008 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
1009 ETH_TXQ_FLAGS_NOOFFLOADS,
1015 fm10k_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1018 uint16_t mac_num = 0;
1019 uint32_t vid_idx, vid_bit, mac_index;
1020 struct fm10k_hw *hw;
1021 struct fm10k_macvlan_filter_info *macvlan;
1022 struct rte_eth_dev_data *data = dev->data;
1024 hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1025 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1027 if (vlan_id > ETH_VLAN_ID_MAX) {
1028 PMD_INIT_LOG(ERR, "Invalid vlan_id: must be < 4096");
1032 vid_idx = FM10K_VFTA_IDX(vlan_id);
1033 vid_bit = FM10K_VFTA_BIT(vlan_id);
1034 /* this VLAN ID is already in the VLAN filter table, return SUCCESS */
1035 if (on && (macvlan->vfta[vid_idx] & vid_bit))
1037 /* this VLAN ID is NOT in the VLAN filter table, cannot remove */
1038 if (!on && !(macvlan->vfta[vid_idx] & vid_bit)) {
1039 PMD_INIT_LOG(ERR, "Invalid vlan_id: not existing "
1040 "in the VLAN filter table");
1045 result = fm10k_update_vlan(hw, vlan_id, 0, on);
1046 fm10k_mbx_unlock(hw);
1047 if (result != FM10K_SUCCESS) {
1048 PMD_INIT_LOG(ERR, "VLAN update failed: %d", result);
1052 for (mac_index = 0; (mac_index < FM10K_MAX_MACADDR_NUM) &&
1053 (result == FM10K_SUCCESS); mac_index++) {
1054 if (is_zero_ether_addr(&data->mac_addrs[mac_index]))
1056 if (mac_num > macvlan->mac_num - 1) {
1057 PMD_INIT_LOG(ERR, "MAC address number "
1062 result = fm10k_update_uc_addr(hw, hw->mac.dglort_map,
1063 data->mac_addrs[mac_index].addr_bytes,
1065 fm10k_mbx_unlock(hw);
1068 if (result != FM10K_SUCCESS) {
1069 PMD_INIT_LOG(ERR, "MAC address update failed: %d", result);
1074 macvlan->vlan_num++;
1075 macvlan->vfta[vid_idx] |= vid_bit;
1077 macvlan->vlan_num--;
1078 macvlan->vfta[vid_idx] &= ~vid_bit;
1084 fm10k_vlan_offload_set(__rte_unused struct rte_eth_dev *dev, int mask)
1086 if (mask & ETH_VLAN_STRIP_MASK) {
1087 if (!dev->data->dev_conf.rxmode.hw_vlan_strip)
1088 PMD_INIT_LOG(ERR, "VLAN stripping is "
1089 "always on in fm10k");
1092 if (mask & ETH_VLAN_EXTEND_MASK) {
1093 if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1094 PMD_INIT_LOG(ERR, "VLAN QinQ is not "
1095 "supported in fm10k");
1098 if (mask & ETH_VLAN_FILTER_MASK) {
1099 if (!dev->data->dev_conf.rxmode.hw_vlan_filter)
1100 PMD_INIT_LOG(ERR, "VLAN filter is always on in fm10k");
1104 /* Add/Remove a MAC address, and update filters */
1106 fm10k_MAC_filter_set(struct rte_eth_dev *dev, const u8 *mac, bool add)
1109 struct fm10k_hw *hw;
1110 struct fm10k_macvlan_filter_info *macvlan;
1112 hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1113 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1116 for (j = 0; j < FM10K_VFTA_SIZE; j++) {
1117 if (macvlan->vfta[j]) {
1118 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1119 if (macvlan->vfta[j] & (1 << k)) {
1120 if (i + 1 > macvlan->vlan_num) {
1121 PMD_INIT_LOG(ERR, "vlan number "
1126 fm10k_update_uc_addr(hw,
1127 hw->mac.dglort_map, mac,
1128 j * FM10K_UINT32_BIT_SIZE + k,
1130 fm10k_mbx_unlock(hw);
1143 /* Add a MAC address, and update filters */
1145 fm10k_macaddr_add(struct rte_eth_dev *dev,
1146 struct ether_addr *mac_addr,
1147 __rte_unused uint32_t index,
1148 __rte_unused uint32_t pool)
1150 fm10k_MAC_filter_set(dev, mac_addr->addr_bytes, TRUE);
1153 /* Remove a MAC address, and update filters */
1155 fm10k_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
1157 struct rte_eth_dev_data *data = dev->data;
1159 if (index < FM10K_MAX_MACADDR_NUM)
1160 fm10k_MAC_filter_set(dev, data->mac_addrs[index].addr_bytes,
1164 /* Remove all VLAN and MAC address table entries */
1166 fm10k_MACVLAN_remove_all(struct rte_eth_dev *dev)
1169 struct fm10k_macvlan_filter_info *macvlan;
1171 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
1172 for (j = 0; j < FM10K_VFTA_SIZE; j++) {
1173 if (macvlan->vfta[j]) {
1174 for (k = 0; k < FM10K_UINT32_BIT_SIZE; k++) {
1175 if (macvlan->vfta[j] & (1 << k))
1176 fm10k_vlan_filter_set(dev,
1177 j * FM10K_UINT32_BIT_SIZE + k, false);
1184 check_nb_desc(uint16_t min, uint16_t max, uint16_t mult, uint16_t request)
1186 if ((request < min) || (request > max) || ((request % mult) != 0))
1193 * Create a memzone for hardware descriptor rings. Malloc cannot be used since
1194 * the physical address is required. If the memzone is already created, then
1195 * this function returns a pointer to the existing memzone.
1197 static inline const struct rte_memzone *
1198 allocate_hw_ring(const char *driver_name, const char *ring_name,
1199 uint8_t port_id, uint16_t queue_id, int socket_id,
1200 uint32_t size, uint32_t align)
1202 char name[RTE_MEMZONE_NAMESIZE];
1203 const struct rte_memzone *mz;
1205 snprintf(name, sizeof(name), "%s_%s_%d_%d_%d",
1206 driver_name, ring_name, port_id, queue_id, socket_id);
1208 /* return the memzone if it already exists */
1209 mz = rte_memzone_lookup(name);
1213 #ifdef RTE_LIBRTE_XEN_DOM0
1214 return rte_memzone_reserve_bounded(name, size, socket_id, 0, align,
1217 return rte_memzone_reserve_aligned(name, size, socket_id, 0, align);
1222 check_thresh(uint16_t min, uint16_t max, uint16_t div, uint16_t request)
1224 if ((request < min) || (request > max) || ((div % request) != 0))
1231 handle_rxconf(struct fm10k_rx_queue *q, const struct rte_eth_rxconf *conf)
1233 uint16_t rx_free_thresh;
1235 if (conf->rx_free_thresh == 0)
1236 rx_free_thresh = FM10K_RX_FREE_THRESH_DEFAULT(q);
1238 rx_free_thresh = conf->rx_free_thresh;
1240 /* make sure the requested threshold satisfies the constraints */
1241 if (check_thresh(FM10K_RX_FREE_THRESH_MIN(q),
1242 FM10K_RX_FREE_THRESH_MAX(q),
1243 FM10K_RX_FREE_THRESH_DIV(q),
1245 PMD_INIT_LOG(ERR, "rx_free_thresh (%u) must be "
1246 "less than or equal to %u, "
1247 "greater than or equal to %u, "
1248 "and a divisor of %u",
1249 rx_free_thresh, FM10K_RX_FREE_THRESH_MAX(q),
1250 FM10K_RX_FREE_THRESH_MIN(q),
1251 FM10K_RX_FREE_THRESH_DIV(q));
1255 q->alloc_thresh = rx_free_thresh;
1256 q->drop_en = conf->rx_drop_en;
1257 q->rx_deferred_start = conf->rx_deferred_start;
1263 * Hardware requires specific alignment for Rx packet buffers. At
1264 * least one of the following two conditions must be satisfied.
1265 * 1. Address is 512B aligned
1266 * 2. Address is 8B aligned and buffer does not cross 4K boundary.
1268 * As such, the driver may need to adjust the DMA address within the
1269 * buffer by up to 512B.
1271 * return 1 if the element size is valid, otherwise return 0.
1274 mempool_element_size_valid(struct rte_mempool *mp)
1278 /* elt_size includes mbuf header and headroom */
1279 min_size = mp->elt_size - sizeof(struct rte_mbuf) -
1280 RTE_PKTMBUF_HEADROOM;
1282 /* account for up to 512B of alignment */
1283 min_size -= FM10K_RX_DATABUF_ALIGN;
1285 /* sanity check for overflow */
1286 if (min_size > mp->elt_size)
1294 fm10k_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1295 uint16_t nb_desc, unsigned int socket_id,
1296 const struct rte_eth_rxconf *conf, struct rte_mempool *mp)
1298 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1299 struct fm10k_rx_queue *q;
1300 const struct rte_memzone *mz;
1302 PMD_INIT_FUNC_TRACE();
1304 /* make sure the mempool element size can account for alignment. */
1305 if (!mempool_element_size_valid(mp)) {
1306 PMD_INIT_LOG(ERR, "Error : Mempool element size is too small");
1310 /* make sure a valid number of descriptors have been requested */
1311 if (check_nb_desc(FM10K_MIN_RX_DESC, FM10K_MAX_RX_DESC,
1312 FM10K_MULT_RX_DESC, nb_desc)) {
1313 PMD_INIT_LOG(ERR, "Number of Rx descriptors (%u) must be "
1314 "less than or equal to %"PRIu32", "
1315 "greater than or equal to %u, "
1316 "and a multiple of %u",
1317 nb_desc, (uint32_t)FM10K_MAX_RX_DESC, FM10K_MIN_RX_DESC,
1318 FM10K_MULT_RX_DESC);
1323 * if this queue existed already, free the associated memory. The
1324 * queue cannot be reused in case we need to allocate memory on
1325 * different socket than was previously used.
1327 if (dev->data->rx_queues[queue_id] != NULL) {
1328 rx_queue_free(dev->data->rx_queues[queue_id]);
1329 dev->data->rx_queues[queue_id] = NULL;
1332 /* allocate memory for the queue structure */
1333 q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1336 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1342 q->nb_desc = nb_desc;
1343 q->port_id = dev->data->port_id;
1344 q->queue_id = queue_id;
1345 q->tail_ptr = (volatile uint32_t *)
1346 &((uint32_t *)hw->hw_addr)[FM10K_RDT(queue_id)];
1347 if (handle_rxconf(q, conf))
1350 /* allocate memory for the software ring */
1351 q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1352 nb_desc * sizeof(struct rte_mbuf *),
1353 RTE_CACHE_LINE_SIZE, socket_id);
1354 if (q->sw_ring == NULL) {
1355 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1361 * allocate memory for the hardware descriptor ring. A memzone large
1362 * enough to hold the maximum ring size is requested to allow for
1363 * resizing in later calls to the queue setup function.
1365 mz = allocate_hw_ring(dev->driver->pci_drv.name, "rx_ring",
1366 dev->data->port_id, queue_id, socket_id,
1367 FM10K_MAX_RX_RING_SZ, FM10K_ALIGN_RX_DESC);
1369 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1370 rte_free(q->sw_ring);
1374 q->hw_ring = mz->addr;
1375 #ifdef RTE_LIBRTE_XEN_DOM0
1376 q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1378 q->hw_ring_phys_addr = mz->phys_addr;
1381 dev->data->rx_queues[queue_id] = q;
1386 fm10k_rx_queue_release(void *queue)
1388 PMD_INIT_FUNC_TRACE();
1390 rx_queue_free(queue);
1394 handle_txconf(struct fm10k_tx_queue *q, const struct rte_eth_txconf *conf)
1396 uint16_t tx_free_thresh;
1397 uint16_t tx_rs_thresh;
1399 /* constraint MACROs require that tx_free_thresh is configured
1400 * before tx_rs_thresh */
1401 if (conf->tx_free_thresh == 0)
1402 tx_free_thresh = FM10K_TX_FREE_THRESH_DEFAULT(q);
1404 tx_free_thresh = conf->tx_free_thresh;
1406 /* make sure the requested threshold satisfies the constraints */
1407 if (check_thresh(FM10K_TX_FREE_THRESH_MIN(q),
1408 FM10K_TX_FREE_THRESH_MAX(q),
1409 FM10K_TX_FREE_THRESH_DIV(q),
1411 PMD_INIT_LOG(ERR, "tx_free_thresh (%u) must be "
1412 "less than or equal to %u, "
1413 "greater than or equal to %u, "
1414 "and a divisor of %u",
1415 tx_free_thresh, FM10K_TX_FREE_THRESH_MAX(q),
1416 FM10K_TX_FREE_THRESH_MIN(q),
1417 FM10K_TX_FREE_THRESH_DIV(q));
1421 q->free_thresh = tx_free_thresh;
1423 if (conf->tx_rs_thresh == 0)
1424 tx_rs_thresh = FM10K_TX_RS_THRESH_DEFAULT(q);
1426 tx_rs_thresh = conf->tx_rs_thresh;
1428 q->tx_deferred_start = conf->tx_deferred_start;
1430 /* make sure the requested threshold satisfies the constraints */
1431 if (check_thresh(FM10K_TX_RS_THRESH_MIN(q),
1432 FM10K_TX_RS_THRESH_MAX(q),
1433 FM10K_TX_RS_THRESH_DIV(q),
1435 PMD_INIT_LOG(ERR, "tx_rs_thresh (%u) must be "
1436 "less than or equal to %u, "
1437 "greater than or equal to %u, "
1438 "and a divisor of %u",
1439 tx_rs_thresh, FM10K_TX_RS_THRESH_MAX(q),
1440 FM10K_TX_RS_THRESH_MIN(q),
1441 FM10K_TX_RS_THRESH_DIV(q));
1445 q->rs_thresh = tx_rs_thresh;
1451 fm10k_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id,
1452 uint16_t nb_desc, unsigned int socket_id,
1453 const struct rte_eth_txconf *conf)
1455 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1456 struct fm10k_tx_queue *q;
1457 const struct rte_memzone *mz;
1459 PMD_INIT_FUNC_TRACE();
1461 /* make sure a valid number of descriptors have been requested */
1462 if (check_nb_desc(FM10K_MIN_TX_DESC, FM10K_MAX_TX_DESC,
1463 FM10K_MULT_TX_DESC, nb_desc)) {
1464 PMD_INIT_LOG(ERR, "Number of Tx descriptors (%u) must be "
1465 "less than or equal to %"PRIu32", "
1466 "greater than or equal to %u, "
1467 "and a multiple of %u",
1468 nb_desc, (uint32_t)FM10K_MAX_TX_DESC, FM10K_MIN_TX_DESC,
1469 FM10K_MULT_TX_DESC);
1474 * if this queue existed already, free the associated memory. The
1475 * queue cannot be reused in case we need to allocate memory on
1476 * different socket than was previously used.
1478 if (dev->data->tx_queues[queue_id] != NULL) {
1479 tx_queue_free(dev->data->tx_queues[queue_id]);
1480 dev->data->tx_queues[queue_id] = NULL;
1483 /* allocate memory for the queue structure */
1484 q = rte_zmalloc_socket("fm10k", sizeof(*q), RTE_CACHE_LINE_SIZE,
1487 PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
1492 q->nb_desc = nb_desc;
1493 q->port_id = dev->data->port_id;
1494 q->queue_id = queue_id;
1495 q->tail_ptr = (volatile uint32_t *)
1496 &((uint32_t *)hw->hw_addr)[FM10K_TDT(queue_id)];
1497 if (handle_txconf(q, conf))
1500 /* allocate memory for the software ring */
1501 q->sw_ring = rte_zmalloc_socket("fm10k sw ring",
1502 nb_desc * sizeof(struct rte_mbuf *),
1503 RTE_CACHE_LINE_SIZE, socket_id);
1504 if (q->sw_ring == NULL) {
1505 PMD_INIT_LOG(ERR, "Cannot allocate software ring");
1511 * allocate memory for the hardware descriptor ring. A memzone large
1512 * enough to hold the maximum ring size is requested to allow for
1513 * resizing in later calls to the queue setup function.
1515 mz = allocate_hw_ring(dev->driver->pci_drv.name, "tx_ring",
1516 dev->data->port_id, queue_id, socket_id,
1517 FM10K_MAX_TX_RING_SZ, FM10K_ALIGN_TX_DESC);
1519 PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
1520 rte_free(q->sw_ring);
1524 q->hw_ring = mz->addr;
1525 #ifdef RTE_LIBRTE_XEN_DOM0
1526 q->hw_ring_phys_addr = rte_mem_phy2mch(mz->memseg_id, mz->phys_addr);
1528 q->hw_ring_phys_addr = mz->phys_addr;
1532 * allocate memory for the RS bit tracker. Enough slots to hold the
1533 * descriptor index for each RS bit needing to be set are required.
1535 q->rs_tracker.list = rte_zmalloc_socket("fm10k rs tracker",
1536 ((nb_desc + 1) / q->rs_thresh) *
1538 RTE_CACHE_LINE_SIZE, socket_id);
1539 if (q->rs_tracker.list == NULL) {
1540 PMD_INIT_LOG(ERR, "Cannot allocate RS bit tracker");
1541 rte_free(q->sw_ring);
1546 dev->data->tx_queues[queue_id] = q;
1551 fm10k_tx_queue_release(void *queue)
1553 PMD_INIT_FUNC_TRACE();
1555 tx_queue_free(queue);
1559 fm10k_reta_update(struct rte_eth_dev *dev,
1560 struct rte_eth_rss_reta_entry64 *reta_conf,
1563 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1564 uint16_t i, j, idx, shift;
1568 PMD_INIT_FUNC_TRACE();
1570 if (reta_size > FM10K_MAX_RSS_INDICES) {
1571 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1572 "(%d) doesn't match the number hardware can supported "
1573 "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1578 * Update Redirection Table RETA[n], n=0..31. The redirection table has
1579 * 128-entries in 32 registers
1581 for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1582 idx = i / RTE_RETA_GROUP_SIZE;
1583 shift = i % RTE_RETA_GROUP_SIZE;
1584 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1585 BIT_MASK_PER_UINT32);
1590 if (mask != BIT_MASK_PER_UINT32)
1591 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1593 for (j = 0; j < CHARS_PER_UINT32; j++) {
1594 if (mask & (0x1 << j)) {
1596 reta &= ~(UINT8_MAX << CHAR_BIT * j);
1597 reta |= reta_conf[idx].reta[shift + j] <<
1601 FM10K_WRITE_REG(hw, FM10K_RETA(0, i >> 2), reta);
1608 fm10k_reta_query(struct rte_eth_dev *dev,
1609 struct rte_eth_rss_reta_entry64 *reta_conf,
1612 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1613 uint16_t i, j, idx, shift;
1617 PMD_INIT_FUNC_TRACE();
1619 if (reta_size < FM10K_MAX_RSS_INDICES) {
1620 PMD_INIT_LOG(ERR, "The size of hash lookup table configured "
1621 "(%d) doesn't match the number hardware can supported "
1622 "(%d)", reta_size, FM10K_MAX_RSS_INDICES);
1627 * Read Redirection Table RETA[n], n=0..31. The redirection table has
1628 * 128-entries in 32 registers
1630 for (i = 0; i < FM10K_MAX_RSS_INDICES; i += CHARS_PER_UINT32) {
1631 idx = i / RTE_RETA_GROUP_SIZE;
1632 shift = i % RTE_RETA_GROUP_SIZE;
1633 mask = (uint8_t)((reta_conf[idx].mask >> shift) &
1634 BIT_MASK_PER_UINT32);
1638 reta = FM10K_READ_REG(hw, FM10K_RETA(0, i >> 2));
1639 for (j = 0; j < CHARS_PER_UINT32; j++) {
1640 if (mask & (0x1 << j))
1641 reta_conf[idx].reta[shift + j] = ((reta >>
1642 CHAR_BIT * j) & UINT8_MAX);
1650 fm10k_rss_hash_update(struct rte_eth_dev *dev,
1651 struct rte_eth_rss_conf *rss_conf)
1653 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1654 uint32_t *key = (uint32_t *)rss_conf->rss_key;
1656 uint64_t hf = rss_conf->rss_hf;
1659 PMD_INIT_FUNC_TRACE();
1661 if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1662 FM10K_RSSRK_ENTRIES_PER_REG)
1669 mrqc |= (hf & ETH_RSS_IPV4) ? FM10K_MRQC_IPV4 : 0;
1670 mrqc |= (hf & ETH_RSS_IPV6) ? FM10K_MRQC_IPV6 : 0;
1671 mrqc |= (hf & ETH_RSS_IPV6_EX) ? FM10K_MRQC_IPV6 : 0;
1672 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_TCP) ? FM10K_MRQC_TCP_IPV4 : 0;
1673 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_TCP) ? FM10K_MRQC_TCP_IPV6 : 0;
1674 mrqc |= (hf & ETH_RSS_IPV6_TCP_EX) ? FM10K_MRQC_TCP_IPV6 : 0;
1675 mrqc |= (hf & ETH_RSS_NONFRAG_IPV4_UDP) ? FM10K_MRQC_UDP_IPV4 : 0;
1676 mrqc |= (hf & ETH_RSS_NONFRAG_IPV6_UDP) ? FM10K_MRQC_UDP_IPV6 : 0;
1677 mrqc |= (hf & ETH_RSS_IPV6_UDP_EX) ? FM10K_MRQC_UDP_IPV6 : 0;
1679 /* If the mapping doesn't fit any supported, return */
1684 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1685 FM10K_WRITE_REG(hw, FM10K_RSSRK(0, i), key[i]);
1687 FM10K_WRITE_REG(hw, FM10K_MRQC(0), mrqc);
1693 fm10k_rss_hash_conf_get(struct rte_eth_dev *dev,
1694 struct rte_eth_rss_conf *rss_conf)
1696 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1697 uint32_t *key = (uint32_t *)rss_conf->rss_key;
1702 PMD_INIT_FUNC_TRACE();
1704 if (rss_conf->rss_key_len < FM10K_RSSRK_SIZE *
1705 FM10K_RSSRK_ENTRIES_PER_REG)
1709 for (i = 0; i < FM10K_RSSRK_SIZE; ++i)
1710 key[i] = FM10K_READ_REG(hw, FM10K_RSSRK(0, i));
1712 mrqc = FM10K_READ_REG(hw, FM10K_MRQC(0));
1714 hf |= (mrqc & FM10K_MRQC_IPV4) ? ETH_RSS_IPV4 : 0;
1715 hf |= (mrqc & FM10K_MRQC_IPV6) ? ETH_RSS_IPV6 : 0;
1716 hf |= (mrqc & FM10K_MRQC_IPV6) ? ETH_RSS_IPV6_EX : 0;
1717 hf |= (mrqc & FM10K_MRQC_TCP_IPV4) ? ETH_RSS_NONFRAG_IPV4_TCP : 0;
1718 hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_NONFRAG_IPV6_TCP : 0;
1719 hf |= (mrqc & FM10K_MRQC_TCP_IPV6) ? ETH_RSS_IPV6_TCP_EX : 0;
1720 hf |= (mrqc & FM10K_MRQC_UDP_IPV4) ? ETH_RSS_NONFRAG_IPV4_UDP : 0;
1721 hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_NONFRAG_IPV6_UDP : 0;
1722 hf |= (mrqc & FM10K_MRQC_UDP_IPV6) ? ETH_RSS_IPV6_UDP_EX : 0;
1724 rss_conf->rss_hf = hf;
1730 fm10k_dev_enable_intr_pf(struct rte_eth_dev *dev)
1732 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1733 uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1735 /* Bind all local non-queue interrupt to vector 0 */
1738 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1739 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1740 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1741 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1742 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1743 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1745 /* Enable misc causes */
1746 FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1747 FM10K_EIMR_ENABLE(THI_FAULT) |
1748 FM10K_EIMR_ENABLE(FUM_FAULT) |
1749 FM10K_EIMR_ENABLE(MAILBOX) |
1750 FM10K_EIMR_ENABLE(SWITCHREADY) |
1751 FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1752 FM10K_EIMR_ENABLE(SRAMERROR) |
1753 FM10K_EIMR_ENABLE(VFLR));
1756 FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1757 FM10K_ITR_MASK_CLEAR);
1758 FM10K_WRITE_FLUSH(hw);
1762 fm10k_dev_disable_intr_pf(struct rte_eth_dev *dev)
1764 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1765 uint32_t int_map = FM10K_INT_MAP_DISABLE;
1769 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_Mailbox), int_map);
1770 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_PCIeFault), int_map);
1771 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchUpDown), int_map);
1772 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SwitchEvent), int_map);
1773 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_SRAM), int_map);
1774 FM10K_WRITE_REG(hw, FM10K_INT_MAP(fm10k_int_VFLR), int_map);
1776 /* Disable misc causes */
1777 FM10K_WRITE_REG(hw, FM10K_EIMR, FM10K_EIMR_DISABLE(PCA_FAULT) |
1778 FM10K_EIMR_DISABLE(THI_FAULT) |
1779 FM10K_EIMR_DISABLE(FUM_FAULT) |
1780 FM10K_EIMR_DISABLE(MAILBOX) |
1781 FM10K_EIMR_DISABLE(SWITCHREADY) |
1782 FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
1783 FM10K_EIMR_DISABLE(SRAMERROR) |
1784 FM10K_EIMR_DISABLE(VFLR));
1787 FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_MASK_SET);
1788 FM10K_WRITE_FLUSH(hw);
1792 fm10k_dev_enable_intr_vf(struct rte_eth_dev *dev)
1794 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1795 uint32_t int_map = FM10K_INT_MAP_IMMEDIATE;
1797 /* Bind all local non-queue interrupt to vector 0 */
1800 /* Only INT 0 available, other 15 are reserved. */
1801 FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1804 FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
1805 FM10K_ITR_MASK_CLEAR);
1806 FM10K_WRITE_FLUSH(hw);
1810 fm10k_dev_disable_intr_vf(struct rte_eth_dev *dev)
1812 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1813 uint32_t int_map = FM10K_INT_MAP_DISABLE;
1817 /* Only INT 0 available, other 15 are reserved. */
1818 FM10K_WRITE_REG(hw, FM10K_VFINT_MAP, int_map);
1821 FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_MASK_SET);
1822 FM10K_WRITE_FLUSH(hw);
1826 fm10k_dev_handle_fault(struct fm10k_hw *hw, uint32_t eicr)
1828 struct fm10k_fault fault;
1830 const char *estr = "Unknown error";
1832 /* Process PCA fault */
1833 if (eicr & FM10K_EICR_PCA_FAULT) {
1834 err = fm10k_get_fault(hw, FM10K_PCA_FAULT, &fault);
1837 switch (fault.type) {
1839 estr = "PCA_NO_FAULT"; break;
1840 case PCA_UNMAPPED_ADDR:
1841 estr = "PCA_UNMAPPED_ADDR"; break;
1842 case PCA_BAD_QACCESS_PF:
1843 estr = "PCA_BAD_QACCESS_PF"; break;
1844 case PCA_BAD_QACCESS_VF:
1845 estr = "PCA_BAD_QACCESS_VF"; break;
1846 case PCA_MALICIOUS_REQ:
1847 estr = "PCA_MALICIOUS_REQ"; break;
1848 case PCA_POISONED_TLP:
1849 estr = "PCA_POISONED_TLP"; break;
1851 estr = "PCA_TLP_ABORT"; break;
1855 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1856 estr, fault.func ? "VF" : "PF", fault.func,
1857 fault.address, fault.specinfo);
1860 /* Process THI fault */
1861 if (eicr & FM10K_EICR_THI_FAULT) {
1862 err = fm10k_get_fault(hw, FM10K_THI_FAULT, &fault);
1865 switch (fault.type) {
1867 estr = "THI_NO_FAULT"; break;
1868 case THI_MAL_DIS_Q_FAULT:
1869 estr = "THI_MAL_DIS_Q_FAULT"; break;
1873 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1874 estr, fault.func ? "VF" : "PF", fault.func,
1875 fault.address, fault.specinfo);
1878 /* Process FUM fault */
1879 if (eicr & FM10K_EICR_FUM_FAULT) {
1880 err = fm10k_get_fault(hw, FM10K_FUM_FAULT, &fault);
1883 switch (fault.type) {
1885 estr = "FUM_NO_FAULT"; break;
1886 case FUM_UNMAPPED_ADDR:
1887 estr = "FUM_UNMAPPED_ADDR"; break;
1888 case FUM_POISONED_TLP:
1889 estr = "FUM_POISONED_TLP"; break;
1890 case FUM_BAD_VF_QACCESS:
1891 estr = "FUM_BAD_VF_QACCESS"; break;
1892 case FUM_ADD_DECODE_ERR:
1893 estr = "FUM_ADD_DECODE_ERR"; break;
1895 estr = "FUM_RO_ERROR"; break;
1896 case FUM_QPRC_CRC_ERROR:
1897 estr = "FUM_QPRC_CRC_ERROR"; break;
1898 case FUM_CSR_TIMEOUT:
1899 estr = "FUM_CSR_TIMEOUT"; break;
1900 case FUM_INVALID_TYPE:
1901 estr = "FUM_INVALID_TYPE"; break;
1902 case FUM_INVALID_LENGTH:
1903 estr = "FUM_INVALID_LENGTH"; break;
1904 case FUM_INVALID_BE:
1905 estr = "FUM_INVALID_BE"; break;
1906 case FUM_INVALID_ALIGN:
1907 estr = "FUM_INVALID_ALIGN"; break;
1911 PMD_INIT_LOG(ERR, "%s: %s(%d) Addr:0x%"PRIx64" Spec: 0x%x",
1912 estr, fault.func ? "VF" : "PF", fault.func,
1913 fault.address, fault.specinfo);
1918 PMD_INIT_LOG(ERR, "Failed to handle fault event.");
1923 * PF interrupt handler triggered by NIC for handling specific interrupt.
1926 * Pointer to interrupt handle.
1928 * The address of parameter (struct rte_eth_dev *) regsitered before.
1934 fm10k_dev_interrupt_handler_pf(
1935 __rte_unused struct rte_intr_handle *handle,
1938 struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1939 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1940 uint32_t cause, status;
1942 if (hw->mac.type != fm10k_mac_pf)
1945 cause = FM10K_READ_REG(hw, FM10K_EICR);
1947 /* Handle PCI fault cases */
1948 if (cause & FM10K_EICR_FAULT_MASK) {
1949 PMD_INIT_LOG(ERR, "INT: find fault!");
1950 fm10k_dev_handle_fault(hw, cause);
1953 /* Handle switch up/down */
1954 if (cause & FM10K_EICR_SWITCHNOTREADY)
1955 PMD_INIT_LOG(ERR, "INT: Switch is not ready");
1957 if (cause & FM10K_EICR_SWITCHREADY)
1958 PMD_INIT_LOG(INFO, "INT: Switch is ready");
1960 /* Handle mailbox message */
1962 hw->mbx.ops.process(hw, &hw->mbx);
1963 fm10k_mbx_unlock(hw);
1965 /* Handle SRAM error */
1966 if (cause & FM10K_EICR_SRAMERROR) {
1967 PMD_INIT_LOG(ERR, "INT: SRAM error on PEP");
1969 status = FM10K_READ_REG(hw, FM10K_SRAM_IP);
1970 /* Write to clear pending bits */
1971 FM10K_WRITE_REG(hw, FM10K_SRAM_IP, status);
1973 /* Todo: print out error message after shared code updates */
1976 /* Clear these 3 events if having any */
1977 cause &= FM10K_EICR_SWITCHNOTREADY | FM10K_EICR_MAILBOX |
1978 FM10K_EICR_SWITCHREADY;
1980 FM10K_WRITE_REG(hw, FM10K_EICR, cause);
1982 /* Re-enable interrupt from device side */
1983 FM10K_WRITE_REG(hw, FM10K_ITR(0), FM10K_ITR_AUTOMASK |
1984 FM10K_ITR_MASK_CLEAR);
1985 /* Re-enable interrupt from host side */
1986 rte_intr_enable(&(dev->pci_dev->intr_handle));
1990 * VF interrupt handler triggered by NIC for handling specific interrupt.
1993 * Pointer to interrupt handle.
1995 * The address of parameter (struct rte_eth_dev *) regsitered before.
2001 fm10k_dev_interrupt_handler_vf(
2002 __rte_unused struct rte_intr_handle *handle,
2005 struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
2006 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2008 if (hw->mac.type != fm10k_mac_vf)
2011 /* Handle mailbox message if lock is acquired */
2013 hw->mbx.ops.process(hw, &hw->mbx);
2014 fm10k_mbx_unlock(hw);
2016 /* Re-enable interrupt from device side */
2017 FM10K_WRITE_REG(hw, FM10K_VFITR(0), FM10K_ITR_AUTOMASK |
2018 FM10K_ITR_MASK_CLEAR);
2019 /* Re-enable interrupt from host side */
2020 rte_intr_enable(&(dev->pci_dev->intr_handle));
2023 /* Mailbox message handler in VF */
2024 static const struct fm10k_msg_data fm10k_msgdata_vf[] = {
2025 FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
2026 FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_msg_mac_vlan_vf),
2027 FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
2028 FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2031 /* Mailbox message handler in PF */
2032 static const struct fm10k_msg_data fm10k_msgdata_pf[] = {
2033 FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
2034 FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
2035 FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_msg_lport_map_pf),
2036 FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
2037 FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
2038 FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_msg_update_pvid_pf),
2039 FM10K_TLV_MSG_ERROR_HANDLER(fm10k_tlv_msg_error),
2043 fm10k_setup_mbx_service(struct fm10k_hw *hw)
2047 /* Initialize mailbox lock */
2048 fm10k_mbx_initlock(hw);
2050 /* Replace default message handler with new ones */
2051 if (hw->mac.type == fm10k_mac_pf)
2052 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_pf);
2054 err = hw->mbx.ops.register_handlers(&hw->mbx, fm10k_msgdata_vf);
2057 PMD_INIT_LOG(ERR, "Failed to register mailbox handler.err:%d",
2061 /* Connect to SM for PF device or PF for VF device */
2062 return hw->mbx.ops.connect(hw, &hw->mbx);
2066 fm10k_close_mbx_service(struct fm10k_hw *hw)
2068 /* Disconnect from SM for PF device or PF for VF device */
2069 hw->mbx.ops.disconnect(hw, &hw->mbx);
2072 static const struct eth_dev_ops fm10k_eth_dev_ops = {
2073 .dev_configure = fm10k_dev_configure,
2074 .dev_start = fm10k_dev_start,
2075 .dev_stop = fm10k_dev_stop,
2076 .dev_close = fm10k_dev_close,
2077 .promiscuous_enable = fm10k_dev_promiscuous_enable,
2078 .promiscuous_disable = fm10k_dev_promiscuous_disable,
2079 .allmulticast_enable = fm10k_dev_allmulticast_enable,
2080 .allmulticast_disable = fm10k_dev_allmulticast_disable,
2081 .stats_get = fm10k_stats_get,
2082 .stats_reset = fm10k_stats_reset,
2083 .link_update = fm10k_link_update,
2084 .dev_infos_get = fm10k_dev_infos_get,
2085 .vlan_filter_set = fm10k_vlan_filter_set,
2086 .vlan_offload_set = fm10k_vlan_offload_set,
2087 .mac_addr_add = fm10k_macaddr_add,
2088 .mac_addr_remove = fm10k_macaddr_remove,
2089 .rx_queue_start = fm10k_dev_rx_queue_start,
2090 .rx_queue_stop = fm10k_dev_rx_queue_stop,
2091 .tx_queue_start = fm10k_dev_tx_queue_start,
2092 .tx_queue_stop = fm10k_dev_tx_queue_stop,
2093 .rx_queue_setup = fm10k_rx_queue_setup,
2094 .rx_queue_release = fm10k_rx_queue_release,
2095 .tx_queue_setup = fm10k_tx_queue_setup,
2096 .tx_queue_release = fm10k_tx_queue_release,
2097 .reta_update = fm10k_reta_update,
2098 .reta_query = fm10k_reta_query,
2099 .rss_hash_update = fm10k_rss_hash_update,
2100 .rss_hash_conf_get = fm10k_rss_hash_conf_get,
2104 eth_fm10k_dev_init(struct rte_eth_dev *dev)
2106 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2108 struct fm10k_macvlan_filter_info *macvlan;
2110 PMD_INIT_FUNC_TRACE();
2112 dev->dev_ops = &fm10k_eth_dev_ops;
2113 dev->rx_pkt_burst = &fm10k_recv_pkts;
2114 dev->tx_pkt_burst = &fm10k_xmit_pkts;
2116 if (dev->data->scattered_rx)
2117 dev->rx_pkt_burst = &fm10k_recv_scattered_pkts;
2119 /* only initialize in the primary process */
2120 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2123 macvlan = FM10K_DEV_PRIVATE_TO_MACVLAN(dev->data->dev_private);
2124 memset(macvlan, 0, sizeof(*macvlan));
2125 /* Vendor and Device ID need to be set before init of shared code */
2126 memset(hw, 0, sizeof(*hw));
2127 hw->device_id = dev->pci_dev->id.device_id;
2128 hw->vendor_id = dev->pci_dev->id.vendor_id;
2129 hw->subsystem_device_id = dev->pci_dev->id.subsystem_device_id;
2130 hw->subsystem_vendor_id = dev->pci_dev->id.subsystem_vendor_id;
2131 hw->revision_id = 0;
2132 hw->hw_addr = (void *)dev->pci_dev->mem_resource[0].addr;
2133 if (hw->hw_addr == NULL) {
2134 PMD_INIT_LOG(ERR, "Bad mem resource."
2135 " Try to blacklist unused devices.");
2139 /* Store fm10k_adapter pointer */
2140 hw->back = dev->data->dev_private;
2142 /* Initialize the shared code */
2143 diag = fm10k_init_shared_code(hw);
2144 if (diag != FM10K_SUCCESS) {
2145 PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
2150 * Inialize bus info. Normally we would call fm10k_get_bus_info(), but
2151 * there is no way to get link status without reading BAR4. Until this
2152 * works, assume we have maximum bandwidth.
2153 * @todo - fix bus info
2155 hw->bus_caps.speed = fm10k_bus_speed_8000;
2156 hw->bus_caps.width = fm10k_bus_width_pcie_x8;
2157 hw->bus_caps.payload = fm10k_bus_payload_512;
2158 hw->bus.speed = fm10k_bus_speed_8000;
2159 hw->bus.width = fm10k_bus_width_pcie_x8;
2160 hw->bus.payload = fm10k_bus_payload_256;
2162 /* Initialize the hw */
2163 diag = fm10k_init_hw(hw);
2164 if (diag != FM10K_SUCCESS) {
2165 PMD_INIT_LOG(ERR, "Hardware init failed: %d", diag);
2169 /* Initialize MAC address(es) */
2170 dev->data->mac_addrs = rte_zmalloc("fm10k",
2171 ETHER_ADDR_LEN * FM10K_MAX_MACADDR_NUM, 0);
2172 if (dev->data->mac_addrs == NULL) {
2173 PMD_INIT_LOG(ERR, "Cannot allocate memory for MAC addresses");
2177 diag = fm10k_read_mac_addr(hw);
2179 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2180 &dev->data->mac_addrs[0]);
2182 if (diag != FM10K_SUCCESS ||
2183 !is_valid_assigned_ether_addr(dev->data->mac_addrs)) {
2185 /* Generate a random addr */
2186 eth_random_addr(hw->mac.addr);
2187 memcpy(hw->mac.perm_addr, hw->mac.addr, ETH_ALEN);
2188 ether_addr_copy((const struct ether_addr *)hw->mac.addr,
2189 &dev->data->mac_addrs[0]);
2192 /* Reset the hw statistics */
2193 fm10k_stats_reset(dev);
2196 diag = fm10k_reset_hw(hw);
2197 if (diag != FM10K_SUCCESS) {
2198 PMD_INIT_LOG(ERR, "Hardware reset failed: %d", diag);
2202 /* Setup mailbox service */
2203 diag = fm10k_setup_mbx_service(hw);
2204 if (diag != FM10K_SUCCESS) {
2205 PMD_INIT_LOG(ERR, "Failed to setup mailbox: %d", diag);
2209 /*PF/VF has different interrupt handling mechanism */
2210 if (hw->mac.type == fm10k_mac_pf) {
2211 /* register callback func to eal lib */
2212 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2213 fm10k_dev_interrupt_handler_pf, (void *)dev);
2215 /* enable MISC interrupt */
2216 fm10k_dev_enable_intr_pf(dev);
2218 rte_intr_callback_register(&(dev->pci_dev->intr_handle),
2219 fm10k_dev_interrupt_handler_vf, (void *)dev);
2221 fm10k_dev_enable_intr_vf(dev);
2224 /* Enable uio intr after callback registered */
2225 rte_intr_enable(&(dev->pci_dev->intr_handle));
2227 hw->mac.ops.update_int_moderator(hw);
2229 /* Make sure Switch Manager is ready before going forward. */
2230 if (hw->mac.type == fm10k_mac_pf) {
2231 int switch_ready = 0;
2234 for (i = 0; i < MAX_QUERY_SWITCH_STATE_TIMES; i++) {
2236 hw->mac.ops.get_host_state(hw, &switch_ready);
2237 fm10k_mbx_unlock(hw);
2240 /* Delay some time to acquire async LPORT_MAP info. */
2241 rte_delay_us(WAIT_SWITCH_MSG_US);
2244 if (switch_ready == 0) {
2245 PMD_INIT_LOG(ERR, "switch is not ready");
2251 * Below function will trigger operations on mailbox, acquire lock to
2252 * avoid race condition from interrupt handler. Operations on mailbox
2253 * FIFO will trigger interrupt to PF/SM, in which interrupt handler
2254 * will handle and generate an interrupt to our side. Then, FIFO in
2255 * mailbox will be touched.
2258 /* Enable port first */
2259 hw->mac.ops.update_lport_state(hw, hw->mac.dglort_map, 1, 1);
2261 /* Set unicast mode by default. App can change to other mode in other
2264 hw->mac.ops.update_xcast_mode(hw, hw->mac.dglort_map,
2265 FM10K_XCAST_MODE_NONE);
2267 fm10k_mbx_unlock(hw);
2269 /* Add default mac address */
2270 fm10k_MAC_filter_set(dev, hw->mac.addr, true);
2276 eth_fm10k_dev_uninit(struct rte_eth_dev *dev)
2278 struct fm10k_hw *hw = FM10K_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2280 PMD_INIT_FUNC_TRACE();
2282 /* only uninitialize in the primary process */
2283 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2286 /* safe to close dev here */
2287 fm10k_dev_close(dev);
2289 dev->dev_ops = NULL;
2290 dev->rx_pkt_burst = NULL;
2291 dev->tx_pkt_burst = NULL;
2293 /* disable uio/vfio intr */
2294 rte_intr_disable(&(dev->pci_dev->intr_handle));
2296 /*PF/VF has different interrupt handling mechanism */
2297 if (hw->mac.type == fm10k_mac_pf) {
2298 /* disable interrupt */
2299 fm10k_dev_disable_intr_pf(dev);
2301 /* unregister callback func to eal lib */
2302 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2303 fm10k_dev_interrupt_handler_pf, (void *)dev);
2305 /* disable interrupt */
2306 fm10k_dev_disable_intr_vf(dev);
2308 rte_intr_callback_unregister(&(dev->pci_dev->intr_handle),
2309 fm10k_dev_interrupt_handler_vf, (void *)dev);
2312 /* free mac memory */
2313 if (dev->data->mac_addrs) {
2314 rte_free(dev->data->mac_addrs);
2315 dev->data->mac_addrs = NULL;
2318 memset(hw, 0, sizeof(*hw));
2324 * The set of PCI devices this driver supports. This driver will enable both PF
2325 * and SRIOV-VF devices.
2327 static const struct rte_pci_id pci_id_fm10k_map[] = {
2328 #define RTE_PCI_DEV_ID_DECL_FM10K(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2329 #define RTE_PCI_DEV_ID_DECL_FM10KVF(vend, dev) { RTE_PCI_DEVICE(vend, dev) },
2330 #include "rte_pci_dev_ids.h"
2331 { .vendor_id = 0, /* sentinel */ },
2334 static struct eth_driver rte_pmd_fm10k = {
2336 .name = "rte_pmd_fm10k",
2337 .id_table = pci_id_fm10k_map,
2338 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
2340 .eth_dev_init = eth_fm10k_dev_init,
2341 .eth_dev_uninit = eth_fm10k_dev_uninit,
2342 .dev_private_size = sizeof(struct fm10k_adapter),
2346 * Driver initialization routine.
2347 * Invoked once at EAL init time.
2348 * Register itself as the [Poll Mode] Driver of PCI FM10K devices.
2351 rte_pmd_fm10k_init(__rte_unused const char *name,
2352 __rte_unused const char *params)
2354 PMD_INIT_FUNC_TRACE();
2355 rte_eth_driver_register(&rte_pmd_fm10k);
2359 static struct rte_driver rte_fm10k_driver = {
2361 .init = rte_pmd_fm10k_init,
2364 PMD_REGISTER_DRIVER(rte_fm10k_driver);