1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
13 #include <rte_bus_pci.h>
14 #include <rte_memzone.h>
15 #include <rte_malloc.h>
17 #include <rte_string_fns.h>
18 #include <ethdev_driver.h>
19 #include <rte_geneve.h>
21 #include "enic_compat.h"
23 #include "wq_enet_desc.h"
24 #include "rq_enet_desc.h"
25 #include "cq_enet_desc.h"
26 #include "vnic_enet.h"
31 #include "vnic_intr.h"
34 static inline int enic_is_sriov_vf(struct enic *enic)
36 return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
39 static int is_zero_addr(uint8_t *addr)
41 return !(addr[0] | addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
44 static int is_mcast_addr(uint8_t *addr)
49 static int is_eth_addr_valid(uint8_t *addr)
51 return !is_mcast_addr(addr) && !is_zero_addr(addr);
55 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
59 if (!rq || !rq->mbuf_ring) {
60 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
64 for (i = 0; i < rq->ring.desc_count; i++) {
65 if (rq->mbuf_ring[i]) {
66 rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
67 rq->mbuf_ring[i] = NULL;
72 void enic_free_wq_buf(struct rte_mbuf **buf)
74 struct rte_mbuf *mbuf = *buf;
76 rte_pktmbuf_free_seg(mbuf);
80 static void enic_log_q_error(struct enic *enic)
83 uint32_t error_status;
85 for (i = 0; i < enic->wq_count; i++) {
86 error_status = vnic_wq_error_status(&enic->wq[i]);
88 dev_err(enic, "WQ[%d] error_status %d\n", i,
92 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
93 if (!enic->rq[i].in_use)
95 error_status = vnic_rq_error_status(&enic->rq[i]);
97 dev_err(enic, "RQ[%d] error_status %d\n", i,
102 static void enic_clear_soft_stats(struct enic *enic)
104 struct enic_soft_stats *soft_stats = &enic->soft_stats;
105 rte_atomic64_clear(&soft_stats->rx_nombuf);
106 rte_atomic64_clear(&soft_stats->rx_packet_errors);
107 rte_atomic64_clear(&soft_stats->tx_oversized);
110 static void enic_init_soft_stats(struct enic *enic)
112 struct enic_soft_stats *soft_stats = &enic->soft_stats;
113 rte_atomic64_init(&soft_stats->rx_nombuf);
114 rte_atomic64_init(&soft_stats->rx_packet_errors);
115 rte_atomic64_init(&soft_stats->tx_oversized);
116 enic_clear_soft_stats(enic);
119 int enic_dev_stats_clear(struct enic *enic)
123 ret = vnic_dev_stats_clear(enic->vdev);
125 dev_err(enic, "Error in clearing stats\n");
128 enic_clear_soft_stats(enic);
133 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
135 struct vnic_stats *stats;
136 struct enic_soft_stats *soft_stats = &enic->soft_stats;
137 int64_t rx_truncated;
138 uint64_t rx_packet_errors;
139 int ret = vnic_dev_stats_dump(enic->vdev, &stats);
142 dev_err(enic, "Error in getting stats\n");
146 /* The number of truncated packets can only be calculated by
147 * subtracting a hardware counter from error packets received by
148 * the driver. Note: this causes transient inaccuracies in the
149 * ipackets count. Also, the length of truncated packets are
150 * counted in ibytes even though truncated packets are dropped
151 * which can make ibytes be slightly higher than it should be.
153 rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
154 rx_truncated = rx_packet_errors - stats->rx.rx_errors;
156 r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
157 r_stats->opackets = stats->tx.tx_frames_ok;
159 r_stats->ibytes = stats->rx.rx_bytes_ok;
160 r_stats->obytes = stats->tx.tx_bytes_ok;
162 r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
163 r_stats->oerrors = stats->tx.tx_errors
164 + rte_atomic64_read(&soft_stats->tx_oversized);
166 r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
168 r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
172 int enic_del_mac_address(struct enic *enic, int mac_index)
174 struct rte_eth_dev *eth_dev = enic->rte_dev;
175 uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
177 return vnic_dev_del_addr(enic->vdev, mac_addr);
180 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
184 if (!is_eth_addr_valid(mac_addr)) {
185 dev_err(enic, "invalid mac address\n");
189 err = vnic_dev_add_addr(enic->vdev, mac_addr);
191 dev_err(enic, "add mac addr failed\n");
195 void enic_free_rq_buf(struct rte_mbuf **mbuf)
200 rte_pktmbuf_free(*mbuf);
204 void enic_init_vnic_resources(struct enic *enic)
206 unsigned int error_interrupt_enable = 1;
207 unsigned int error_interrupt_offset = 0;
208 unsigned int rxq_interrupt_enable = 0;
209 unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
210 unsigned int index = 0;
212 struct vnic_rq *data_rq;
214 if (enic->rte_dev->data->dev_conf.intr_conf.rxq)
215 rxq_interrupt_enable = 1;
217 for (index = 0; index < enic->rq_count; index++) {
218 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
220 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
222 error_interrupt_enable,
223 error_interrupt_offset);
225 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index, enic)];
227 vnic_rq_init(data_rq,
229 error_interrupt_enable,
230 error_interrupt_offset);
231 vnic_cq_init(&enic->cq[cq_idx],
232 0 /* flow_control_enable */,
233 1 /* color_enable */,
236 1 /* cq_tail_color */,
237 rxq_interrupt_enable,
238 1 /* cq_entry_enable */,
239 0 /* cq_message_enable */,
240 rxq_interrupt_offset,
241 0 /* cq_message_addr */);
242 if (rxq_interrupt_enable)
243 rxq_interrupt_offset++;
246 for (index = 0; index < enic->wq_count; index++) {
247 vnic_wq_init(&enic->wq[index],
248 enic_cq_wq(enic, index),
249 error_interrupt_enable,
250 error_interrupt_offset);
251 /* Compute unsupported ol flags for enic_prep_pkts() */
252 enic->wq[index].tx_offload_notsup_mask =
253 RTE_MBUF_F_TX_OFFLOAD_MASK ^ enic->tx_offload_mask;
255 cq_idx = enic_cq_wq(enic, index);
256 vnic_cq_init(&enic->cq[cq_idx],
257 0 /* flow_control_enable */,
258 1 /* color_enable */,
261 1 /* cq_tail_color */,
262 0 /* interrupt_enable */,
263 0 /* cq_entry_enable */,
264 1 /* cq_message_enable */,
265 0 /* interrupt offset */,
266 (uint64_t)enic->wq[index].cqmsg_rz->iova);
269 for (index = 0; index < enic->intr_count; index++) {
270 vnic_intr_init(&enic->intr[index],
271 enic->config.intr_timer_usec,
272 enic->config.intr_timer_type,
273 /*mask_on_assertion*/1);
279 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
282 struct rq_enet_desc *rqd = rq->ring.descs;
285 uint32_t max_rx_pktlen;
291 dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
292 rq->ring.desc_count);
295 * If *not* using scatter and the mbuf size is greater than the
296 * requested max packet size (mtu + eth overhead), then reduce the
297 * posted buffer size to max packet size. HW still receives packets
298 * larger than max packet size, but they will be truncated, which we
299 * drop in the rx handler. Not ideal, but better than returning
300 * large packets when the user is not expecting them.
302 max_rx_pktlen = enic_mtu_to_max_rx_pktlen(enic->rte_dev->data->mtu);
303 rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
304 if (max_rx_pktlen < rq_buf_len && !rq->data_queue_enable)
305 rq_buf_len = max_rx_pktlen;
306 for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
307 mb = rte_mbuf_raw_alloc(rq->mp);
309 dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
310 (unsigned)rq->index);
314 mb->data_off = RTE_PKTMBUF_HEADROOM;
315 dma_addr = (dma_addr_t)(mb->buf_iova
316 + RTE_PKTMBUF_HEADROOM);
317 rq_enet_desc_enc(rqd, dma_addr,
318 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
319 : RQ_ENET_TYPE_NOT_SOP),
321 rq->mbuf_ring[i] = mb;
324 * Do not post the buffers to the NIC until we enable the RQ via
327 rq->need_initial_post = true;
328 /* Initialize fetch index while RQ is disabled */
329 iowrite32(0, &rq->ctrl->fetch_index);
334 * Post the Rx buffers for the first time. enic_alloc_rx_queue_mbufs() has
335 * allocated the buffers and filled the RQ descriptor ring. Just need to push
336 * the post index to the NIC.
339 enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)
341 if (!rq->in_use || !rq->need_initial_post)
344 /* make sure all prior writes are complete before doing the PIO write */
347 /* Post all but the last buffer to VIC. */
348 rq->posted_index = rq->ring.desc_count - 1;
352 dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
353 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
354 iowrite32(rq->posted_index, &rq->ctrl->posted_index);
356 rq->need_initial_post = false;
360 enic_alloc_consistent(void *priv, size_t size,
361 dma_addr_t *dma_handle, uint8_t *name)
364 const struct rte_memzone *rz;
366 struct enic *enic = (struct enic *)priv;
367 struct enic_memzone_entry *mze;
369 rz = rte_memzone_reserve_aligned((const char *)name, size,
370 SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_PAGE_SIZE);
372 pr_err("%s : Failed to allocate memory requested for %s\n",
378 *dma_handle = (dma_addr_t)rz->iova;
380 mze = rte_malloc("enic memzone entry",
381 sizeof(struct enic_memzone_entry), 0);
384 pr_err("%s : Failed to allocate memory for memzone list\n",
386 rte_memzone_free(rz);
392 rte_spinlock_lock(&enic->memzone_list_lock);
393 LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
394 rte_spinlock_unlock(&enic->memzone_list_lock);
400 enic_free_consistent(void *priv,
401 __rte_unused size_t size,
403 dma_addr_t dma_handle)
405 struct enic_memzone_entry *mze;
406 struct enic *enic = (struct enic *)priv;
408 rte_spinlock_lock(&enic->memzone_list_lock);
409 LIST_FOREACH(mze, &enic->memzone_list, entries) {
410 if (mze->rz->addr == vaddr &&
411 mze->rz->iova == dma_handle)
415 rte_spinlock_unlock(&enic->memzone_list_lock);
417 "Tried to free memory, but couldn't find it in the memzone list\n");
420 LIST_REMOVE(mze, entries);
421 rte_spinlock_unlock(&enic->memzone_list_lock);
422 rte_memzone_free(mze->rz);
426 int enic_link_update(struct rte_eth_dev *eth_dev)
428 struct enic *enic = pmd_priv(eth_dev);
429 struct rte_eth_link link;
431 memset(&link, 0, sizeof(link));
432 link.link_status = enic_get_link_status(enic);
433 link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX;
434 link.link_speed = vnic_dev_port_speed(enic->vdev);
436 return rte_eth_linkstatus_set(eth_dev, &link);
440 enic_intr_handler(void *arg)
442 struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
443 struct enic *enic = pmd_priv(dev);
445 vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
447 enic_link_update(dev);
448 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
449 enic_log_q_error(enic);
450 /* Re-enable irq in case of INTx */
451 rte_intr_ack(enic->pdev->intr_handle);
454 static int enic_rxq_intr_init(struct enic *enic)
456 struct rte_intr_handle *intr_handle;
457 uint32_t rxq_intr_count, i;
460 intr_handle = enic->rte_dev->intr_handle;
461 if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
464 * Rx queue interrupts only work when we have MSI-X interrupts,
465 * one per queue. Sharing one interrupt is technically
466 * possible with VIC, but it is not worth the complications it brings.
468 if (!rte_intr_cap_multiple(intr_handle)) {
469 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
470 " (vfio-pci driver)\n");
473 rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
474 err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
476 dev_err(enic, "Failed to enable event fds for Rx queue"
481 if (rte_intr_vec_list_alloc(intr_handle, "enic_intr_vec",
483 dev_err(enic, "Failed to allocate intr_vec\n");
486 for (i = 0; i < rxq_intr_count; i++)
487 if (rte_intr_vec_list_index_set(intr_handle, i,
488 i + ENICPMD_RXQ_INTR_OFFSET))
493 static void enic_rxq_intr_deinit(struct enic *enic)
495 struct rte_intr_handle *intr_handle;
497 intr_handle = enic->rte_dev->intr_handle;
498 rte_intr_efd_disable(intr_handle);
500 rte_intr_vec_list_free(intr_handle);
503 static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
505 struct wq_enet_desc *desc;
510 * Fill WQ descriptor fields that never change. Every descriptor is
511 * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
512 * descriptors (i.e. request one completion update every 32 packets).
514 wq = &enic->wq[queue_idx];
515 desc = (struct wq_enet_desc *)wq->ring.descs;
516 for (i = 0; i < wq->ring.desc_count; i++, desc++) {
517 desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
518 if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
519 desc->header_length_flags |=
520 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
525 * The 'strong' version is in enic_rxtx_vec_avx2.c. This weak version is used
526 * used when that file is not compiled.
529 enic_use_vector_rx_handler(__rte_unused struct rte_eth_dev *eth_dev)
534 void enic_pick_rx_handler(struct rte_eth_dev *eth_dev)
536 struct enic *enic = pmd_priv(eth_dev);
539 ENICPMD_LOG(DEBUG, " use the normal Rx handler for 64B CQ entry");
540 eth_dev->rx_pkt_burst = &enic_recv_pkts_64;
545 * 1. The vectorized handler if possible and requested.
546 * 2. The non-scatter, simplified handler if scatter Rx is not used.
547 * 3. The default handler as a fallback.
549 if (enic_use_vector_rx_handler(eth_dev))
551 if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
552 ENICPMD_LOG(DEBUG, " use the non-scatter Rx handler");
553 eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
555 ENICPMD_LOG(DEBUG, " use the normal Rx handler");
556 eth_dev->rx_pkt_burst = &enic_recv_pkts;
560 /* Secondary process uses this to set the Tx handler */
561 void enic_pick_tx_handler(struct rte_eth_dev *eth_dev)
563 struct enic *enic = pmd_priv(eth_dev);
565 if (enic->use_simple_tx_handler) {
566 ENICPMD_LOG(DEBUG, " use the simple tx handler");
567 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
569 ENICPMD_LOG(DEBUG, " use the default tx handler");
570 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
574 int enic_enable(struct enic *enic)
578 struct rte_eth_dev *eth_dev = enic->rte_dev;
579 uint64_t simple_tx_offloads;
582 if (enic->enable_avx2_rx) {
583 struct rte_mbuf mb_def = { .buf_addr = 0 };
586 * mbuf_initializer contains const-after-init fields of
587 * receive mbufs (i.e. 64 bits of fields from rearm_data).
588 * It is currently used by the vectorized handler.
591 mb_def.data_off = RTE_PKTMBUF_HEADROOM;
592 mb_def.port = enic->port_id;
593 rte_mbuf_refcnt_set(&mb_def, 1);
594 rte_compiler_barrier();
595 p = (uintptr_t)&mb_def.rearm_data;
596 enic->mbuf_initializer = *(uint64_t *)p;
599 eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
600 eth_dev->data->dev_link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX;
602 /* vnic notification of link status has already been turned on in
603 * enic_dev_init() which is called during probe time. Here we are
604 * just turning on interrupt vector 0 if needed.
606 if (eth_dev->data->dev_conf.intr_conf.lsc)
607 vnic_dev_notify_set(enic->vdev, 0);
609 err = enic_rxq_intr_init(enic);
613 /* Initialize flowman if not already initialized during probe */
614 if (enic->fm == NULL && enic_fm_init(enic))
615 dev_warning(enic, "Init of flowman failed.\n");
617 for (index = 0; index < enic->rq_count; index++) {
618 err = enic_alloc_rx_queue_mbufs(enic,
619 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
621 dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
624 err = enic_alloc_rx_queue_mbufs(enic,
625 &enic->rq[enic_rte_rq_idx_to_data_idx(index, enic)]);
627 /* release the allocated mbufs for the sop rq*/
628 enic_rxmbuf_queue_release(enic,
629 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
631 dev_err(enic, "Failed to alloc data RX queue mbufs\n");
637 * Use the simple TX handler if possible. Only checksum offloads
638 * and vlan insertion are supported.
640 simple_tx_offloads = enic->tx_offload_capa &
641 (RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM |
642 RTE_ETH_TX_OFFLOAD_VLAN_INSERT |
643 RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
644 RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
645 RTE_ETH_TX_OFFLOAD_TCP_CKSUM);
646 if ((eth_dev->data->dev_conf.txmode.offloads &
647 ~simple_tx_offloads) == 0) {
648 ENICPMD_LOG(DEBUG, " use the simple tx handler");
649 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
650 for (index = 0; index < enic->wq_count; index++)
651 enic_prep_wq_for_simple_tx(enic, index);
652 enic->use_simple_tx_handler = 1;
654 ENICPMD_LOG(DEBUG, " use the default tx handler");
655 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
658 enic_pick_rx_handler(eth_dev);
660 for (index = 0; index < enic->wq_count; index++)
661 enic_start_wq(enic, index);
662 for (index = 0; index < enic->rq_count; index++)
663 enic_start_rq(enic, index);
665 vnic_dev_add_addr(enic->vdev, enic->mac_addr);
667 vnic_dev_enable_wait(enic->vdev);
669 /* Register and enable error interrupt */
670 rte_intr_callback_register(enic->pdev->intr_handle,
671 enic_intr_handler, (void *)enic->rte_dev);
673 rte_intr_enable(enic->pdev->intr_handle);
674 /* Unmask LSC interrupt */
675 vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
680 int enic_alloc_intr_resources(struct enic *enic)
685 dev_info(enic, "vNIC resources used: "\
686 "wq %d rq %d cq %d intr %d\n",
687 enic->wq_count, enic_vnic_rq_count(enic),
688 enic->cq_count, enic->intr_count);
690 for (i = 0; i < enic->intr_count; i++) {
691 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
693 enic_free_vnic_resources(enic);
700 void enic_free_rq(void *rxq)
702 struct vnic_rq *rq_sop, *rq_data;
708 rq_sop = (struct vnic_rq *)rxq;
709 enic = vnic_dev_priv(rq_sop->vdev);
710 rq_data = &enic->rq[rq_sop->data_queue_idx];
712 if (rq_sop->free_mbufs) {
713 struct rte_mbuf **mb;
716 mb = rq_sop->free_mbufs;
717 for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs;
718 i < ENIC_RX_BURST_MAX; i++)
719 rte_pktmbuf_free(mb[i]);
720 rte_free(rq_sop->free_mbufs);
721 rq_sop->free_mbufs = NULL;
722 rq_sop->num_free_mbufs = 0;
725 enic_rxmbuf_queue_release(enic, rq_sop);
727 enic_rxmbuf_queue_release(enic, rq_data);
729 rte_free(rq_sop->mbuf_ring);
731 rte_free(rq_data->mbuf_ring);
733 rq_sop->mbuf_ring = NULL;
734 rq_data->mbuf_ring = NULL;
736 vnic_rq_free(rq_sop);
738 vnic_rq_free(rq_data);
740 vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
746 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
748 struct rte_eth_dev_data *data = enic->dev_data;
749 vnic_wq_enable(&enic->wq[queue_idx]);
750 data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
753 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
755 struct rte_eth_dev_data *data = enic->dev_data;
758 ret = vnic_wq_disable(&enic->wq[queue_idx]);
762 data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
766 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
768 struct rte_eth_dev_data *data = enic->dev_data;
769 struct vnic_rq *rq_sop;
770 struct vnic_rq *rq_data;
771 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
772 rq_data = &enic->rq[rq_sop->data_queue_idx];
774 if (rq_data->in_use) {
775 vnic_rq_enable(rq_data);
776 enic_initial_post_rx(enic, rq_data);
779 vnic_rq_enable(rq_sop);
780 enic_initial_post_rx(enic, rq_sop);
781 data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
784 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
786 struct rte_eth_dev_data *data = enic->dev_data;
787 int ret1 = 0, ret2 = 0;
788 struct vnic_rq *rq_sop;
789 struct vnic_rq *rq_data;
790 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
791 rq_data = &enic->rq[rq_sop->data_queue_idx];
793 ret2 = vnic_rq_disable(rq_sop);
796 ret1 = vnic_rq_disable(rq_data);
803 data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
807 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
808 unsigned int socket_id, struct rte_mempool *mp,
809 uint16_t nb_desc, uint16_t free_thresh)
811 struct enic_vf_representor *vf;
813 uint16_t sop_queue_idx;
814 uint16_t data_queue_idx;
816 struct vnic_rq *rq_sop;
817 struct vnic_rq *rq_data;
818 unsigned int mbuf_size, mbufs_per_pkt;
819 unsigned int nb_sop_desc, nb_data_desc;
820 uint16_t min_sop, max_sop, min_data, max_data;
821 uint32_t max_rx_pktlen;
824 * Representor uses a reserved PF queue. Translate representor
825 * queue number to PF queue number.
827 if (enic_is_vf_rep(enic)) {
828 RTE_ASSERT(queue_idx == 0);
829 vf = VF_ENIC_TO_VF_REP(enic);
830 sop_queue_idx = vf->pf_rq_sop_idx;
831 data_queue_idx = vf->pf_rq_data_idx;
833 queue_idx = sop_queue_idx;
835 sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
836 data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx, enic);
838 cq_idx = enic_cq_rq(enic, sop_queue_idx);
839 rq_sop = &enic->rq[sop_queue_idx];
840 rq_data = &enic->rq[data_queue_idx];
842 rq_sop->data_queue_idx = data_queue_idx;
844 rq_data->data_queue_idx = 0;
845 rq_sop->socket_id = socket_id;
847 rq_data->socket_id = socket_id;
850 rq_sop->rx_free_thresh = free_thresh;
851 rq_data->rx_free_thresh = free_thresh;
852 dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
855 mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
856 RTE_PKTMBUF_HEADROOM);
857 /* max_rx_pktlen includes the ethernet header and CRC. */
858 max_rx_pktlen = enic_mtu_to_max_rx_pktlen(enic->rte_dev->data->mtu);
860 if (enic->rte_dev->data->dev_conf.rxmode.offloads &
861 RTE_ETH_RX_OFFLOAD_SCATTER) {
862 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
863 /* ceil((max pkt len)/mbuf_size) */
864 mbufs_per_pkt = (max_rx_pktlen + mbuf_size - 1) / mbuf_size;
866 dev_info(enic, "Scatter rx mode disabled\n");
868 if (max_rx_pktlen > mbuf_size) {
869 dev_warning(enic, "The maximum Rx packet size (%u) is"
870 " larger than the mbuf size (%u), and"
871 " scatter is disabled. Larger packets will"
873 max_rx_pktlen, mbuf_size);
877 if (mbufs_per_pkt > 1) {
878 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
879 rq_sop->data_queue_enable = 1;
882 * HW does not directly support MTU. HW always
883 * receives packet sizes up to the "max" MTU.
884 * If not using scatter, we can achieve the effect of dropping
885 * larger packets by reducing the size of posted buffers.
886 * See enic_alloc_rx_queue_mbufs().
888 if (enic->rte_dev->data->mtu < enic->max_mtu) {
890 "mtu is ignored when scatter rx mode is in use.\n");
893 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
895 rq_sop->data_queue_enable = 0;
899 /* number of descriptors have to be a multiple of 32 */
900 nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK;
901 nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK;
903 rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
904 rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
906 if (mbufs_per_pkt > 1) {
907 min_sop = ENIC_RX_BURST_MAX;
908 max_sop = ((enic->config.rq_desc_count /
909 (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK);
910 min_data = min_sop * (mbufs_per_pkt - 1);
911 max_data = enic->config.rq_desc_count;
913 min_sop = ENIC_RX_BURST_MAX;
914 max_sop = enic->config.rq_desc_count;
919 if (nb_desc < (min_sop + min_data)) {
921 "Number of rx descs too low, adjusting to minimum\n");
922 nb_sop_desc = min_sop;
923 nb_data_desc = min_data;
924 } else if (nb_desc > (max_sop + max_data)) {
926 "Number of rx_descs too high, adjusting to maximum\n");
927 nb_sop_desc = max_sop;
928 nb_data_desc = max_data;
930 if (mbufs_per_pkt > 1) {
931 dev_info(enic, "For max packet size %u and mbuf size %u valid"
932 " rx descriptor range is %u to %u\n",
933 max_rx_pktlen, mbuf_size, min_sop + min_data,
936 dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
937 nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
939 /* Allocate sop queue resources */
940 rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
941 nb_sop_desc, sizeof(struct rq_enet_desc));
943 dev_err(enic, "error in allocation of sop rq\n");
946 nb_sop_desc = rq_sop->ring.desc_count;
948 if (rq_data->in_use) {
949 /* Allocate data queue resources */
950 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
952 sizeof(struct rq_enet_desc));
954 dev_err(enic, "error in allocation of data rq\n");
955 goto err_free_rq_sop;
957 nb_data_desc = rq_data->ring.desc_count;
959 /* Enable 64B CQ entry if requested */
960 if (enic->cq64 && vnic_dev_set_cq_entry_size(enic->vdev,
961 sop_queue_idx, VNIC_RQ_CQ_ENTRY_SIZE_64)) {
962 dev_err(enic, "failed to enable 64B CQ entry on sop rq\n");
963 goto err_free_rq_data;
965 if (rq_data->in_use && enic->cq64 &&
966 vnic_dev_set_cq_entry_size(enic->vdev, data_queue_idx,
967 VNIC_RQ_CQ_ENTRY_SIZE_64)) {
968 dev_err(enic, "failed to enable 64B CQ entry on data rq\n");
969 goto err_free_rq_data;
972 rc = vnic_cq_alloc(enic->vdev, &enic->cq[cq_idx], cq_idx,
973 socket_id, nb_sop_desc + nb_data_desc,
974 enic->cq64 ? sizeof(struct cq_enet_rq_desc_64) :
975 sizeof(struct cq_enet_rq_desc));
977 dev_err(enic, "error in allocation of cq for rq\n");
978 goto err_free_rq_data;
981 /* Allocate the mbuf rings */
982 rq_sop->mbuf_ring = (struct rte_mbuf **)
983 rte_zmalloc_socket("rq->mbuf_ring",
984 sizeof(struct rte_mbuf *) * nb_sop_desc,
985 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
986 if (rq_sop->mbuf_ring == NULL)
989 if (rq_data->in_use) {
990 rq_data->mbuf_ring = (struct rte_mbuf **)
991 rte_zmalloc_socket("rq->mbuf_ring",
992 sizeof(struct rte_mbuf *) * nb_data_desc,
993 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
994 if (rq_data->mbuf_ring == NULL)
995 goto err_free_sop_mbuf;
998 rq_sop->free_mbufs = (struct rte_mbuf **)
999 rte_zmalloc_socket("rq->free_mbufs",
1000 sizeof(struct rte_mbuf *) *
1002 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
1003 if (rq_sop->free_mbufs == NULL)
1004 goto err_free_data_mbuf;
1005 rq_sop->num_free_mbufs = 0;
1007 rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
1012 rte_free(rq_data->mbuf_ring);
1014 rte_free(rq_sop->mbuf_ring);
1016 /* cleanup on error */
1017 vnic_cq_free(&enic->cq[cq_idx]);
1019 if (rq_data->in_use)
1020 vnic_rq_free(rq_data);
1022 vnic_rq_free(rq_sop);
1027 void enic_free_wq(void *txq)
1035 wq = (struct vnic_wq *)txq;
1036 enic = vnic_dev_priv(wq->vdev);
1037 rte_memzone_free(wq->cqmsg_rz);
1039 vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
1042 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
1043 unsigned int socket_id, uint16_t nb_desc)
1045 struct enic_vf_representor *vf;
1048 unsigned int cq_index;
1049 char name[RTE_MEMZONE_NAMESIZE];
1050 static int instance;
1053 * Representor uses a reserved PF queue. Translate representor
1054 * queue number to PF queue number.
1056 if (enic_is_vf_rep(enic)) {
1057 RTE_ASSERT(queue_idx == 0);
1058 vf = VF_ENIC_TO_VF_REP(enic);
1059 queue_idx = vf->pf_wq_idx;
1060 cq_index = vf->pf_wq_cq_idx;
1063 cq_index = enic_cq_wq(enic, queue_idx);
1065 wq = &enic->wq[queue_idx];
1066 wq->socket_id = socket_id;
1068 * rte_eth_tx_queue_setup() checks min, max, and alignment. So just
1069 * print an info message for diagnostics.
1071 dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc);
1073 /* Allocate queue resources */
1074 err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
1076 sizeof(struct wq_enet_desc));
1078 dev_err(enic, "error in allocation of wq\n");
1082 err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
1084 sizeof(struct cq_enet_wq_desc));
1087 dev_err(enic, "error in allocation of cq for wq\n");
1090 /* setup up CQ message */
1091 snprintf((char *)name, sizeof(name),
1092 "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
1095 wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
1096 sizeof(uint32_t), SOCKET_ID_ANY,
1097 RTE_MEMZONE_IOVA_CONTIG, ENIC_PAGE_SIZE);
1104 int enic_disable(struct enic *enic)
1109 for (i = 0; i < enic->intr_count; i++) {
1110 vnic_intr_mask(&enic->intr[i]);
1111 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
1113 enic_rxq_intr_deinit(enic);
1114 rte_intr_disable(enic->pdev->intr_handle);
1115 rte_intr_callback_unregister(enic->pdev->intr_handle,
1117 (void *)enic->rte_dev);
1119 vnic_dev_disable(enic->vdev);
1121 enic_fm_destroy(enic);
1123 if (!enic_is_sriov_vf(enic))
1124 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
1126 for (i = 0; i < enic->wq_count; i++) {
1127 err = vnic_wq_disable(&enic->wq[i]);
1131 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
1132 if (enic->rq[i].in_use) {
1133 err = vnic_rq_disable(&enic->rq[i]);
1139 /* If we were using interrupts, set the interrupt vector to -1
1140 * to disable interrupts. We are not disabling link notifications,
1141 * though, as we want the polling of link status to continue working.
1143 if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
1144 vnic_dev_notify_set(enic->vdev, -1);
1146 vnic_dev_set_reset_flag(enic->vdev, 1);
1148 for (i = 0; i < enic->wq_count; i++)
1149 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
1151 for (i = 0; i < enic_vnic_rq_count(enic); i++)
1152 if (enic->rq[i].in_use)
1153 vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
1154 for (i = 0; i < enic->cq_count; i++)
1155 vnic_cq_clean(&enic->cq[i]);
1156 for (i = 0; i < enic->intr_count; i++)
1157 vnic_intr_clean(&enic->intr[i]);
1162 static int enic_dev_wait(struct vnic_dev *vdev,
1163 int (*start)(struct vnic_dev *, int),
1164 int (*finished)(struct vnic_dev *, int *),
1171 err = start(vdev, arg);
1175 /* Wait for func to complete...2 seconds max */
1176 for (i = 0; i < 2000; i++) {
1177 err = finished(vdev, &done);
1187 static int enic_dev_open(struct enic *enic)
1190 int flags = CMD_OPENF_IG_DESCCACHE;
1192 err = enic_dev_wait(enic->vdev, vnic_dev_open,
1193 vnic_dev_open_done, flags);
1195 dev_err(enic_get_dev(enic),
1196 "vNIC device open failed, err %d\n", err);
1201 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
1203 dma_addr_t rss_key_buf_pa;
1204 union vnic_rss_key *rss_key_buf_va = NULL;
1206 uint8_t name[RTE_MEMZONE_NAMESIZE];
1208 RTE_ASSERT(user_key != NULL);
1209 snprintf((char *)name, sizeof(name), "rss_key-%s", enic->bdf_name);
1210 rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1211 &rss_key_buf_pa, name);
1212 if (!rss_key_buf_va)
1215 for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1216 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1218 err = enic_set_rss_key(enic,
1220 sizeof(union vnic_rss_key));
1222 /* Save for later queries */
1224 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1225 sizeof(union vnic_rss_key));
1227 enic_free_consistent(enic, sizeof(union vnic_rss_key),
1228 rss_key_buf_va, rss_key_buf_pa);
1233 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1235 dma_addr_t rss_cpu_buf_pa;
1236 union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1238 uint8_t name[RTE_MEMZONE_NAMESIZE];
1240 snprintf((char *)name, sizeof(name), "rss_cpu-%s", enic->bdf_name);
1241 rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1242 &rss_cpu_buf_pa, name);
1243 if (!rss_cpu_buf_va)
1246 rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1248 err = enic_set_rss_cpu(enic,
1250 sizeof(union vnic_rss_cpu));
1252 enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1253 rss_cpu_buf_va, rss_cpu_buf_pa);
1255 /* Save for later queries */
1257 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1261 static int enic_set_niccfg(struct enic *enic, uint8_t rss_default_cpu,
1262 uint8_t rss_hash_type, uint8_t rss_hash_bits, uint8_t rss_base_cpu,
1265 const uint8_t tso_ipid_split_en = 0;
1268 err = enic_set_nic_cfg(enic,
1269 rss_default_cpu, rss_hash_type,
1270 rss_hash_bits, rss_base_cpu,
1271 rss_enable, tso_ipid_split_en,
1272 enic->ig_vlan_strip_en);
1277 /* Initialize RSS with defaults, called from dev_configure */
1278 int enic_init_rss_nic_cfg(struct enic *enic)
1280 static uint8_t default_rss_key[] = {
1281 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1282 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1283 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1284 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1286 struct rte_eth_rss_conf rss_conf;
1287 union vnic_rss_cpu rss_cpu;
1290 rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1292 * If setting key for the first time, and the user gives us none, then
1293 * push the default key to NIC.
1295 if (rss_conf.rss_key == NULL) {
1296 rss_conf.rss_key = default_rss_key;
1297 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1299 ret = enic_set_rss_conf(enic, &rss_conf);
1301 dev_err(enic, "Failed to configure RSS\n");
1304 if (enic->rss_enable) {
1305 /* If enabling RSS, use the default reta */
1306 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1307 rss_cpu.cpu[i / 4].b[i % 4] =
1308 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1310 ret = enic_set_rss_reta(enic, &rss_cpu);
1312 dev_err(enic, "Failed to set RSS indirection table\n");
1317 int enic_setup_finish(struct enic *enic)
1319 enic_init_soft_stats(enic);
1321 /* switchdev: enable promisc mode on PF */
1322 if (enic->switchdev_mode) {
1323 vnic_dev_packet_filter(enic->vdev,
1334 vnic_dev_packet_filter(enic->vdev,
1347 static int enic_rss_conf_valid(struct enic *enic,
1348 struct rte_eth_rss_conf *rss_conf)
1350 /* RSS is disabled per VIC settings. Ignore rss_conf. */
1351 if (enic->flow_type_rss_offloads == 0)
1353 if (rss_conf->rss_key != NULL &&
1354 rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1355 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1356 rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1359 if (rss_conf->rss_hf != 0 &&
1360 (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1361 dev_err(enic, "Given rss_hf contains none of the supported"
1368 /* Set hash type and key according to rss_conf */
1369 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1371 struct rte_eth_dev *eth_dev;
1373 uint8_t rss_hash_type;
1377 RTE_ASSERT(rss_conf != NULL);
1378 ret = enic_rss_conf_valid(enic, rss_conf);
1380 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1384 eth_dev = enic->rte_dev;
1386 rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1387 if (enic->rq_count > 1 &&
1388 (eth_dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) &&
1391 if (rss_hf & (RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 |
1392 RTE_ETH_RSS_NONFRAG_IPV4_OTHER))
1393 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1394 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP)
1395 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1396 if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP) {
1397 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4;
1398 if (enic->udp_rss_weak) {
1400 * 'TCP' is not a typo. The "weak" version of
1401 * UDP RSS requires both the TCP and UDP bits
1402 * be set. It does enable TCP RSS as well.
1404 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1407 if (rss_hf & (RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_IPV6_EX |
1408 RTE_ETH_RSS_FRAG_IPV6 | RTE_ETH_RSS_NONFRAG_IPV6_OTHER))
1409 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1410 if (rss_hf & (RTE_ETH_RSS_NONFRAG_IPV6_TCP | RTE_ETH_RSS_IPV6_TCP_EX))
1411 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1412 if (rss_hf & (RTE_ETH_RSS_NONFRAG_IPV6_UDP | RTE_ETH_RSS_IPV6_UDP_EX)) {
1413 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6;
1414 if (enic->udp_rss_weak)
1415 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1422 /* Set the hash key if provided */
1423 if (rss_enable && rss_conf->rss_key) {
1424 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1426 dev_err(enic, "Failed to set RSS key\n");
1431 ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1432 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1435 enic->rss_hf = rss_hf;
1436 enic->rss_hash_type = rss_hash_type;
1437 enic->rss_enable = rss_enable;
1439 dev_err(enic, "Failed to update RSS configurations."
1440 " hash=0x%x\n", rss_hash_type);
1445 int enic_set_vlan_strip(struct enic *enic)
1448 * Unfortunately, VLAN strip on/off and RSS on/off are configured
1449 * together. So, re-do niccfg, preserving the current RSS settings.
1451 return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1452 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1456 int enic_add_packet_filter(struct enic *enic)
1458 /* switchdev ignores packet filters */
1459 if (enic->switchdev_mode) {
1460 ENICPMD_LOG(DEBUG, " switchdev: ignore packet filter");
1463 /* Args -> directed, multicast, broadcast, promisc, allmulti */
1464 return vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1465 enic->promisc, enic->allmulti);
1468 int enic_get_link_status(struct enic *enic)
1470 return vnic_dev_link_status(enic->vdev);
1473 static void enic_dev_deinit(struct enic *enic)
1475 /* stop link status checking */
1476 vnic_dev_notify_unset(enic->vdev);
1478 /* mac_addrs is freed by rte_eth_dev_release_port() */
1480 rte_free(enic->intr);
1486 int enic_set_vnic_res(struct enic *enic)
1488 struct rte_eth_dev *eth_dev = enic->rte_dev;
1490 unsigned int required_rq, required_wq, required_cq, required_intr;
1492 /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1493 required_rq = eth_dev->data->nb_rx_queues * 2;
1494 required_wq = eth_dev->data->nb_tx_queues;
1495 required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1496 required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1497 if (eth_dev->data->dev_conf.intr_conf.rxq) {
1498 required_intr += eth_dev->data->nb_rx_queues;
1500 ENICPMD_LOG(DEBUG, "Required queues for PF: rq %u wq %u cq %u",
1501 required_rq, required_wq, required_cq);
1502 if (enic->vf_required_rq) {
1503 /* Queues needed for VF representors */
1504 required_rq += enic->vf_required_rq;
1505 required_wq += enic->vf_required_wq;
1506 required_cq += enic->vf_required_cq;
1507 ENICPMD_LOG(DEBUG, "Required queues for VF representors: rq %u wq %u cq %u",
1508 enic->vf_required_rq, enic->vf_required_wq,
1509 enic->vf_required_cq);
1512 if (enic->conf_rq_count < required_rq) {
1513 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1514 eth_dev->data->nb_rx_queues,
1515 required_rq, enic->conf_rq_count);
1518 if (enic->conf_wq_count < required_wq) {
1519 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1520 eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1524 if (enic->conf_cq_count < required_cq) {
1525 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1526 required_cq, enic->conf_cq_count);
1529 if (enic->conf_intr_count < required_intr) {
1530 dev_err(dev, "Not enough Interrupts to support Rx queue"
1531 " interrupts. Required:%u, Configured:%u\n",
1532 required_intr, enic->conf_intr_count);
1537 enic->rq_count = eth_dev->data->nb_rx_queues;
1538 enic->wq_count = eth_dev->data->nb_tx_queues;
1539 enic->cq_count = enic->rq_count + enic->wq_count;
1540 enic->intr_count = required_intr;
1546 /* Initialize the completion queue for an RQ */
1548 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1550 struct vnic_rq *sop_rq, *data_rq;
1551 unsigned int cq_idx;
1554 sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1555 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx, enic)];
1556 cq_idx = enic_cq_rq(enic, rq_idx);
1558 vnic_cq_clean(&enic->cq[cq_idx]);
1559 vnic_cq_init(&enic->cq[cq_idx],
1560 0 /* flow_control_enable */,
1561 1 /* color_enable */,
1564 1 /* cq_tail_color */,
1565 0 /* interrupt_enable */,
1566 1 /* cq_entry_enable */,
1567 0 /* cq_message_enable */,
1568 0 /* interrupt offset */,
1569 0 /* cq_message_addr */);
1572 vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1573 enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1574 sop_rq->ring.desc_count - 1, 1, 0);
1575 if (data_rq->in_use) {
1576 vnic_rq_init_start(data_rq,
1578 enic_rte_rq_idx_to_data_idx(rq_idx, enic)),
1579 0, data_rq->ring.desc_count - 1, 1, 0);
1582 rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1586 if (data_rq->in_use) {
1587 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1589 enic_rxmbuf_queue_release(enic, sop_rq);
1597 /* The Cisco NIC can send and receive packets up to a max packet size
1598 * determined by the NIC type and firmware. There is also an MTU
1599 * configured into the NIC via the CIMC/UCSM management interface
1600 * which can be overridden by this function (up to the max packet size).
1601 * Depending on the network setup, doing so may cause packet drops
1602 * and unexpected behavior.
1604 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1606 unsigned int rq_idx;
1609 uint16_t old_mtu; /* previous setting */
1610 uint16_t config_mtu; /* Value configured into NIC via CIMC/UCSM */
1611 struct rte_eth_dev *eth_dev = enic->rte_dev;
1613 old_mtu = eth_dev->data->mtu;
1614 config_mtu = enic->config.mtu;
1616 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1617 return -E_RTE_SECONDARY;
1619 if (new_mtu > enic->max_mtu) {
1621 "MTU not updated: requested (%u) greater than max (%u)\n",
1622 new_mtu, enic->max_mtu);
1625 if (new_mtu < ENIC_MIN_MTU) {
1627 "MTU not updated: requested (%u) less than min (%u)\n",
1628 new_mtu, ENIC_MIN_MTU);
1631 if (new_mtu > config_mtu)
1633 "MTU (%u) is greater than value configured in NIC (%u)\n",
1634 new_mtu, config_mtu);
1637 * If the device has not started (enic_enable), nothing to do.
1638 * Later, enic_enable() will set up RQs reflecting the new maximum
1641 if (!eth_dev->data->dev_started)
1645 * The device has started, re-do RQs on the fly. In the process, we
1646 * pick up the new maximum packet length.
1648 * Some applications rely on the ability to change MTU without stopping
1649 * the device. So keep this behavior for now.
1651 rte_spinlock_lock(&enic->mtu_lock);
1653 /* Stop traffic on all RQs */
1654 for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1655 rq = &enic->rq[rq_idx];
1656 if (rq->is_sop && rq->in_use) {
1657 rc = enic_stop_rq(enic,
1658 enic_sop_rq_idx_to_rte_idx(rq_idx));
1660 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1666 /* replace Rx function with a no-op to avoid getting stale pkts */
1667 eth_dev->rx_pkt_burst = rte_eth_pkt_burst_dummy;
1668 rte_eth_fp_ops[enic->port_id].rx_pkt_burst = eth_dev->rx_pkt_burst;
1671 /* Allow time for threads to exit the real Rx function. */
1674 /* now it is safe to reconfigure the RQs */
1677 /* free and reallocate RQs with the new MTU */
1678 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1679 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1684 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1685 rq->tot_nb_desc, rq->rx_free_thresh);
1688 "Fatal MTU alloc error- No traffic will pass\n");
1692 rc = enic_reinit_rq(enic, rq_idx);
1695 "Fatal MTU RQ reinit- No traffic will pass\n");
1700 /* put back the real receive function */
1702 enic_pick_rx_handler(eth_dev);
1703 rte_eth_fp_ops[enic->port_id].rx_pkt_burst = eth_dev->rx_pkt_burst;
1706 /* restart Rx traffic */
1707 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1708 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1709 if (rq->is_sop && rq->in_use)
1710 enic_start_rq(enic, rq_idx);
1714 dev_info(enic, "MTU changed from %u to %u\n", old_mtu, new_mtu);
1715 rte_spinlock_unlock(&enic->mtu_lock);
1720 enic_disable_overlay_offload(struct enic *enic)
1723 * Disabling fails if the feature is provisioned but
1724 * not enabled. So ignore result and do not log error.
1727 vnic_dev_overlay_offload_ctrl(enic->vdev,
1728 OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_DISABLE);
1731 vnic_dev_overlay_offload_ctrl(enic->vdev,
1732 OVERLAY_FEATURE_GENEVE, OVERLAY_OFFLOAD_DISABLE);
1737 enic_enable_overlay_offload(struct enic *enic)
1739 if (enic->vxlan && vnic_dev_overlay_offload_ctrl(enic->vdev,
1740 OVERLAY_FEATURE_VXLAN, OVERLAY_OFFLOAD_ENABLE) != 0) {
1741 dev_err(NULL, "failed to enable VXLAN offload\n");
1744 if (enic->geneve && vnic_dev_overlay_offload_ctrl(enic->vdev,
1745 OVERLAY_FEATURE_GENEVE, OVERLAY_OFFLOAD_ENABLE) != 0) {
1746 dev_err(NULL, "failed to enable Geneve offload\n");
1749 enic->tx_offload_capa |=
1750 RTE_ETH_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1751 (enic->geneve ? RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO : 0) |
1752 (enic->vxlan ? RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO : 0);
1753 enic->tx_offload_mask |=
1754 RTE_MBUF_F_TX_OUTER_IPV6 |
1755 RTE_MBUF_F_TX_OUTER_IPV4 |
1756 RTE_MBUF_F_TX_OUTER_IP_CKSUM |
1757 RTE_MBUF_F_TX_TUNNEL_MASK;
1758 enic->overlay_offload = true;
1760 if (enic->vxlan && enic->geneve)
1761 dev_info(NULL, "Overlay offload is enabled (VxLAN, Geneve)\n");
1762 else if (enic->vxlan)
1763 dev_info(NULL, "Overlay offload is enabled (VxLAN)\n");
1765 dev_info(NULL, "Overlay offload is enabled (Geneve)\n");
1771 enic_reset_overlay_port(struct enic *enic)
1774 enic->vxlan_port = RTE_VXLAN_DEFAULT_PORT;
1776 * Reset the vxlan port to the default, as the NIC firmware
1777 * does not reset it automatically and keeps the old setting.
1779 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1780 OVERLAY_CFG_VXLAN_PORT_UPDATE,
1781 RTE_VXLAN_DEFAULT_PORT)) {
1782 dev_err(enic, "failed to update vxlan port\n");
1787 enic->geneve_port = RTE_GENEVE_DEFAULT_PORT;
1788 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1789 OVERLAY_CFG_GENEVE_PORT_UPDATE,
1790 RTE_GENEVE_DEFAULT_PORT)) {
1791 dev_err(enic, "failed to update vxlan port\n");
1798 static int enic_dev_init(struct enic *enic)
1801 struct rte_eth_dev *eth_dev = enic->rte_dev;
1803 vnic_dev_intr_coal_timer_info_default(enic->vdev);
1805 /* Get vNIC configuration
1807 err = enic_get_vnic_config(enic);
1809 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1813 /* Get available resource counts */
1814 enic_get_res_counts(enic);
1815 if (enic->conf_rq_count == 1) {
1816 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1817 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1818 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1821 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1822 enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1823 enic->conf_cq_count, 8);
1824 enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1825 enic->conf_intr_count, 8);
1826 enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1827 enic->conf_rq_count, 8);
1828 enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1829 enic->conf_wq_count, 8);
1830 if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1831 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1834 if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1835 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1838 if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1839 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1842 if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1843 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1847 eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr",
1848 sizeof(struct rte_ether_addr) *
1849 ENIC_UNICAST_PERFECT_FILTERS, 0);
1850 if (!eth_dev->data->mac_addrs) {
1851 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1854 rte_ether_addr_copy((struct rte_ether_addr *)enic->mac_addr,
1855 eth_dev->data->mac_addrs);
1857 vnic_dev_set_reset_flag(enic->vdev, 0);
1859 LIST_INIT(&enic->flows);
1861 /* set up link status checking */
1862 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1864 enic->overlay_offload = false;
1866 * First, explicitly disable overlay offload as the setting is
1867 * sticky, and resetting vNIC may not disable it.
1869 enic_disable_overlay_offload(enic);
1870 /* Then, enable overlay offload according to vNIC flags */
1871 if (!enic->disable_overlay && (enic->vxlan || enic->geneve)) {
1872 err = enic_enable_overlay_offload(enic);
1874 dev_info(NULL, "failed to enable overlay offload\n");
1879 * Reset the vxlan/geneve port if HW parsing is available. It
1880 * is always enabled regardless of overlay offload
1883 err = enic_reset_overlay_port(enic);
1887 if (enic_fm_init(enic))
1888 dev_warning(enic, "Init of flowman failed.\n");
1892 static void lock_devcmd(void *priv)
1894 struct enic *enic = priv;
1896 rte_spinlock_lock(&enic->devcmd_lock);
1899 static void unlock_devcmd(void *priv)
1901 struct enic *enic = priv;
1903 rte_spinlock_unlock(&enic->devcmd_lock);
1906 int enic_probe(struct enic *enic)
1908 struct rte_pci_device *pdev = enic->pdev;
1911 dev_debug(enic, "Initializing ENIC PMD\n");
1913 /* if this is a secondary process the hardware is already initialized */
1914 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1917 enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1918 enic->bar0.len = pdev->mem_resource[0].len;
1920 /* Register vNIC device */
1921 enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1923 dev_err(enic, "vNIC registration failed, aborting\n");
1927 LIST_INIT(&enic->memzone_list);
1928 rte_spinlock_init(&enic->memzone_list_lock);
1930 vnic_register_cbacks(enic->vdev,
1931 enic_alloc_consistent,
1932 enic_free_consistent);
1935 * Allocate the consistent memory for stats upfront so both primary and
1936 * secondary processes can dump stats.
1938 err = vnic_dev_alloc_stats_mem(enic->vdev);
1940 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1941 goto err_out_unregister;
1943 /* Issue device open to get device in known state */
1944 err = enic_dev_open(enic);
1946 dev_err(enic, "vNIC dev open failed, aborting\n");
1947 goto err_out_unregister;
1950 /* Set ingress vlan rewrite mode before vnic initialization */
1951 dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n",
1952 enic->ig_vlan_rewrite_mode);
1953 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1954 enic->ig_vlan_rewrite_mode);
1957 "Failed to set ingress vlan rewrite mode, aborting.\n");
1958 goto err_out_dev_close;
1961 /* Issue device init to initialize the vnic-to-switch link.
1962 * We'll start with carrier off and wait for link UP
1963 * notification later to turn on carrier. We don't need
1964 * to wait here for the vnic-to-switch link initialization
1965 * to complete; link UP notification is the indication that
1966 * the process is complete.
1969 err = vnic_dev_init(enic->vdev, 0);
1971 dev_err(enic, "vNIC dev init failed, aborting\n");
1972 goto err_out_dev_close;
1975 err = enic_dev_init(enic);
1977 dev_err(enic, "Device initialization failed, aborting\n");
1978 goto err_out_dev_close;
1981 /* Use a PF spinlock to serialize devcmd from PF and VF representors */
1982 if (enic->switchdev_mode) {
1983 rte_spinlock_init(&enic->devcmd_lock);
1984 vnic_register_lock(enic->vdev, lock_devcmd, unlock_devcmd);
1989 vnic_dev_close(enic->vdev);
1991 vnic_dev_unregister(enic->vdev);
1996 void enic_remove(struct enic *enic)
1998 enic_dev_deinit(enic);
1999 vnic_dev_close(enic->vdev);
2000 vnic_dev_unregister(enic->vdev);