1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
13 #include <rte_bus_pci.h>
14 #include <rte_memzone.h>
15 #include <rte_malloc.h>
17 #include <rte_string_fns.h>
18 #include <rte_ethdev_driver.h>
20 #include "enic_compat.h"
22 #include "wq_enet_desc.h"
23 #include "rq_enet_desc.h"
24 #include "cq_enet_desc.h"
25 #include "vnic_enet.h"
30 #include "vnic_intr.h"
33 static inline int enic_is_sriov_vf(struct enic *enic)
35 return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
38 static int is_zero_addr(uint8_t *addr)
40 return !(addr[0] | addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
43 static int is_mcast_addr(uint8_t *addr)
48 static int is_eth_addr_valid(uint8_t *addr)
50 return !is_mcast_addr(addr) && !is_zero_addr(addr);
54 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
58 if (!rq || !rq->mbuf_ring) {
59 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
63 for (i = 0; i < rq->ring.desc_count; i++) {
64 if (rq->mbuf_ring[i]) {
65 rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
66 rq->mbuf_ring[i] = NULL;
71 void enic_free_wq_buf(struct rte_mbuf **buf)
73 struct rte_mbuf *mbuf = *buf;
75 rte_pktmbuf_free_seg(mbuf);
79 static void enic_log_q_error(struct enic *enic)
82 uint32_t error_status;
84 for (i = 0; i < enic->wq_count; i++) {
85 error_status = vnic_wq_error_status(&enic->wq[i]);
87 dev_err(enic, "WQ[%d] error_status %d\n", i,
91 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
92 if (!enic->rq[i].in_use)
94 error_status = vnic_rq_error_status(&enic->rq[i]);
96 dev_err(enic, "RQ[%d] error_status %d\n", i,
101 static void enic_clear_soft_stats(struct enic *enic)
103 struct enic_soft_stats *soft_stats = &enic->soft_stats;
104 rte_atomic64_clear(&soft_stats->rx_nombuf);
105 rte_atomic64_clear(&soft_stats->rx_packet_errors);
106 rte_atomic64_clear(&soft_stats->tx_oversized);
109 static void enic_init_soft_stats(struct enic *enic)
111 struct enic_soft_stats *soft_stats = &enic->soft_stats;
112 rte_atomic64_init(&soft_stats->rx_nombuf);
113 rte_atomic64_init(&soft_stats->rx_packet_errors);
114 rte_atomic64_init(&soft_stats->tx_oversized);
115 enic_clear_soft_stats(enic);
118 int enic_dev_stats_clear(struct enic *enic)
122 ret = vnic_dev_stats_clear(enic->vdev);
124 dev_err(enic, "Error in clearing stats\n");
127 enic_clear_soft_stats(enic);
132 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
134 struct vnic_stats *stats;
135 struct enic_soft_stats *soft_stats = &enic->soft_stats;
136 int64_t rx_truncated;
137 uint64_t rx_packet_errors;
138 int ret = vnic_dev_stats_dump(enic->vdev, &stats);
141 dev_err(enic, "Error in getting stats\n");
145 /* The number of truncated packets can only be calculated by
146 * subtracting a hardware counter from error packets received by
147 * the driver. Note: this causes transient inaccuracies in the
148 * ipackets count. Also, the length of truncated packets are
149 * counted in ibytes even though truncated packets are dropped
150 * which can make ibytes be slightly higher than it should be.
152 rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
153 rx_truncated = rx_packet_errors - stats->rx.rx_errors;
155 r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
156 r_stats->opackets = stats->tx.tx_frames_ok;
158 r_stats->ibytes = stats->rx.rx_bytes_ok;
159 r_stats->obytes = stats->tx.tx_bytes_ok;
161 r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
162 r_stats->oerrors = stats->tx.tx_errors
163 + rte_atomic64_read(&soft_stats->tx_oversized);
165 r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
167 r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
171 int enic_del_mac_address(struct enic *enic, int mac_index)
173 struct rte_eth_dev *eth_dev = enic->rte_dev;
174 uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
176 return vnic_dev_del_addr(enic->vdev, mac_addr);
179 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
183 if (!is_eth_addr_valid(mac_addr)) {
184 dev_err(enic, "invalid mac address\n");
188 err = vnic_dev_add_addr(enic->vdev, mac_addr);
190 dev_err(enic, "add mac addr failed\n");
194 void enic_free_rq_buf(struct rte_mbuf **mbuf)
199 rte_pktmbuf_free(*mbuf);
203 void enic_init_vnic_resources(struct enic *enic)
205 unsigned int error_interrupt_enable = 1;
206 unsigned int error_interrupt_offset = 0;
207 unsigned int rxq_interrupt_enable = 0;
208 unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
209 unsigned int index = 0;
211 struct vnic_rq *data_rq;
213 if (enic->rte_dev->data->dev_conf.intr_conf.rxq)
214 rxq_interrupt_enable = 1;
216 for (index = 0; index < enic->rq_count; index++) {
217 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
219 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
221 error_interrupt_enable,
222 error_interrupt_offset);
224 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index, enic)];
226 vnic_rq_init(data_rq,
228 error_interrupt_enable,
229 error_interrupt_offset);
230 vnic_cq_init(&enic->cq[cq_idx],
231 0 /* flow_control_enable */,
232 1 /* color_enable */,
235 1 /* cq_tail_color */,
236 rxq_interrupt_enable,
237 1 /* cq_entry_enable */,
238 0 /* cq_message_enable */,
239 rxq_interrupt_offset,
240 0 /* cq_message_addr */);
241 if (rxq_interrupt_enable)
242 rxq_interrupt_offset++;
245 for (index = 0; index < enic->wq_count; index++) {
246 vnic_wq_init(&enic->wq[index],
247 enic_cq_wq(enic, index),
248 error_interrupt_enable,
249 error_interrupt_offset);
250 /* Compute unsupported ol flags for enic_prep_pkts() */
251 enic->wq[index].tx_offload_notsup_mask =
252 PKT_TX_OFFLOAD_MASK ^ enic->tx_offload_mask;
254 cq_idx = enic_cq_wq(enic, index);
255 vnic_cq_init(&enic->cq[cq_idx],
256 0 /* flow_control_enable */,
257 1 /* color_enable */,
260 1 /* cq_tail_color */,
261 0 /* interrupt_enable */,
262 0 /* cq_entry_enable */,
263 1 /* cq_message_enable */,
264 0 /* interrupt offset */,
265 (uint64_t)enic->wq[index].cqmsg_rz->iova);
268 for (index = 0; index < enic->intr_count; index++) {
269 vnic_intr_init(&enic->intr[index],
270 enic->config.intr_timer_usec,
271 enic->config.intr_timer_type,
272 /*mask_on_assertion*/1);
278 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
281 struct rq_enet_desc *rqd = rq->ring.descs;
284 uint32_t max_rx_pkt_len;
290 dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
291 rq->ring.desc_count);
294 * If *not* using scatter and the mbuf size is greater than the
295 * requested max packet size (max_rx_pkt_len), then reduce the
296 * posted buffer size to max_rx_pkt_len. HW still receives packets
297 * larger than max_rx_pkt_len, but they will be truncated, which we
298 * drop in the rx handler. Not ideal, but better than returning
299 * large packets when the user is not expecting them.
301 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
302 rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
303 if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
304 rq_buf_len = max_rx_pkt_len;
305 for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
306 mb = rte_mbuf_raw_alloc(rq->mp);
308 dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
309 (unsigned)rq->index);
313 mb->data_off = RTE_PKTMBUF_HEADROOM;
314 dma_addr = (dma_addr_t)(mb->buf_iova
315 + RTE_PKTMBUF_HEADROOM);
316 rq_enet_desc_enc(rqd, dma_addr,
317 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
318 : RQ_ENET_TYPE_NOT_SOP),
320 rq->mbuf_ring[i] = mb;
323 * Do not post the buffers to the NIC until we enable the RQ via
326 rq->need_initial_post = true;
327 /* Initialize fetch index while RQ is disabled */
328 iowrite32(0, &rq->ctrl->fetch_index);
333 * Post the Rx buffers for the first time. enic_alloc_rx_queue_mbufs() has
334 * allocated the buffers and filled the RQ descriptor ring. Just need to push
335 * the post index to the NIC.
338 enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)
340 if (!rq->in_use || !rq->need_initial_post)
343 /* make sure all prior writes are complete before doing the PIO write */
346 /* Post all but the last buffer to VIC. */
347 rq->posted_index = rq->ring.desc_count - 1;
351 dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
352 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
353 iowrite32(rq->posted_index, &rq->ctrl->posted_index);
355 rq->need_initial_post = false;
359 enic_alloc_consistent(void *priv, size_t size,
360 dma_addr_t *dma_handle, uint8_t *name)
363 const struct rte_memzone *rz;
365 struct enic *enic = (struct enic *)priv;
366 struct enic_memzone_entry *mze;
368 rz = rte_memzone_reserve_aligned((const char *)name, size,
369 SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_PAGE_SIZE);
371 pr_err("%s : Failed to allocate memory requested for %s\n",
377 *dma_handle = (dma_addr_t)rz->iova;
379 mze = rte_malloc("enic memzone entry",
380 sizeof(struct enic_memzone_entry), 0);
383 pr_err("%s : Failed to allocate memory for memzone list\n",
385 rte_memzone_free(rz);
391 rte_spinlock_lock(&enic->memzone_list_lock);
392 LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
393 rte_spinlock_unlock(&enic->memzone_list_lock);
399 enic_free_consistent(void *priv,
400 __rte_unused size_t size,
402 dma_addr_t dma_handle)
404 struct enic_memzone_entry *mze;
405 struct enic *enic = (struct enic *)priv;
407 rte_spinlock_lock(&enic->memzone_list_lock);
408 LIST_FOREACH(mze, &enic->memzone_list, entries) {
409 if (mze->rz->addr == vaddr &&
410 mze->rz->iova == dma_handle)
414 rte_spinlock_unlock(&enic->memzone_list_lock);
416 "Tried to free memory, but couldn't find it in the memzone list\n");
419 LIST_REMOVE(mze, entries);
420 rte_spinlock_unlock(&enic->memzone_list_lock);
421 rte_memzone_free(mze->rz);
425 int enic_link_update(struct rte_eth_dev *eth_dev)
427 struct enic *enic = pmd_priv(eth_dev);
428 struct rte_eth_link link;
430 memset(&link, 0, sizeof(link));
431 link.link_status = enic_get_link_status(enic);
432 link.link_duplex = ETH_LINK_FULL_DUPLEX;
433 link.link_speed = vnic_dev_port_speed(enic->vdev);
435 return rte_eth_linkstatus_set(eth_dev, &link);
439 enic_intr_handler(void *arg)
441 struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
442 struct enic *enic = pmd_priv(dev);
444 vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
446 enic_link_update(dev);
447 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
448 enic_log_q_error(enic);
449 /* Re-enable irq in case of INTx */
450 rte_intr_ack(&enic->pdev->intr_handle);
453 static int enic_rxq_intr_init(struct enic *enic)
455 struct rte_intr_handle *intr_handle;
456 uint32_t rxq_intr_count, i;
459 intr_handle = enic->rte_dev->intr_handle;
460 if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
463 * Rx queue interrupts only work when we have MSI-X interrupts,
464 * one per queue. Sharing one interrupt is technically
465 * possible with VIC, but it is not worth the complications it brings.
467 if (!rte_intr_cap_multiple(intr_handle)) {
468 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
469 " (vfio-pci driver)\n");
472 rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
473 err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
475 dev_err(enic, "Failed to enable event fds for Rx queue"
479 intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
480 rxq_intr_count * sizeof(int), 0);
481 if (intr_handle->intr_vec == NULL) {
482 dev_err(enic, "Failed to allocate intr_vec\n");
485 for (i = 0; i < rxq_intr_count; i++)
486 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
490 static void enic_rxq_intr_deinit(struct enic *enic)
492 struct rte_intr_handle *intr_handle;
494 intr_handle = enic->rte_dev->intr_handle;
495 rte_intr_efd_disable(intr_handle);
496 if (intr_handle->intr_vec != NULL) {
497 rte_free(intr_handle->intr_vec);
498 intr_handle->intr_vec = NULL;
502 static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
504 struct wq_enet_desc *desc;
509 * Fill WQ descriptor fields that never change. Every descriptor is
510 * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
511 * descriptors (i.e. request one completion update every 32 packets).
513 wq = &enic->wq[queue_idx];
514 desc = (struct wq_enet_desc *)wq->ring.descs;
515 for (i = 0; i < wq->ring.desc_count; i++, desc++) {
516 desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
517 if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
518 desc->header_length_flags |=
519 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
524 * The 'strong' version is in enic_rxtx_vec_avx2.c. This weak version is used
525 * used when that file is not compiled.
528 enic_use_vector_rx_handler(__rte_unused struct rte_eth_dev *eth_dev)
533 void enic_pick_rx_handler(struct rte_eth_dev *eth_dev)
535 struct enic *enic = pmd_priv(eth_dev);
538 ENICPMD_LOG(DEBUG, " use the normal Rx handler for 64B CQ entry");
539 eth_dev->rx_pkt_burst = &enic_recv_pkts_64;
544 * 1. The vectorized handler if possible and requested.
545 * 2. The non-scatter, simplified handler if scatter Rx is not used.
546 * 3. The default handler as a fallback.
548 if (enic_use_vector_rx_handler(eth_dev))
550 if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
551 ENICPMD_LOG(DEBUG, " use the non-scatter Rx handler");
552 eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
554 ENICPMD_LOG(DEBUG, " use the normal Rx handler");
555 eth_dev->rx_pkt_burst = &enic_recv_pkts;
559 /* Secondary process uses this to set the Tx handler */
560 void enic_pick_tx_handler(struct rte_eth_dev *eth_dev)
562 struct enic *enic = pmd_priv(eth_dev);
564 if (enic->use_simple_tx_handler) {
565 ENICPMD_LOG(DEBUG, " use the simple tx handler");
566 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
568 ENICPMD_LOG(DEBUG, " use the default tx handler");
569 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
573 int enic_enable(struct enic *enic)
577 struct rte_eth_dev *eth_dev = enic->rte_dev;
578 uint64_t simple_tx_offloads;
581 if (enic->enable_avx2_rx) {
582 struct rte_mbuf mb_def = { .buf_addr = 0 };
585 * mbuf_initializer contains const-after-init fields of
586 * receive mbufs (i.e. 64 bits of fields from rearm_data).
587 * It is currently used by the vectorized handler.
590 mb_def.data_off = RTE_PKTMBUF_HEADROOM;
591 mb_def.port = enic->port_id;
592 rte_mbuf_refcnt_set(&mb_def, 1);
593 rte_compiler_barrier();
594 p = (uintptr_t)&mb_def.rearm_data;
595 enic->mbuf_initializer = *(uint64_t *)p;
598 eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
599 eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
601 /* vnic notification of link status has already been turned on in
602 * enic_dev_init() which is called during probe time. Here we are
603 * just turning on interrupt vector 0 if needed.
605 if (eth_dev->data->dev_conf.intr_conf.lsc)
606 vnic_dev_notify_set(enic->vdev, 0);
608 err = enic_rxq_intr_init(enic);
612 /* Initialize flowman if not already initialized during probe */
613 if (enic->fm == NULL && enic_fm_init(enic))
614 dev_warning(enic, "Init of flowman failed.\n");
616 for (index = 0; index < enic->rq_count; index++) {
617 err = enic_alloc_rx_queue_mbufs(enic,
618 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
620 dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
623 err = enic_alloc_rx_queue_mbufs(enic,
624 &enic->rq[enic_rte_rq_idx_to_data_idx(index, enic)]);
626 /* release the allocated mbufs for the sop rq*/
627 enic_rxmbuf_queue_release(enic,
628 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
630 dev_err(enic, "Failed to alloc data RX queue mbufs\n");
636 * Use the simple TX handler if possible. Only checksum offloads
637 * and vlan insertion are supported.
639 simple_tx_offloads = enic->tx_offload_capa &
640 (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
641 DEV_TX_OFFLOAD_VLAN_INSERT |
642 DEV_TX_OFFLOAD_IPV4_CKSUM |
643 DEV_TX_OFFLOAD_UDP_CKSUM |
644 DEV_TX_OFFLOAD_TCP_CKSUM);
645 if ((eth_dev->data->dev_conf.txmode.offloads &
646 ~simple_tx_offloads) == 0) {
647 ENICPMD_LOG(DEBUG, " use the simple tx handler");
648 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
649 for (index = 0; index < enic->wq_count; index++)
650 enic_prep_wq_for_simple_tx(enic, index);
651 enic->use_simple_tx_handler = 1;
653 ENICPMD_LOG(DEBUG, " use the default tx handler");
654 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
657 enic_pick_rx_handler(eth_dev);
659 for (index = 0; index < enic->wq_count; index++)
660 enic_start_wq(enic, index);
661 for (index = 0; index < enic->rq_count; index++)
662 enic_start_rq(enic, index);
664 vnic_dev_add_addr(enic->vdev, enic->mac_addr);
666 vnic_dev_enable_wait(enic->vdev);
668 /* Register and enable error interrupt */
669 rte_intr_callback_register(&(enic->pdev->intr_handle),
670 enic_intr_handler, (void *)enic->rte_dev);
672 rte_intr_enable(&(enic->pdev->intr_handle));
673 /* Unmask LSC interrupt */
674 vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
679 int enic_alloc_intr_resources(struct enic *enic)
684 dev_info(enic, "vNIC resources used: "\
685 "wq %d rq %d cq %d intr %d\n",
686 enic->wq_count, enic_vnic_rq_count(enic),
687 enic->cq_count, enic->intr_count);
689 for (i = 0; i < enic->intr_count; i++) {
690 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
692 enic_free_vnic_resources(enic);
699 void enic_free_rq(void *rxq)
701 struct vnic_rq *rq_sop, *rq_data;
707 rq_sop = (struct vnic_rq *)rxq;
708 enic = vnic_dev_priv(rq_sop->vdev);
709 rq_data = &enic->rq[rq_sop->data_queue_idx];
711 if (rq_sop->free_mbufs) {
712 struct rte_mbuf **mb;
715 mb = rq_sop->free_mbufs;
716 for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs;
717 i < ENIC_RX_BURST_MAX; i++)
718 rte_pktmbuf_free(mb[i]);
719 rte_free(rq_sop->free_mbufs);
720 rq_sop->free_mbufs = NULL;
721 rq_sop->num_free_mbufs = 0;
724 enic_rxmbuf_queue_release(enic, rq_sop);
726 enic_rxmbuf_queue_release(enic, rq_data);
728 rte_free(rq_sop->mbuf_ring);
730 rte_free(rq_data->mbuf_ring);
732 rq_sop->mbuf_ring = NULL;
733 rq_data->mbuf_ring = NULL;
735 vnic_rq_free(rq_sop);
737 vnic_rq_free(rq_data);
739 vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
745 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
747 struct rte_eth_dev_data *data = enic->dev_data;
748 vnic_wq_enable(&enic->wq[queue_idx]);
749 data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
752 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
754 struct rte_eth_dev_data *data = enic->dev_data;
757 ret = vnic_wq_disable(&enic->wq[queue_idx]);
761 data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
765 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
767 struct rte_eth_dev_data *data = enic->dev_data;
768 struct vnic_rq *rq_sop;
769 struct vnic_rq *rq_data;
770 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
771 rq_data = &enic->rq[rq_sop->data_queue_idx];
773 if (rq_data->in_use) {
774 vnic_rq_enable(rq_data);
775 enic_initial_post_rx(enic, rq_data);
778 vnic_rq_enable(rq_sop);
779 enic_initial_post_rx(enic, rq_sop);
780 data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
783 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
785 struct rte_eth_dev_data *data = enic->dev_data;
786 int ret1 = 0, ret2 = 0;
787 struct vnic_rq *rq_sop;
788 struct vnic_rq *rq_data;
789 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
790 rq_data = &enic->rq[rq_sop->data_queue_idx];
792 ret2 = vnic_rq_disable(rq_sop);
795 ret1 = vnic_rq_disable(rq_data);
802 data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
806 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
807 unsigned int socket_id, struct rte_mempool *mp,
808 uint16_t nb_desc, uint16_t free_thresh)
810 struct enic_vf_representor *vf;
812 uint16_t sop_queue_idx;
813 uint16_t data_queue_idx;
815 struct vnic_rq *rq_sop;
816 struct vnic_rq *rq_data;
817 unsigned int mbuf_size, mbufs_per_pkt;
818 unsigned int nb_sop_desc, nb_data_desc;
819 uint16_t min_sop, max_sop, min_data, max_data;
820 uint32_t max_rx_pkt_len;
823 * Representor uses a reserved PF queue. Translate representor
824 * queue number to PF queue number.
826 if (enic_is_vf_rep(enic)) {
827 RTE_ASSERT(queue_idx == 0);
828 vf = VF_ENIC_TO_VF_REP(enic);
829 sop_queue_idx = vf->pf_rq_sop_idx;
830 data_queue_idx = vf->pf_rq_data_idx;
832 queue_idx = sop_queue_idx;
834 sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
835 data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx, enic);
837 cq_idx = enic_cq_rq(enic, sop_queue_idx);
838 rq_sop = &enic->rq[sop_queue_idx];
839 rq_data = &enic->rq[data_queue_idx];
841 rq_sop->data_queue_idx = data_queue_idx;
843 rq_data->data_queue_idx = 0;
844 rq_sop->socket_id = socket_id;
846 rq_data->socket_id = socket_id;
849 rq_sop->rx_free_thresh = free_thresh;
850 rq_data->rx_free_thresh = free_thresh;
851 dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
854 mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
855 RTE_PKTMBUF_HEADROOM);
856 /* max_rx_pkt_len includes the ethernet header and CRC. */
857 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
859 if (enic->rte_dev->data->dev_conf.rxmode.offloads &
860 DEV_RX_OFFLOAD_SCATTER) {
861 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
862 /* ceil((max pkt len)/mbuf_size) */
863 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
865 dev_info(enic, "Scatter rx mode disabled\n");
867 if (max_rx_pkt_len > mbuf_size) {
868 dev_warning(enic, "The maximum Rx packet size (%u) is"
869 " larger than the mbuf size (%u), and"
870 " scatter is disabled. Larger packets will"
872 max_rx_pkt_len, mbuf_size);
876 if (mbufs_per_pkt > 1) {
877 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
878 rq_sop->data_queue_enable = 1;
881 * HW does not directly support rxmode.max_rx_pkt_len. HW always
882 * receives packet sizes up to the "max" MTU.
883 * If not using scatter, we can achieve the effect of dropping
884 * larger packets by reducing the size of posted buffers.
885 * See enic_alloc_rx_queue_mbufs().
888 enic_mtu_to_max_rx_pktlen(enic->max_mtu)) {
889 dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
890 " when scatter rx mode is in use.\n");
893 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
895 rq_sop->data_queue_enable = 0;
899 /* number of descriptors have to be a multiple of 32 */
900 nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK;
901 nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK;
903 rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
904 rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
906 if (mbufs_per_pkt > 1) {
907 min_sop = ENIC_RX_BURST_MAX;
908 max_sop = ((enic->config.rq_desc_count /
909 (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK);
910 min_data = min_sop * (mbufs_per_pkt - 1);
911 max_data = enic->config.rq_desc_count;
913 min_sop = ENIC_RX_BURST_MAX;
914 max_sop = enic->config.rq_desc_count;
919 if (nb_desc < (min_sop + min_data)) {
921 "Number of rx descs too low, adjusting to minimum\n");
922 nb_sop_desc = min_sop;
923 nb_data_desc = min_data;
924 } else if (nb_desc > (max_sop + max_data)) {
926 "Number of rx_descs too high, adjusting to maximum\n");
927 nb_sop_desc = max_sop;
928 nb_data_desc = max_data;
930 if (mbufs_per_pkt > 1) {
931 dev_info(enic, "For max packet size %u and mbuf size %u valid"
932 " rx descriptor range is %u to %u\n",
933 max_rx_pkt_len, mbuf_size, min_sop + min_data,
936 dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
937 nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
939 /* Allocate sop queue resources */
940 rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
941 nb_sop_desc, sizeof(struct rq_enet_desc));
943 dev_err(enic, "error in allocation of sop rq\n");
946 nb_sop_desc = rq_sop->ring.desc_count;
948 if (rq_data->in_use) {
949 /* Allocate data queue resources */
950 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
952 sizeof(struct rq_enet_desc));
954 dev_err(enic, "error in allocation of data rq\n");
955 goto err_free_rq_sop;
957 nb_data_desc = rq_data->ring.desc_count;
959 /* Enable 64B CQ entry if requested */
960 if (enic->cq64 && vnic_dev_set_cq_entry_size(enic->vdev,
961 sop_queue_idx, VNIC_RQ_CQ_ENTRY_SIZE_64)) {
962 dev_err(enic, "failed to enable 64B CQ entry on sop rq\n");
963 goto err_free_rq_data;
965 if (rq_data->in_use && enic->cq64 &&
966 vnic_dev_set_cq_entry_size(enic->vdev, data_queue_idx,
967 VNIC_RQ_CQ_ENTRY_SIZE_64)) {
968 dev_err(enic, "failed to enable 64B CQ entry on data rq\n");
969 goto err_free_rq_data;
972 rc = vnic_cq_alloc(enic->vdev, &enic->cq[cq_idx], cq_idx,
973 socket_id, nb_sop_desc + nb_data_desc,
974 enic->cq64 ? sizeof(struct cq_enet_rq_desc_64) :
975 sizeof(struct cq_enet_rq_desc));
977 dev_err(enic, "error in allocation of cq for rq\n");
978 goto err_free_rq_data;
981 /* Allocate the mbuf rings */
982 rq_sop->mbuf_ring = (struct rte_mbuf **)
983 rte_zmalloc_socket("rq->mbuf_ring",
984 sizeof(struct rte_mbuf *) * nb_sop_desc,
985 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
986 if (rq_sop->mbuf_ring == NULL)
989 if (rq_data->in_use) {
990 rq_data->mbuf_ring = (struct rte_mbuf **)
991 rte_zmalloc_socket("rq->mbuf_ring",
992 sizeof(struct rte_mbuf *) * nb_data_desc,
993 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
994 if (rq_data->mbuf_ring == NULL)
995 goto err_free_sop_mbuf;
998 rq_sop->free_mbufs = (struct rte_mbuf **)
999 rte_zmalloc_socket("rq->free_mbufs",
1000 sizeof(struct rte_mbuf *) *
1002 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
1003 if (rq_sop->free_mbufs == NULL)
1004 goto err_free_data_mbuf;
1005 rq_sop->num_free_mbufs = 0;
1007 rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
1012 rte_free(rq_data->mbuf_ring);
1014 rte_free(rq_sop->mbuf_ring);
1016 /* cleanup on error */
1017 vnic_cq_free(&enic->cq[cq_idx]);
1019 if (rq_data->in_use)
1020 vnic_rq_free(rq_data);
1022 vnic_rq_free(rq_sop);
1027 void enic_free_wq(void *txq)
1035 wq = (struct vnic_wq *)txq;
1036 enic = vnic_dev_priv(wq->vdev);
1037 rte_memzone_free(wq->cqmsg_rz);
1039 vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
1042 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
1043 unsigned int socket_id, uint16_t nb_desc)
1045 struct enic_vf_representor *vf;
1048 unsigned int cq_index;
1049 char name[RTE_MEMZONE_NAMESIZE];
1050 static int instance;
1053 * Representor uses a reserved PF queue. Translate representor
1054 * queue number to PF queue number.
1056 if (enic_is_vf_rep(enic)) {
1057 RTE_ASSERT(queue_idx == 0);
1058 vf = VF_ENIC_TO_VF_REP(enic);
1059 queue_idx = vf->pf_wq_idx;
1060 cq_index = vf->pf_wq_cq_idx;
1063 cq_index = enic_cq_wq(enic, queue_idx);
1065 wq = &enic->wq[queue_idx];
1066 wq->socket_id = socket_id;
1068 * rte_eth_tx_queue_setup() checks min, max, and alignment. So just
1069 * print an info message for diagnostics.
1071 dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc);
1073 /* Allocate queue resources */
1074 err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
1076 sizeof(struct wq_enet_desc));
1078 dev_err(enic, "error in allocation of wq\n");
1082 err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
1084 sizeof(struct cq_enet_wq_desc));
1087 dev_err(enic, "error in allocation of cq for wq\n");
1090 /* setup up CQ message */
1091 snprintf((char *)name, sizeof(name),
1092 "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
1095 wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
1096 sizeof(uint32_t), SOCKET_ID_ANY,
1097 RTE_MEMZONE_IOVA_CONTIG, ENIC_PAGE_SIZE);
1104 int enic_disable(struct enic *enic)
1109 for (i = 0; i < enic->intr_count; i++) {
1110 vnic_intr_mask(&enic->intr[i]);
1111 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
1113 enic_rxq_intr_deinit(enic);
1114 rte_intr_disable(&enic->pdev->intr_handle);
1115 rte_intr_callback_unregister(&enic->pdev->intr_handle,
1117 (void *)enic->rte_dev);
1119 vnic_dev_disable(enic->vdev);
1121 enic_fm_destroy(enic);
1123 if (!enic_is_sriov_vf(enic))
1124 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
1126 for (i = 0; i < enic->wq_count; i++) {
1127 err = vnic_wq_disable(&enic->wq[i]);
1131 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
1132 if (enic->rq[i].in_use) {
1133 err = vnic_rq_disable(&enic->rq[i]);
1139 /* If we were using interrupts, set the interrupt vector to -1
1140 * to disable interrupts. We are not disabling link notifcations,
1141 * though, as we want the polling of link status to continue working.
1143 if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
1144 vnic_dev_notify_set(enic->vdev, -1);
1146 vnic_dev_set_reset_flag(enic->vdev, 1);
1148 for (i = 0; i < enic->wq_count; i++)
1149 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
1151 for (i = 0; i < enic_vnic_rq_count(enic); i++)
1152 if (enic->rq[i].in_use)
1153 vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
1154 for (i = 0; i < enic->cq_count; i++)
1155 vnic_cq_clean(&enic->cq[i]);
1156 for (i = 0; i < enic->intr_count; i++)
1157 vnic_intr_clean(&enic->intr[i]);
1162 static int enic_dev_wait(struct vnic_dev *vdev,
1163 int (*start)(struct vnic_dev *, int),
1164 int (*finished)(struct vnic_dev *, int *),
1171 err = start(vdev, arg);
1175 /* Wait for func to complete...2 seconds max */
1176 for (i = 0; i < 2000; i++) {
1177 err = finished(vdev, &done);
1187 static int enic_dev_open(struct enic *enic)
1190 int flags = CMD_OPENF_IG_DESCCACHE;
1192 err = enic_dev_wait(enic->vdev, vnic_dev_open,
1193 vnic_dev_open_done, flags);
1195 dev_err(enic_get_dev(enic),
1196 "vNIC device open failed, err %d\n", err);
1201 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
1203 dma_addr_t rss_key_buf_pa;
1204 union vnic_rss_key *rss_key_buf_va = NULL;
1206 uint8_t name[RTE_MEMZONE_NAMESIZE];
1208 RTE_ASSERT(user_key != NULL);
1209 snprintf((char *)name, sizeof(name), "rss_key-%s", enic->bdf_name);
1210 rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1211 &rss_key_buf_pa, name);
1212 if (!rss_key_buf_va)
1215 for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1216 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1218 err = enic_set_rss_key(enic,
1220 sizeof(union vnic_rss_key));
1222 /* Save for later queries */
1224 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1225 sizeof(union vnic_rss_key));
1227 enic_free_consistent(enic, sizeof(union vnic_rss_key),
1228 rss_key_buf_va, rss_key_buf_pa);
1233 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1235 dma_addr_t rss_cpu_buf_pa;
1236 union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1238 uint8_t name[RTE_MEMZONE_NAMESIZE];
1240 snprintf((char *)name, sizeof(name), "rss_cpu-%s", enic->bdf_name);
1241 rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1242 &rss_cpu_buf_pa, name);
1243 if (!rss_cpu_buf_va)
1246 rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1248 err = enic_set_rss_cpu(enic,
1250 sizeof(union vnic_rss_cpu));
1252 enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1253 rss_cpu_buf_va, rss_cpu_buf_pa);
1255 /* Save for later queries */
1257 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1261 static int enic_set_niccfg(struct enic *enic, uint8_t rss_default_cpu,
1262 uint8_t rss_hash_type, uint8_t rss_hash_bits, uint8_t rss_base_cpu,
1265 const uint8_t tso_ipid_split_en = 0;
1268 err = enic_set_nic_cfg(enic,
1269 rss_default_cpu, rss_hash_type,
1270 rss_hash_bits, rss_base_cpu,
1271 rss_enable, tso_ipid_split_en,
1272 enic->ig_vlan_strip_en);
1277 /* Initialize RSS with defaults, called from dev_configure */
1278 int enic_init_rss_nic_cfg(struct enic *enic)
1280 static uint8_t default_rss_key[] = {
1281 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1282 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1283 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1284 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1286 struct rte_eth_rss_conf rss_conf;
1287 union vnic_rss_cpu rss_cpu;
1290 rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1292 * If setting key for the first time, and the user gives us none, then
1293 * push the default key to NIC.
1295 if (rss_conf.rss_key == NULL) {
1296 rss_conf.rss_key = default_rss_key;
1297 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1299 ret = enic_set_rss_conf(enic, &rss_conf);
1301 dev_err(enic, "Failed to configure RSS\n");
1304 if (enic->rss_enable) {
1305 /* If enabling RSS, use the default reta */
1306 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1307 rss_cpu.cpu[i / 4].b[i % 4] =
1308 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1310 ret = enic_set_rss_reta(enic, &rss_cpu);
1312 dev_err(enic, "Failed to set RSS indirection table\n");
1317 int enic_setup_finish(struct enic *enic)
1319 enic_init_soft_stats(enic);
1321 /* switchdev: enable promisc mode on PF */
1322 if (enic->switchdev_mode) {
1323 vnic_dev_packet_filter(enic->vdev,
1334 vnic_dev_packet_filter(enic->vdev,
1347 static int enic_rss_conf_valid(struct enic *enic,
1348 struct rte_eth_rss_conf *rss_conf)
1350 /* RSS is disabled per VIC settings. Ignore rss_conf. */
1351 if (enic->flow_type_rss_offloads == 0)
1353 if (rss_conf->rss_key != NULL &&
1354 rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1355 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1356 rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1359 if (rss_conf->rss_hf != 0 &&
1360 (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1361 dev_err(enic, "Given rss_hf contains none of the supported"
1368 /* Set hash type and key according to rss_conf */
1369 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1371 struct rte_eth_dev *eth_dev;
1373 uint8_t rss_hash_type;
1377 RTE_ASSERT(rss_conf != NULL);
1378 ret = enic_rss_conf_valid(enic, rss_conf);
1380 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1384 eth_dev = enic->rte_dev;
1386 rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1387 if (enic->rq_count > 1 &&
1388 (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1391 if (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1392 ETH_RSS_NONFRAG_IPV4_OTHER))
1393 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1394 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1395 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1396 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
1397 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4;
1398 if (enic->udp_rss_weak) {
1400 * 'TCP' is not a typo. The "weak" version of
1401 * UDP RSS requires both the TCP and UDP bits
1402 * be set. It does enable TCP RSS as well.
1404 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1407 if (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_IPV6_EX |
1408 ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER))
1409 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1410 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX))
1411 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1412 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_UDP_EX)) {
1413 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6;
1414 if (enic->udp_rss_weak)
1415 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1422 /* Set the hash key if provided */
1423 if (rss_enable && rss_conf->rss_key) {
1424 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1426 dev_err(enic, "Failed to set RSS key\n");
1431 ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1432 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1435 enic->rss_hf = rss_hf;
1436 enic->rss_hash_type = rss_hash_type;
1437 enic->rss_enable = rss_enable;
1439 dev_err(enic, "Failed to update RSS configurations."
1440 " hash=0x%x\n", rss_hash_type);
1445 int enic_set_vlan_strip(struct enic *enic)
1448 * Unfortunately, VLAN strip on/off and RSS on/off are configured
1449 * together. So, re-do niccfg, preserving the current RSS settings.
1451 return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1452 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1456 int enic_add_packet_filter(struct enic *enic)
1458 /* switchdev ignores packet filters */
1459 if (enic->switchdev_mode) {
1460 ENICPMD_LOG(DEBUG, " switchdev: ignore packet filter");
1463 /* Args -> directed, multicast, broadcast, promisc, allmulti */
1464 return vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1465 enic->promisc, enic->allmulti);
1468 int enic_get_link_status(struct enic *enic)
1470 return vnic_dev_link_status(enic->vdev);
1473 static void enic_dev_deinit(struct enic *enic)
1475 /* stop link status checking */
1476 vnic_dev_notify_unset(enic->vdev);
1478 /* mac_addrs is freed by rte_eth_dev_release_port() */
1480 rte_free(enic->intr);
1486 int enic_set_vnic_res(struct enic *enic)
1488 struct rte_eth_dev *eth_dev = enic->rte_dev;
1490 unsigned int required_rq, required_wq, required_cq, required_intr;
1492 /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1493 required_rq = eth_dev->data->nb_rx_queues * 2;
1494 required_wq = eth_dev->data->nb_tx_queues;
1495 required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1496 required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1497 if (eth_dev->data->dev_conf.intr_conf.rxq) {
1498 required_intr += eth_dev->data->nb_rx_queues;
1500 ENICPMD_LOG(DEBUG, "Required queues for PF: rq %u wq %u cq %u",
1501 required_rq, required_wq, required_cq);
1502 if (enic->vf_required_rq) {
1503 /* Queues needed for VF representors */
1504 required_rq += enic->vf_required_rq;
1505 required_wq += enic->vf_required_wq;
1506 required_cq += enic->vf_required_cq;
1507 ENICPMD_LOG(DEBUG, "Required queues for VF representors: rq %u wq %u cq %u",
1508 enic->vf_required_rq, enic->vf_required_wq,
1509 enic->vf_required_cq);
1512 if (enic->conf_rq_count < required_rq) {
1513 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1514 eth_dev->data->nb_rx_queues,
1515 required_rq, enic->conf_rq_count);
1518 if (enic->conf_wq_count < required_wq) {
1519 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1520 eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1524 if (enic->conf_cq_count < required_cq) {
1525 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1526 required_cq, enic->conf_cq_count);
1529 if (enic->conf_intr_count < required_intr) {
1530 dev_err(dev, "Not enough Interrupts to support Rx queue"
1531 " interrupts. Required:%u, Configured:%u\n",
1532 required_intr, enic->conf_intr_count);
1537 enic->rq_count = eth_dev->data->nb_rx_queues;
1538 enic->wq_count = eth_dev->data->nb_tx_queues;
1539 enic->cq_count = enic->rq_count + enic->wq_count;
1540 enic->intr_count = required_intr;
1546 /* Initialize the completion queue for an RQ */
1548 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1550 struct vnic_rq *sop_rq, *data_rq;
1551 unsigned int cq_idx;
1554 sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1555 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx, enic)];
1556 cq_idx = enic_cq_rq(enic, rq_idx);
1558 vnic_cq_clean(&enic->cq[cq_idx]);
1559 vnic_cq_init(&enic->cq[cq_idx],
1560 0 /* flow_control_enable */,
1561 1 /* color_enable */,
1564 1 /* cq_tail_color */,
1565 0 /* interrupt_enable */,
1566 1 /* cq_entry_enable */,
1567 0 /* cq_message_enable */,
1568 0 /* interrupt offset */,
1569 0 /* cq_message_addr */);
1572 vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1573 enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1574 sop_rq->ring.desc_count - 1, 1, 0);
1575 if (data_rq->in_use) {
1576 vnic_rq_init_start(data_rq,
1578 enic_rte_rq_idx_to_data_idx(rq_idx, enic)),
1579 0, data_rq->ring.desc_count - 1, 1, 0);
1582 rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1586 if (data_rq->in_use) {
1587 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1589 enic_rxmbuf_queue_release(enic, sop_rq);
1597 /* The Cisco NIC can send and receive packets up to a max packet size
1598 * determined by the NIC type and firmware. There is also an MTU
1599 * configured into the NIC via the CIMC/UCSM management interface
1600 * which can be overridden by this function (up to the max packet size).
1601 * Depending on the network setup, doing so may cause packet drops
1602 * and unexpected behavior.
1604 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1606 unsigned int rq_idx;
1609 uint16_t old_mtu; /* previous setting */
1610 uint16_t config_mtu; /* Value configured into NIC via CIMC/UCSM */
1611 struct rte_eth_dev *eth_dev = enic->rte_dev;
1613 old_mtu = eth_dev->data->mtu;
1614 config_mtu = enic->config.mtu;
1616 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1617 return -E_RTE_SECONDARY;
1619 if (new_mtu > enic->max_mtu) {
1621 "MTU not updated: requested (%u) greater than max (%u)\n",
1622 new_mtu, enic->max_mtu);
1625 if (new_mtu < ENIC_MIN_MTU) {
1627 "MTU not updated: requested (%u) less than min (%u)\n",
1628 new_mtu, ENIC_MIN_MTU);
1631 if (new_mtu > config_mtu)
1633 "MTU (%u) is greater than value configured in NIC (%u)\n",
1634 new_mtu, config_mtu);
1636 /* Update the MTU and maximum packet length */
1637 eth_dev->data->mtu = new_mtu;
1638 eth_dev->data->dev_conf.rxmode.max_rx_pkt_len =
1639 enic_mtu_to_max_rx_pktlen(new_mtu);
1642 * If the device has not started (enic_enable), nothing to do.
1643 * Later, enic_enable() will set up RQs reflecting the new maximum
1646 if (!eth_dev->data->dev_started)
1650 * The device has started, re-do RQs on the fly. In the process, we
1651 * pick up the new maximum packet length.
1653 * Some applications rely on the ability to change MTU without stopping
1654 * the device. So keep this behavior for now.
1656 rte_spinlock_lock(&enic->mtu_lock);
1658 /* Stop traffic on all RQs */
1659 for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1660 rq = &enic->rq[rq_idx];
1661 if (rq->is_sop && rq->in_use) {
1662 rc = enic_stop_rq(enic,
1663 enic_sop_rq_idx_to_rte_idx(rq_idx));
1665 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1671 /* replace Rx function with a no-op to avoid getting stale pkts */
1672 eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1675 /* Allow time for threads to exit the real Rx function. */
1678 /* now it is safe to reconfigure the RQs */
1681 /* free and reallocate RQs with the new MTU */
1682 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1683 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1688 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1689 rq->tot_nb_desc, rq->rx_free_thresh);
1692 "Fatal MTU alloc error- No traffic will pass\n");
1696 rc = enic_reinit_rq(enic, rq_idx);
1699 "Fatal MTU RQ reinit- No traffic will pass\n");
1704 /* put back the real receive function */
1706 enic_pick_rx_handler(eth_dev);
1709 /* restart Rx traffic */
1710 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1711 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1712 if (rq->is_sop && rq->in_use)
1713 enic_start_rq(enic, rq_idx);
1717 dev_info(enic, "MTU changed from %u to %u\n", old_mtu, new_mtu);
1718 rte_spinlock_unlock(&enic->mtu_lock);
1722 static int enic_dev_init(struct enic *enic)
1725 struct rte_eth_dev *eth_dev = enic->rte_dev;
1727 vnic_dev_intr_coal_timer_info_default(enic->vdev);
1729 /* Get vNIC configuration
1731 err = enic_get_vnic_config(enic);
1733 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1737 /* Get available resource counts */
1738 enic_get_res_counts(enic);
1739 if (enic->conf_rq_count == 1) {
1740 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1741 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1742 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1745 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1746 enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1747 enic->conf_cq_count, 8);
1748 enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1749 enic->conf_intr_count, 8);
1750 enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1751 enic->conf_rq_count, 8);
1752 enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1753 enic->conf_wq_count, 8);
1754 if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1755 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1758 if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1759 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1762 if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1763 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1766 if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1767 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1771 eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr",
1772 sizeof(struct rte_ether_addr) *
1773 ENIC_UNICAST_PERFECT_FILTERS, 0);
1774 if (!eth_dev->data->mac_addrs) {
1775 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1778 rte_ether_addr_copy((struct rte_ether_addr *)enic->mac_addr,
1779 eth_dev->data->mac_addrs);
1781 vnic_dev_set_reset_flag(enic->vdev, 0);
1783 LIST_INIT(&enic->flows);
1785 /* set up link status checking */
1786 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1789 * When Geneve with options offload is available, always disable it
1790 * first as it can interfere with user flow rules.
1792 if (enic->geneve_opt_avail) {
1794 * Disabling fails if the feature is provisioned but
1795 * not enabled. So ignore result and do not log error.
1797 vnic_dev_overlay_offload_ctrl(enic->vdev,
1798 OVERLAY_FEATURE_GENEVE,
1799 OVERLAY_OFFLOAD_DISABLE);
1801 enic->overlay_offload = false;
1802 if (enic->disable_overlay && enic->vxlan) {
1804 * Explicitly disable overlay offload as the setting is
1805 * sticky, and resetting vNIC does not disable it.
1807 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1808 OVERLAY_FEATURE_VXLAN,
1809 OVERLAY_OFFLOAD_DISABLE)) {
1810 dev_err(enic, "failed to disable overlay offload\n");
1812 dev_info(enic, "Overlay offload is disabled\n");
1815 if (!enic->disable_overlay && enic->vxlan &&
1816 /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1817 vnic_dev_overlay_offload_ctrl(enic->vdev,
1818 OVERLAY_FEATURE_VXLAN,
1819 OVERLAY_OFFLOAD_ENABLE) == 0) {
1820 enic->tx_offload_capa |=
1821 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1822 DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
1823 DEV_TX_OFFLOAD_VXLAN_TNL_TSO;
1824 enic->tx_offload_mask |=
1827 PKT_TX_OUTER_IP_CKSUM |
1829 enic->overlay_offload = true;
1830 dev_info(enic, "Overlay offload is enabled\n");
1832 /* Geneve with options offload requires overlay offload */
1833 if (enic->overlay_offload && enic->geneve_opt_avail &&
1834 enic->geneve_opt_request) {
1835 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1836 OVERLAY_FEATURE_GENEVE,
1837 OVERLAY_OFFLOAD_ENABLE)) {
1838 dev_err(enic, "failed to enable geneve+option\n");
1840 enic->geneve_opt_enabled = 1;
1841 dev_info(enic, "Geneve with options is enabled\n");
1845 * Reset the vxlan port if HW vxlan parsing is available. It
1846 * is always enabled regardless of overlay offload
1850 enic->vxlan_port = RTE_VXLAN_DEFAULT_PORT;
1852 * Reset the vxlan port to the default, as the NIC firmware
1853 * does not reset it automatically and keeps the old setting.
1855 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1856 OVERLAY_CFG_VXLAN_PORT_UPDATE,
1857 RTE_VXLAN_DEFAULT_PORT)) {
1858 dev_err(enic, "failed to update vxlan port\n");
1863 if (enic_fm_init(enic))
1864 dev_warning(enic, "Init of flowman failed.\n");
1869 static void lock_devcmd(void *priv)
1871 struct enic *enic = priv;
1873 rte_spinlock_lock(&enic->devcmd_lock);
1876 static void unlock_devcmd(void *priv)
1878 struct enic *enic = priv;
1880 rte_spinlock_unlock(&enic->devcmd_lock);
1883 int enic_probe(struct enic *enic)
1885 struct rte_pci_device *pdev = enic->pdev;
1888 dev_debug(enic, "Initializing ENIC PMD\n");
1890 /* if this is a secondary process the hardware is already initialized */
1891 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1894 enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1895 enic->bar0.len = pdev->mem_resource[0].len;
1897 /* Register vNIC device */
1898 enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1900 dev_err(enic, "vNIC registration failed, aborting\n");
1904 LIST_INIT(&enic->memzone_list);
1905 rte_spinlock_init(&enic->memzone_list_lock);
1907 vnic_register_cbacks(enic->vdev,
1908 enic_alloc_consistent,
1909 enic_free_consistent);
1912 * Allocate the consistent memory for stats upfront so both primary and
1913 * secondary processes can dump stats.
1915 err = vnic_dev_alloc_stats_mem(enic->vdev);
1917 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1918 goto err_out_unregister;
1920 /* Issue device open to get device in known state */
1921 err = enic_dev_open(enic);
1923 dev_err(enic, "vNIC dev open failed, aborting\n");
1924 goto err_out_unregister;
1927 /* Set ingress vlan rewrite mode before vnic initialization */
1928 dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n",
1929 enic->ig_vlan_rewrite_mode);
1930 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1931 enic->ig_vlan_rewrite_mode);
1934 "Failed to set ingress vlan rewrite mode, aborting.\n");
1935 goto err_out_dev_close;
1938 /* Issue device init to initialize the vnic-to-switch link.
1939 * We'll start with carrier off and wait for link UP
1940 * notification later to turn on carrier. We don't need
1941 * to wait here for the vnic-to-switch link initialization
1942 * to complete; link UP notification is the indication that
1943 * the process is complete.
1946 err = vnic_dev_init(enic->vdev, 0);
1948 dev_err(enic, "vNIC dev init failed, aborting\n");
1949 goto err_out_dev_close;
1952 err = enic_dev_init(enic);
1954 dev_err(enic, "Device initialization failed, aborting\n");
1955 goto err_out_dev_close;
1958 /* Use a PF spinlock to serialize devcmd from PF and VF representors */
1959 if (enic->switchdev_mode) {
1960 rte_spinlock_init(&enic->devcmd_lock);
1961 vnic_register_lock(enic->vdev, lock_devcmd, unlock_devcmd);
1966 vnic_dev_close(enic->vdev);
1968 vnic_dev_unregister(enic->vdev);
1973 void enic_remove(struct enic *enic)
1975 enic_dev_deinit(enic);
1976 vnic_dev_close(enic->vdev);
1977 vnic_dev_unregister(enic->vdev);