1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
13 #include <rte_bus_pci.h>
14 #include <rte_memzone.h>
15 #include <rte_malloc.h>
17 #include <rte_string_fns.h>
18 #include <rte_ethdev_driver.h>
20 #include "enic_compat.h"
22 #include "wq_enet_desc.h"
23 #include "rq_enet_desc.h"
24 #include "cq_enet_desc.h"
25 #include "vnic_enet.h"
30 #include "vnic_intr.h"
33 static inline int enic_is_sriov_vf(struct enic *enic)
35 return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
38 static int is_zero_addr(uint8_t *addr)
40 return !(addr[0] | addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
43 static int is_mcast_addr(uint8_t *addr)
48 static int is_eth_addr_valid(uint8_t *addr)
50 return !is_mcast_addr(addr) && !is_zero_addr(addr);
54 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
58 if (!rq || !rq->mbuf_ring) {
59 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
63 for (i = 0; i < rq->ring.desc_count; i++) {
64 if (rq->mbuf_ring[i]) {
65 rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
66 rq->mbuf_ring[i] = NULL;
71 static void enic_free_wq_buf(struct rte_mbuf **buf)
73 struct rte_mbuf *mbuf = *buf;
75 rte_pktmbuf_free_seg(mbuf);
79 static void enic_log_q_error(struct enic *enic)
82 uint32_t error_status;
84 for (i = 0; i < enic->wq_count; i++) {
85 error_status = vnic_wq_error_status(&enic->wq[i]);
87 dev_err(enic, "WQ[%d] error_status %d\n", i,
91 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
92 if (!enic->rq[i].in_use)
94 error_status = vnic_rq_error_status(&enic->rq[i]);
96 dev_err(enic, "RQ[%d] error_status %d\n", i,
101 static void enic_clear_soft_stats(struct enic *enic)
103 struct enic_soft_stats *soft_stats = &enic->soft_stats;
104 rte_atomic64_clear(&soft_stats->rx_nombuf);
105 rte_atomic64_clear(&soft_stats->rx_packet_errors);
106 rte_atomic64_clear(&soft_stats->tx_oversized);
109 static void enic_init_soft_stats(struct enic *enic)
111 struct enic_soft_stats *soft_stats = &enic->soft_stats;
112 rte_atomic64_init(&soft_stats->rx_nombuf);
113 rte_atomic64_init(&soft_stats->rx_packet_errors);
114 rte_atomic64_init(&soft_stats->tx_oversized);
115 enic_clear_soft_stats(enic);
118 int enic_dev_stats_clear(struct enic *enic)
122 ret = vnic_dev_stats_clear(enic->vdev);
124 dev_err(enic, "Error in clearing stats\n");
127 enic_clear_soft_stats(enic);
132 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
134 struct vnic_stats *stats;
135 struct enic_soft_stats *soft_stats = &enic->soft_stats;
136 int64_t rx_truncated;
137 uint64_t rx_packet_errors;
138 int ret = vnic_dev_stats_dump(enic->vdev, &stats);
141 dev_err(enic, "Error in getting stats\n");
145 /* The number of truncated packets can only be calculated by
146 * subtracting a hardware counter from error packets received by
147 * the driver. Note: this causes transient inaccuracies in the
148 * ipackets count. Also, the length of truncated packets are
149 * counted in ibytes even though truncated packets are dropped
150 * which can make ibytes be slightly higher than it should be.
152 rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
153 rx_truncated = rx_packet_errors - stats->rx.rx_errors;
155 r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
156 r_stats->opackets = stats->tx.tx_frames_ok;
158 r_stats->ibytes = stats->rx.rx_bytes_ok;
159 r_stats->obytes = stats->tx.tx_bytes_ok;
161 r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
162 r_stats->oerrors = stats->tx.tx_errors
163 + rte_atomic64_read(&soft_stats->tx_oversized);
165 r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
167 r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
171 int enic_del_mac_address(struct enic *enic, int mac_index)
173 struct rte_eth_dev *eth_dev = enic->rte_dev;
174 uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
176 return vnic_dev_del_addr(enic->vdev, mac_addr);
179 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
183 if (!is_eth_addr_valid(mac_addr)) {
184 dev_err(enic, "invalid mac address\n");
188 err = vnic_dev_add_addr(enic->vdev, mac_addr);
190 dev_err(enic, "add mac addr failed\n");
195 enic_free_rq_buf(struct rte_mbuf **mbuf)
200 rte_pktmbuf_free(*mbuf);
204 void enic_init_vnic_resources(struct enic *enic)
206 unsigned int error_interrupt_enable = 1;
207 unsigned int error_interrupt_offset = 0;
208 unsigned int rxq_interrupt_enable = 0;
209 unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
210 unsigned int index = 0;
212 struct vnic_rq *data_rq;
214 if (enic->rte_dev->data->dev_conf.intr_conf.rxq)
215 rxq_interrupt_enable = 1;
217 for (index = 0; index < enic->rq_count; index++) {
218 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
220 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
222 error_interrupt_enable,
223 error_interrupt_offset);
225 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
227 vnic_rq_init(data_rq,
229 error_interrupt_enable,
230 error_interrupt_offset);
232 vnic_cq_init(&enic->cq[cq_idx],
233 0 /* flow_control_enable */,
234 1 /* color_enable */,
237 1 /* cq_tail_color */,
238 rxq_interrupt_enable,
239 1 /* cq_entry_enable */,
240 0 /* cq_message_enable */,
241 rxq_interrupt_offset,
242 0 /* cq_message_addr */);
243 if (rxq_interrupt_enable)
244 rxq_interrupt_offset++;
247 for (index = 0; index < enic->wq_count; index++) {
248 vnic_wq_init(&enic->wq[index],
249 enic_cq_wq(enic, index),
250 error_interrupt_enable,
251 error_interrupt_offset);
252 /* Compute unsupported ol flags for enic_prep_pkts() */
253 enic->wq[index].tx_offload_notsup_mask =
254 PKT_TX_OFFLOAD_MASK ^ enic->tx_offload_mask;
256 cq_idx = enic_cq_wq(enic, index);
257 vnic_cq_init(&enic->cq[cq_idx],
258 0 /* flow_control_enable */,
259 1 /* color_enable */,
262 1 /* cq_tail_color */,
263 0 /* interrupt_enable */,
264 0 /* cq_entry_enable */,
265 1 /* cq_message_enable */,
266 0 /* interrupt offset */,
267 (uint64_t)enic->wq[index].cqmsg_rz->iova);
270 for (index = 0; index < enic->intr_count; index++) {
271 vnic_intr_init(&enic->intr[index],
272 enic->config.intr_timer_usec,
273 enic->config.intr_timer_type,
274 /*mask_on_assertion*/1);
280 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
283 struct rq_enet_desc *rqd = rq->ring.descs;
286 uint32_t max_rx_pkt_len;
292 dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
293 rq->ring.desc_count);
296 * If *not* using scatter and the mbuf size is greater than the
297 * requested max packet size (max_rx_pkt_len), then reduce the
298 * posted buffer size to max_rx_pkt_len. HW still receives packets
299 * larger than max_rx_pkt_len, but they will be truncated, which we
300 * drop in the rx handler. Not ideal, but better than returning
301 * large packets when the user is not expecting them.
303 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
304 rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
305 if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
306 rq_buf_len = max_rx_pkt_len;
307 for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
308 mb = rte_mbuf_raw_alloc(rq->mp);
310 dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
311 (unsigned)rq->index);
315 mb->data_off = RTE_PKTMBUF_HEADROOM;
316 dma_addr = (dma_addr_t)(mb->buf_iova
317 + RTE_PKTMBUF_HEADROOM);
318 rq_enet_desc_enc(rqd, dma_addr,
319 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
320 : RQ_ENET_TYPE_NOT_SOP),
322 rq->mbuf_ring[i] = mb;
325 * Do not post the buffers to the NIC until we enable the RQ via
328 rq->need_initial_post = true;
329 /* Initialize fetch index while RQ is disabled */
330 iowrite32(0, &rq->ctrl->fetch_index);
335 * Post the Rx buffers for the first time. enic_alloc_rx_queue_mbufs() has
336 * allocated the buffers and filled the RQ descriptor ring. Just need to push
337 * the post index to the NIC.
340 enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)
342 if (!rq->in_use || !rq->need_initial_post)
345 /* make sure all prior writes are complete before doing the PIO write */
348 /* Post all but the last buffer to VIC. */
349 rq->posted_index = rq->ring.desc_count - 1;
353 dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
354 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
355 iowrite32(rq->posted_index, &rq->ctrl->posted_index);
357 rq->need_initial_post = false;
361 enic_alloc_consistent(void *priv, size_t size,
362 dma_addr_t *dma_handle, uint8_t *name)
365 const struct rte_memzone *rz;
367 struct enic *enic = (struct enic *)priv;
368 struct enic_memzone_entry *mze;
370 rz = rte_memzone_reserve_aligned((const char *)name, size,
371 SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_PAGE_SIZE);
373 pr_err("%s : Failed to allocate memory requested for %s\n",
379 *dma_handle = (dma_addr_t)rz->iova;
381 mze = rte_malloc("enic memzone entry",
382 sizeof(struct enic_memzone_entry), 0);
385 pr_err("%s : Failed to allocate memory for memzone list\n",
387 rte_memzone_free(rz);
393 rte_spinlock_lock(&enic->memzone_list_lock);
394 LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
395 rte_spinlock_unlock(&enic->memzone_list_lock);
401 enic_free_consistent(void *priv,
402 __rte_unused size_t size,
404 dma_addr_t dma_handle)
406 struct enic_memzone_entry *mze;
407 struct enic *enic = (struct enic *)priv;
409 rte_spinlock_lock(&enic->memzone_list_lock);
410 LIST_FOREACH(mze, &enic->memzone_list, entries) {
411 if (mze->rz->addr == vaddr &&
412 mze->rz->iova == dma_handle)
416 rte_spinlock_unlock(&enic->memzone_list_lock);
418 "Tried to free memory, but couldn't find it in the memzone list\n");
421 LIST_REMOVE(mze, entries);
422 rte_spinlock_unlock(&enic->memzone_list_lock);
423 rte_memzone_free(mze->rz);
427 int enic_link_update(struct rte_eth_dev *eth_dev)
429 struct enic *enic = pmd_priv(eth_dev);
430 struct rte_eth_link link;
432 memset(&link, 0, sizeof(link));
433 link.link_status = enic_get_link_status(enic);
434 link.link_duplex = ETH_LINK_FULL_DUPLEX;
435 link.link_speed = vnic_dev_port_speed(enic->vdev);
437 return rte_eth_linkstatus_set(eth_dev, &link);
441 enic_intr_handler(void *arg)
443 struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
444 struct enic *enic = pmd_priv(dev);
446 vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
448 enic_link_update(dev);
449 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
450 enic_log_q_error(enic);
451 /* Re-enable irq in case of INTx */
452 rte_intr_ack(&enic->pdev->intr_handle);
455 static int enic_rxq_intr_init(struct enic *enic)
457 struct rte_intr_handle *intr_handle;
458 uint32_t rxq_intr_count, i;
461 intr_handle = enic->rte_dev->intr_handle;
462 if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
465 * Rx queue interrupts only work when we have MSI-X interrupts,
466 * one per queue. Sharing one interrupt is technically
467 * possible with VIC, but it is not worth the complications it brings.
469 if (!rte_intr_cap_multiple(intr_handle)) {
470 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
471 " (vfio-pci driver)\n");
474 rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
475 err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
477 dev_err(enic, "Failed to enable event fds for Rx queue"
481 intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
482 rxq_intr_count * sizeof(int), 0);
483 if (intr_handle->intr_vec == NULL) {
484 dev_err(enic, "Failed to allocate intr_vec\n");
487 for (i = 0; i < rxq_intr_count; i++)
488 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
492 static void enic_rxq_intr_deinit(struct enic *enic)
494 struct rte_intr_handle *intr_handle;
496 intr_handle = enic->rte_dev->intr_handle;
497 rte_intr_efd_disable(intr_handle);
498 if (intr_handle->intr_vec != NULL) {
499 rte_free(intr_handle->intr_vec);
500 intr_handle->intr_vec = NULL;
504 static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
506 struct wq_enet_desc *desc;
511 * Fill WQ descriptor fields that never change. Every descriptor is
512 * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
513 * descriptors (i.e. request one completion update every 32 packets).
515 wq = &enic->wq[queue_idx];
516 desc = (struct wq_enet_desc *)wq->ring.descs;
517 for (i = 0; i < wq->ring.desc_count; i++, desc++) {
518 desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
519 if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
520 desc->header_length_flags |=
521 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
526 * The 'strong' version is in enic_rxtx_vec_avx2.c. This weak version is used
527 * used when that file is not compiled.
530 enic_use_vector_rx_handler(__rte_unused struct rte_eth_dev *eth_dev)
535 void enic_pick_rx_handler(struct rte_eth_dev *eth_dev)
537 struct enic *enic = pmd_priv(eth_dev);
541 * 1. The vectorized handler if possible and requested.
542 * 2. The non-scatter, simplified handler if scatter Rx is not used.
543 * 3. The default handler as a fallback.
545 if (enic_use_vector_rx_handler(eth_dev))
547 if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
548 ENICPMD_LOG(DEBUG, " use the non-scatter Rx handler");
549 eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
551 ENICPMD_LOG(DEBUG, " use the normal Rx handler");
552 eth_dev->rx_pkt_burst = &enic_recv_pkts;
556 /* Secondary process uses this to set the Tx handler */
557 void enic_pick_tx_handler(struct rte_eth_dev *eth_dev)
559 struct enic *enic = pmd_priv(eth_dev);
561 if (enic->use_simple_tx_handler) {
562 ENICPMD_LOG(DEBUG, " use the simple tx handler");
563 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
565 ENICPMD_LOG(DEBUG, " use the default tx handler");
566 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
570 int enic_enable(struct enic *enic)
574 struct rte_eth_dev *eth_dev = enic->rte_dev;
575 uint64_t simple_tx_offloads;
578 if (enic->enable_avx2_rx) {
579 struct rte_mbuf mb_def = { .buf_addr = 0 };
582 * mbuf_initializer contains const-after-init fields of
583 * receive mbufs (i.e. 64 bits of fields from rearm_data).
584 * It is currently used by the vectorized handler.
587 mb_def.data_off = RTE_PKTMBUF_HEADROOM;
588 mb_def.port = enic->port_id;
589 rte_mbuf_refcnt_set(&mb_def, 1);
590 rte_compiler_barrier();
591 p = (uintptr_t)&mb_def.rearm_data;
592 enic->mbuf_initializer = *(uint64_t *)p;
595 eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
596 eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
598 /* vnic notification of link status has already been turned on in
599 * enic_dev_init() which is called during probe time. Here we are
600 * just turning on interrupt vector 0 if needed.
602 if (eth_dev->data->dev_conf.intr_conf.lsc)
603 vnic_dev_notify_set(enic->vdev, 0);
605 err = enic_rxq_intr_init(enic);
608 if (enic_clsf_init(enic))
609 dev_warning(enic, "Init of hash table for clsf failed."\
610 "Flow director feature will not work\n");
612 if (enic_fm_init(enic))
613 dev_warning(enic, "Init of flowman failed.\n");
615 for (index = 0; index < enic->rq_count; index++) {
616 err = enic_alloc_rx_queue_mbufs(enic,
617 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
619 dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
622 err = enic_alloc_rx_queue_mbufs(enic,
623 &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
625 /* release the allocated mbufs for the sop rq*/
626 enic_rxmbuf_queue_release(enic,
627 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
629 dev_err(enic, "Failed to alloc data RX queue mbufs\n");
635 * Use the simple TX handler if possible. Only checksum offloads
636 * and vlan insertion are supported.
638 simple_tx_offloads = enic->tx_offload_capa &
639 (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
640 DEV_TX_OFFLOAD_VLAN_INSERT |
641 DEV_TX_OFFLOAD_IPV4_CKSUM |
642 DEV_TX_OFFLOAD_UDP_CKSUM |
643 DEV_TX_OFFLOAD_TCP_CKSUM);
644 if ((eth_dev->data->dev_conf.txmode.offloads &
645 ~simple_tx_offloads) == 0) {
646 ENICPMD_LOG(DEBUG, " use the simple tx handler");
647 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
648 for (index = 0; index < enic->wq_count; index++)
649 enic_prep_wq_for_simple_tx(enic, index);
650 enic->use_simple_tx_handler = 1;
652 ENICPMD_LOG(DEBUG, " use the default tx handler");
653 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
656 enic_pick_rx_handler(eth_dev);
658 for (index = 0; index < enic->wq_count; index++)
659 enic_start_wq(enic, index);
660 for (index = 0; index < enic->rq_count; index++)
661 enic_start_rq(enic, index);
663 vnic_dev_add_addr(enic->vdev, enic->mac_addr);
665 vnic_dev_enable_wait(enic->vdev);
667 /* Register and enable error interrupt */
668 rte_intr_callback_register(&(enic->pdev->intr_handle),
669 enic_intr_handler, (void *)enic->rte_dev);
671 rte_intr_enable(&(enic->pdev->intr_handle));
672 /* Unmask LSC interrupt */
673 vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
678 int enic_alloc_intr_resources(struct enic *enic)
683 dev_info(enic, "vNIC resources used: "\
684 "wq %d rq %d cq %d intr %d\n",
685 enic->wq_count, enic_vnic_rq_count(enic),
686 enic->cq_count, enic->intr_count);
688 for (i = 0; i < enic->intr_count; i++) {
689 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
691 enic_free_vnic_resources(enic);
698 void enic_free_rq(void *rxq)
700 struct vnic_rq *rq_sop, *rq_data;
706 rq_sop = (struct vnic_rq *)rxq;
707 enic = vnic_dev_priv(rq_sop->vdev);
708 rq_data = &enic->rq[rq_sop->data_queue_idx];
710 if (rq_sop->free_mbufs) {
711 struct rte_mbuf **mb;
714 mb = rq_sop->free_mbufs;
715 for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs;
716 i < ENIC_RX_BURST_MAX; i++)
717 rte_pktmbuf_free(mb[i]);
718 rte_free(rq_sop->free_mbufs);
719 rq_sop->free_mbufs = NULL;
720 rq_sop->num_free_mbufs = 0;
723 enic_rxmbuf_queue_release(enic, rq_sop);
725 enic_rxmbuf_queue_release(enic, rq_data);
727 rte_free(rq_sop->mbuf_ring);
729 rte_free(rq_data->mbuf_ring);
731 rq_sop->mbuf_ring = NULL;
732 rq_data->mbuf_ring = NULL;
734 vnic_rq_free(rq_sop);
736 vnic_rq_free(rq_data);
738 vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
744 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
746 struct rte_eth_dev_data *data = enic->dev_data;
747 vnic_wq_enable(&enic->wq[queue_idx]);
748 data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
751 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
753 struct rte_eth_dev_data *data = enic->dev_data;
756 ret = vnic_wq_disable(&enic->wq[queue_idx]);
760 data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
764 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
766 struct rte_eth_dev_data *data = enic->dev_data;
767 struct vnic_rq *rq_sop;
768 struct vnic_rq *rq_data;
769 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
770 rq_data = &enic->rq[rq_sop->data_queue_idx];
772 if (rq_data->in_use) {
773 vnic_rq_enable(rq_data);
774 enic_initial_post_rx(enic, rq_data);
777 vnic_rq_enable(rq_sop);
778 enic_initial_post_rx(enic, rq_sop);
779 data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
782 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
784 struct rte_eth_dev_data *data = enic->dev_data;
785 int ret1 = 0, ret2 = 0;
786 struct vnic_rq *rq_sop;
787 struct vnic_rq *rq_data;
788 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
789 rq_data = &enic->rq[rq_sop->data_queue_idx];
791 ret2 = vnic_rq_disable(rq_sop);
794 ret1 = vnic_rq_disable(rq_data);
801 data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
805 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
806 unsigned int socket_id, struct rte_mempool *mp,
807 uint16_t nb_desc, uint16_t free_thresh)
810 uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
811 uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
812 struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
813 struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
814 unsigned int mbuf_size, mbufs_per_pkt;
815 unsigned int nb_sop_desc, nb_data_desc;
816 uint16_t min_sop, max_sop, min_data, max_data;
817 uint32_t max_rx_pkt_len;
820 rq_sop->data_queue_idx = data_queue_idx;
822 rq_data->data_queue_idx = 0;
823 rq_sop->socket_id = socket_id;
825 rq_data->socket_id = socket_id;
828 rq_sop->rx_free_thresh = free_thresh;
829 rq_data->rx_free_thresh = free_thresh;
830 dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
833 mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
834 RTE_PKTMBUF_HEADROOM);
835 /* max_rx_pkt_len includes the ethernet header and CRC. */
836 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
838 if (enic->rte_dev->data->dev_conf.rxmode.offloads &
839 DEV_RX_OFFLOAD_SCATTER) {
840 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
841 /* ceil((max pkt len)/mbuf_size) */
842 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
844 dev_info(enic, "Scatter rx mode disabled\n");
846 if (max_rx_pkt_len > mbuf_size) {
847 dev_warning(enic, "The maximum Rx packet size (%u) is"
848 " larger than the mbuf size (%u), and"
849 " scatter is disabled. Larger packets will"
851 max_rx_pkt_len, mbuf_size);
855 if (mbufs_per_pkt > 1) {
856 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
857 rq_sop->data_queue_enable = 1;
860 * HW does not directly support rxmode.max_rx_pkt_len. HW always
861 * receives packet sizes up to the "max" MTU.
862 * If not using scatter, we can achieve the effect of dropping
863 * larger packets by reducing the size of posted buffers.
864 * See enic_alloc_rx_queue_mbufs().
867 enic_mtu_to_max_rx_pktlen(enic->max_mtu)) {
868 dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
869 " when scatter rx mode is in use.\n");
872 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
874 rq_sop->data_queue_enable = 0;
878 /* number of descriptors have to be a multiple of 32 */
879 nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK;
880 nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK;
882 rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
883 rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
885 if (mbufs_per_pkt > 1) {
886 min_sop = ENIC_RX_BURST_MAX;
887 max_sop = ((enic->config.rq_desc_count /
888 (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK);
889 min_data = min_sop * (mbufs_per_pkt - 1);
890 max_data = enic->config.rq_desc_count;
892 min_sop = ENIC_RX_BURST_MAX;
893 max_sop = enic->config.rq_desc_count;
898 if (nb_desc < (min_sop + min_data)) {
900 "Number of rx descs too low, adjusting to minimum\n");
901 nb_sop_desc = min_sop;
902 nb_data_desc = min_data;
903 } else if (nb_desc > (max_sop + max_data)) {
905 "Number of rx_descs too high, adjusting to maximum\n");
906 nb_sop_desc = max_sop;
907 nb_data_desc = max_data;
909 if (mbufs_per_pkt > 1) {
910 dev_info(enic, "For max packet size %u and mbuf size %u valid"
911 " rx descriptor range is %u to %u\n",
912 max_rx_pkt_len, mbuf_size, min_sop + min_data,
915 dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
916 nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
918 /* Allocate sop queue resources */
919 rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
920 nb_sop_desc, sizeof(struct rq_enet_desc));
922 dev_err(enic, "error in allocation of sop rq\n");
925 nb_sop_desc = rq_sop->ring.desc_count;
927 if (rq_data->in_use) {
928 /* Allocate data queue resources */
929 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
931 sizeof(struct rq_enet_desc));
933 dev_err(enic, "error in allocation of data rq\n");
934 goto err_free_rq_sop;
936 nb_data_desc = rq_data->ring.desc_count;
938 rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
939 socket_id, nb_sop_desc + nb_data_desc,
940 sizeof(struct cq_enet_rq_desc));
942 dev_err(enic, "error in allocation of cq for rq\n");
943 goto err_free_rq_data;
946 /* Allocate the mbuf rings */
947 rq_sop->mbuf_ring = (struct rte_mbuf **)
948 rte_zmalloc_socket("rq->mbuf_ring",
949 sizeof(struct rte_mbuf *) * nb_sop_desc,
950 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
951 if (rq_sop->mbuf_ring == NULL)
954 if (rq_data->in_use) {
955 rq_data->mbuf_ring = (struct rte_mbuf **)
956 rte_zmalloc_socket("rq->mbuf_ring",
957 sizeof(struct rte_mbuf *) * nb_data_desc,
958 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
959 if (rq_data->mbuf_ring == NULL)
960 goto err_free_sop_mbuf;
963 rq_sop->free_mbufs = (struct rte_mbuf **)
964 rte_zmalloc_socket("rq->free_mbufs",
965 sizeof(struct rte_mbuf *) *
967 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
968 if (rq_sop->free_mbufs == NULL)
969 goto err_free_data_mbuf;
970 rq_sop->num_free_mbufs = 0;
972 rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
977 rte_free(rq_data->mbuf_ring);
979 rte_free(rq_sop->mbuf_ring);
981 /* cleanup on error */
982 vnic_cq_free(&enic->cq[queue_idx]);
985 vnic_rq_free(rq_data);
987 vnic_rq_free(rq_sop);
992 void enic_free_wq(void *txq)
1000 wq = (struct vnic_wq *)txq;
1001 enic = vnic_dev_priv(wq->vdev);
1002 rte_memzone_free(wq->cqmsg_rz);
1004 vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
1007 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
1008 unsigned int socket_id, uint16_t nb_desc)
1011 struct vnic_wq *wq = &enic->wq[queue_idx];
1012 unsigned int cq_index = enic_cq_wq(enic, queue_idx);
1013 char name[RTE_MEMZONE_NAMESIZE];
1014 static int instance;
1016 wq->socket_id = socket_id;
1018 * rte_eth_tx_queue_setup() checks min, max, and alignment. So just
1019 * print an info message for diagnostics.
1021 dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc);
1023 /* Allocate queue resources */
1024 err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
1026 sizeof(struct wq_enet_desc));
1028 dev_err(enic, "error in allocation of wq\n");
1032 err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
1034 sizeof(struct cq_enet_wq_desc));
1037 dev_err(enic, "error in allocation of cq for wq\n");
1040 /* setup up CQ message */
1041 snprintf((char *)name, sizeof(name),
1042 "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
1045 wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
1046 sizeof(uint32_t), SOCKET_ID_ANY,
1047 RTE_MEMZONE_IOVA_CONTIG, ENIC_PAGE_SIZE);
1054 int enic_disable(struct enic *enic)
1059 for (i = 0; i < enic->intr_count; i++) {
1060 vnic_intr_mask(&enic->intr[i]);
1061 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
1063 enic_rxq_intr_deinit(enic);
1064 rte_intr_disable(&enic->pdev->intr_handle);
1065 rte_intr_callback_unregister(&enic->pdev->intr_handle,
1067 (void *)enic->rte_dev);
1069 vnic_dev_disable(enic->vdev);
1071 enic_clsf_destroy(enic);
1072 enic_fm_destroy(enic);
1074 if (!enic_is_sriov_vf(enic))
1075 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
1077 for (i = 0; i < enic->wq_count; i++) {
1078 err = vnic_wq_disable(&enic->wq[i]);
1082 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
1083 if (enic->rq[i].in_use) {
1084 err = vnic_rq_disable(&enic->rq[i]);
1090 /* If we were using interrupts, set the interrupt vector to -1
1091 * to disable interrupts. We are not disabling link notifcations,
1092 * though, as we want the polling of link status to continue working.
1094 if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
1095 vnic_dev_notify_set(enic->vdev, -1);
1097 vnic_dev_set_reset_flag(enic->vdev, 1);
1099 for (i = 0; i < enic->wq_count; i++)
1100 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
1102 for (i = 0; i < enic_vnic_rq_count(enic); i++)
1103 if (enic->rq[i].in_use)
1104 vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
1105 for (i = 0; i < enic->cq_count; i++)
1106 vnic_cq_clean(&enic->cq[i]);
1107 for (i = 0; i < enic->intr_count; i++)
1108 vnic_intr_clean(&enic->intr[i]);
1113 static int enic_dev_wait(struct vnic_dev *vdev,
1114 int (*start)(struct vnic_dev *, int),
1115 int (*finished)(struct vnic_dev *, int *),
1122 err = start(vdev, arg);
1126 /* Wait for func to complete...2 seconds max */
1127 for (i = 0; i < 2000; i++) {
1128 err = finished(vdev, &done);
1138 static int enic_dev_open(struct enic *enic)
1141 int flags = CMD_OPENF_IG_DESCCACHE;
1143 err = enic_dev_wait(enic->vdev, vnic_dev_open,
1144 vnic_dev_open_done, flags);
1146 dev_err(enic_get_dev(enic),
1147 "vNIC device open failed, err %d\n", err);
1152 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
1154 dma_addr_t rss_key_buf_pa;
1155 union vnic_rss_key *rss_key_buf_va = NULL;
1157 uint8_t name[RTE_MEMZONE_NAMESIZE];
1159 RTE_ASSERT(user_key != NULL);
1160 snprintf((char *)name, sizeof(name), "rss_key-%s", enic->bdf_name);
1161 rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1162 &rss_key_buf_pa, name);
1163 if (!rss_key_buf_va)
1166 for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1167 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1169 err = enic_set_rss_key(enic,
1171 sizeof(union vnic_rss_key));
1173 /* Save for later queries */
1175 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1176 sizeof(union vnic_rss_key));
1178 enic_free_consistent(enic, sizeof(union vnic_rss_key),
1179 rss_key_buf_va, rss_key_buf_pa);
1184 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1186 dma_addr_t rss_cpu_buf_pa;
1187 union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1189 uint8_t name[RTE_MEMZONE_NAMESIZE];
1191 snprintf((char *)name, sizeof(name), "rss_cpu-%s", enic->bdf_name);
1192 rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1193 &rss_cpu_buf_pa, name);
1194 if (!rss_cpu_buf_va)
1197 rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1199 err = enic_set_rss_cpu(enic,
1201 sizeof(union vnic_rss_cpu));
1203 enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1204 rss_cpu_buf_va, rss_cpu_buf_pa);
1206 /* Save for later queries */
1208 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1212 static int enic_set_niccfg(struct enic *enic, uint8_t rss_default_cpu,
1213 uint8_t rss_hash_type, uint8_t rss_hash_bits, uint8_t rss_base_cpu,
1216 const uint8_t tso_ipid_split_en = 0;
1219 err = enic_set_nic_cfg(enic,
1220 rss_default_cpu, rss_hash_type,
1221 rss_hash_bits, rss_base_cpu,
1222 rss_enable, tso_ipid_split_en,
1223 enic->ig_vlan_strip_en);
1228 /* Initialize RSS with defaults, called from dev_configure */
1229 int enic_init_rss_nic_cfg(struct enic *enic)
1231 static uint8_t default_rss_key[] = {
1232 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1233 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1234 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1235 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1237 struct rte_eth_rss_conf rss_conf;
1238 union vnic_rss_cpu rss_cpu;
1241 rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1243 * If setting key for the first time, and the user gives us none, then
1244 * push the default key to NIC.
1246 if (rss_conf.rss_key == NULL) {
1247 rss_conf.rss_key = default_rss_key;
1248 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1250 ret = enic_set_rss_conf(enic, &rss_conf);
1252 dev_err(enic, "Failed to configure RSS\n");
1255 if (enic->rss_enable) {
1256 /* If enabling RSS, use the default reta */
1257 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1258 rss_cpu.cpu[i / 4].b[i % 4] =
1259 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1261 ret = enic_set_rss_reta(enic, &rss_cpu);
1263 dev_err(enic, "Failed to set RSS indirection table\n");
1268 int enic_setup_finish(struct enic *enic)
1270 enic_init_soft_stats(enic);
1273 vnic_dev_packet_filter(enic->vdev,
1286 static int enic_rss_conf_valid(struct enic *enic,
1287 struct rte_eth_rss_conf *rss_conf)
1289 /* RSS is disabled per VIC settings. Ignore rss_conf. */
1290 if (enic->flow_type_rss_offloads == 0)
1292 if (rss_conf->rss_key != NULL &&
1293 rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1294 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1295 rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1298 if (rss_conf->rss_hf != 0 &&
1299 (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1300 dev_err(enic, "Given rss_hf contains none of the supported"
1307 /* Set hash type and key according to rss_conf */
1308 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1310 struct rte_eth_dev *eth_dev;
1312 uint8_t rss_hash_type;
1316 RTE_ASSERT(rss_conf != NULL);
1317 ret = enic_rss_conf_valid(enic, rss_conf);
1319 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1323 eth_dev = enic->rte_dev;
1325 rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1326 if (enic->rq_count > 1 &&
1327 (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1330 if (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1331 ETH_RSS_NONFRAG_IPV4_OTHER))
1332 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1333 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1334 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1335 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
1336 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4;
1337 if (enic->udp_rss_weak) {
1339 * 'TCP' is not a typo. The "weak" version of
1340 * UDP RSS requires both the TCP and UDP bits
1341 * be set. It does enable TCP RSS as well.
1343 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1346 if (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_IPV6_EX |
1347 ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER))
1348 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1349 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX))
1350 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1351 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_UDP_EX)) {
1352 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6;
1353 if (enic->udp_rss_weak)
1354 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1361 /* Set the hash key if provided */
1362 if (rss_enable && rss_conf->rss_key) {
1363 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1365 dev_err(enic, "Failed to set RSS key\n");
1370 ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1371 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1374 enic->rss_hf = rss_hf;
1375 enic->rss_hash_type = rss_hash_type;
1376 enic->rss_enable = rss_enable;
1378 dev_err(enic, "Failed to update RSS configurations."
1379 " hash=0x%x\n", rss_hash_type);
1384 int enic_set_vlan_strip(struct enic *enic)
1387 * Unfortunately, VLAN strip on/off and RSS on/off are configured
1388 * together. So, re-do niccfg, preserving the current RSS settings.
1390 return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1391 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1395 int enic_add_packet_filter(struct enic *enic)
1397 /* Args -> directed, multicast, broadcast, promisc, allmulti */
1398 return vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1399 enic->promisc, enic->allmulti);
1402 int enic_get_link_status(struct enic *enic)
1404 return vnic_dev_link_status(enic->vdev);
1407 static void enic_dev_deinit(struct enic *enic)
1409 /* stop link status checking */
1410 vnic_dev_notify_unset(enic->vdev);
1412 /* mac_addrs is freed by rte_eth_dev_release_port() */
1414 rte_free(enic->intr);
1420 int enic_set_vnic_res(struct enic *enic)
1422 struct rte_eth_dev *eth_dev = enic->rte_dev;
1424 unsigned int required_rq, required_wq, required_cq, required_intr;
1426 /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1427 required_rq = eth_dev->data->nb_rx_queues * 2;
1428 required_wq = eth_dev->data->nb_tx_queues;
1429 required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1430 required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1431 if (eth_dev->data->dev_conf.intr_conf.rxq) {
1432 required_intr += eth_dev->data->nb_rx_queues;
1435 if (enic->conf_rq_count < required_rq) {
1436 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1437 eth_dev->data->nb_rx_queues,
1438 required_rq, enic->conf_rq_count);
1441 if (enic->conf_wq_count < required_wq) {
1442 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1443 eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1447 if (enic->conf_cq_count < required_cq) {
1448 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1449 required_cq, enic->conf_cq_count);
1452 if (enic->conf_intr_count < required_intr) {
1453 dev_err(dev, "Not enough Interrupts to support Rx queue"
1454 " interrupts. Required:%u, Configured:%u\n",
1455 required_intr, enic->conf_intr_count);
1460 enic->rq_count = eth_dev->data->nb_rx_queues;
1461 enic->wq_count = eth_dev->data->nb_tx_queues;
1462 enic->cq_count = enic->rq_count + enic->wq_count;
1463 enic->intr_count = required_intr;
1469 /* Initialize the completion queue for an RQ */
1471 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1473 struct vnic_rq *sop_rq, *data_rq;
1474 unsigned int cq_idx;
1477 sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1478 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1481 vnic_cq_clean(&enic->cq[cq_idx]);
1482 vnic_cq_init(&enic->cq[cq_idx],
1483 0 /* flow_control_enable */,
1484 1 /* color_enable */,
1487 1 /* cq_tail_color */,
1488 0 /* interrupt_enable */,
1489 1 /* cq_entry_enable */,
1490 0 /* cq_message_enable */,
1491 0 /* interrupt offset */,
1492 0 /* cq_message_addr */);
1495 vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1496 enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1497 sop_rq->ring.desc_count - 1, 1, 0);
1498 if (data_rq->in_use) {
1499 vnic_rq_init_start(data_rq,
1501 enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1502 data_rq->ring.desc_count - 1, 1, 0);
1505 rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1509 if (data_rq->in_use) {
1510 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1512 enic_rxmbuf_queue_release(enic, sop_rq);
1520 /* The Cisco NIC can send and receive packets up to a max packet size
1521 * determined by the NIC type and firmware. There is also an MTU
1522 * configured into the NIC via the CIMC/UCSM management interface
1523 * which can be overridden by this function (up to the max packet size).
1524 * Depending on the network setup, doing so may cause packet drops
1525 * and unexpected behavior.
1527 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1529 unsigned int rq_idx;
1532 uint16_t old_mtu; /* previous setting */
1533 uint16_t config_mtu; /* Value configured into NIC via CIMC/UCSM */
1534 struct rte_eth_dev *eth_dev = enic->rte_dev;
1536 old_mtu = eth_dev->data->mtu;
1537 config_mtu = enic->config.mtu;
1539 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1540 return -E_RTE_SECONDARY;
1542 if (new_mtu > enic->max_mtu) {
1544 "MTU not updated: requested (%u) greater than max (%u)\n",
1545 new_mtu, enic->max_mtu);
1548 if (new_mtu < ENIC_MIN_MTU) {
1550 "MTU not updated: requested (%u) less than min (%u)\n",
1551 new_mtu, ENIC_MIN_MTU);
1554 if (new_mtu > config_mtu)
1556 "MTU (%u) is greater than value configured in NIC (%u)\n",
1557 new_mtu, config_mtu);
1559 /* Update the MTU and maximum packet length */
1560 eth_dev->data->mtu = new_mtu;
1561 eth_dev->data->dev_conf.rxmode.max_rx_pkt_len =
1562 enic_mtu_to_max_rx_pktlen(new_mtu);
1565 * If the device has not started (enic_enable), nothing to do.
1566 * Later, enic_enable() will set up RQs reflecting the new maximum
1569 if (!eth_dev->data->dev_started)
1573 * The device has started, re-do RQs on the fly. In the process, we
1574 * pick up the new maximum packet length.
1576 * Some applications rely on the ability to change MTU without stopping
1577 * the device. So keep this behavior for now.
1579 rte_spinlock_lock(&enic->mtu_lock);
1581 /* Stop traffic on all RQs */
1582 for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1583 rq = &enic->rq[rq_idx];
1584 if (rq->is_sop && rq->in_use) {
1585 rc = enic_stop_rq(enic,
1586 enic_sop_rq_idx_to_rte_idx(rq_idx));
1588 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1594 /* replace Rx function with a no-op to avoid getting stale pkts */
1595 eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1598 /* Allow time for threads to exit the real Rx function. */
1601 /* now it is safe to reconfigure the RQs */
1604 /* free and reallocate RQs with the new MTU */
1605 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1606 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1611 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1612 rq->tot_nb_desc, rq->rx_free_thresh);
1615 "Fatal MTU alloc error- No traffic will pass\n");
1619 rc = enic_reinit_rq(enic, rq_idx);
1622 "Fatal MTU RQ reinit- No traffic will pass\n");
1627 /* put back the real receive function */
1629 enic_pick_rx_handler(eth_dev);
1632 /* restart Rx traffic */
1633 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1634 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1635 if (rq->is_sop && rq->in_use)
1636 enic_start_rq(enic, rq_idx);
1640 dev_info(enic, "MTU changed from %u to %u\n", old_mtu, new_mtu);
1641 rte_spinlock_unlock(&enic->mtu_lock);
1645 static int enic_dev_init(struct enic *enic)
1648 struct rte_eth_dev *eth_dev = enic->rte_dev;
1650 vnic_dev_intr_coal_timer_info_default(enic->vdev);
1652 /* Get vNIC configuration
1654 err = enic_get_vnic_config(enic);
1656 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1660 /* Get available resource counts */
1661 enic_get_res_counts(enic);
1662 if (enic->conf_rq_count == 1) {
1663 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1664 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1665 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1668 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1669 enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1670 enic->conf_cq_count, 8);
1671 enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1672 enic->conf_intr_count, 8);
1673 enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1674 enic->conf_rq_count, 8);
1675 enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1676 enic->conf_wq_count, 8);
1677 if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1678 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1681 if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1682 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1685 if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1686 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1689 if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1690 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1694 /* Get the supported filters */
1695 enic_fdir_info(enic);
1697 eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr",
1698 sizeof(struct rte_ether_addr) *
1699 ENIC_UNICAST_PERFECT_FILTERS, 0);
1700 if (!eth_dev->data->mac_addrs) {
1701 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1704 rte_ether_addr_copy((struct rte_ether_addr *)enic->mac_addr,
1705 eth_dev->data->mac_addrs);
1707 vnic_dev_set_reset_flag(enic->vdev, 0);
1709 LIST_INIT(&enic->flows);
1711 /* set up link status checking */
1712 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1715 * When Geneve with options offload is available, always disable it
1716 * first as it can interfere with user flow rules.
1718 if (enic->geneve_opt_avail) {
1720 * Disabling fails if the feature is provisioned but
1721 * not enabled. So ignore result and do not log error.
1723 vnic_dev_overlay_offload_ctrl(enic->vdev,
1724 OVERLAY_FEATURE_GENEVE,
1725 OVERLAY_OFFLOAD_DISABLE);
1727 enic->overlay_offload = false;
1728 if (enic->disable_overlay && enic->vxlan) {
1730 * Explicitly disable overlay offload as the setting is
1731 * sticky, and resetting vNIC does not disable it.
1733 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1734 OVERLAY_FEATURE_VXLAN,
1735 OVERLAY_OFFLOAD_DISABLE)) {
1736 dev_err(enic, "failed to disable overlay offload\n");
1738 dev_info(enic, "Overlay offload is disabled\n");
1741 if (!enic->disable_overlay && enic->vxlan &&
1742 /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1743 vnic_dev_overlay_offload_ctrl(enic->vdev,
1744 OVERLAY_FEATURE_VXLAN,
1745 OVERLAY_OFFLOAD_ENABLE) == 0) {
1746 enic->tx_offload_capa |=
1747 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1748 DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
1749 DEV_TX_OFFLOAD_VXLAN_TNL_TSO;
1750 enic->tx_offload_mask |=
1753 PKT_TX_OUTER_IP_CKSUM |
1755 enic->overlay_offload = true;
1756 dev_info(enic, "Overlay offload is enabled\n");
1758 /* Geneve with options offload requires overlay offload */
1759 if (enic->overlay_offload && enic->geneve_opt_avail &&
1760 enic->geneve_opt_request) {
1761 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1762 OVERLAY_FEATURE_GENEVE,
1763 OVERLAY_OFFLOAD_ENABLE)) {
1764 dev_err(enic, "failed to enable geneve+option\n");
1766 enic->geneve_opt_enabled = 1;
1767 dev_info(enic, "Geneve with options is enabled\n");
1771 * Reset the vxlan port if HW vxlan parsing is available. It
1772 * is always enabled regardless of overlay offload
1776 enic->vxlan_port = RTE_VXLAN_DEFAULT_PORT;
1778 * Reset the vxlan port to the default, as the NIC firmware
1779 * does not reset it automatically and keeps the old setting.
1781 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1782 OVERLAY_CFG_VXLAN_PORT_UPDATE,
1783 RTE_VXLAN_DEFAULT_PORT)) {
1784 dev_err(enic, "failed to update vxlan port\n");
1793 int enic_probe(struct enic *enic)
1795 struct rte_pci_device *pdev = enic->pdev;
1798 dev_debug(enic, "Initializing ENIC PMD\n");
1800 /* if this is a secondary process the hardware is already initialized */
1801 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1804 enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1805 enic->bar0.len = pdev->mem_resource[0].len;
1807 /* Register vNIC device */
1808 enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1810 dev_err(enic, "vNIC registration failed, aborting\n");
1814 LIST_INIT(&enic->memzone_list);
1815 rte_spinlock_init(&enic->memzone_list_lock);
1817 vnic_register_cbacks(enic->vdev,
1818 enic_alloc_consistent,
1819 enic_free_consistent);
1822 * Allocate the consistent memory for stats upfront so both primary and
1823 * secondary processes can dump stats.
1825 err = vnic_dev_alloc_stats_mem(enic->vdev);
1827 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1828 goto err_out_unregister;
1830 /* Issue device open to get device in known state */
1831 err = enic_dev_open(enic);
1833 dev_err(enic, "vNIC dev open failed, aborting\n");
1834 goto err_out_unregister;
1837 /* Set ingress vlan rewrite mode before vnic initialization */
1838 dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n",
1839 enic->ig_vlan_rewrite_mode);
1840 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1841 enic->ig_vlan_rewrite_mode);
1844 "Failed to set ingress vlan rewrite mode, aborting.\n");
1845 goto err_out_dev_close;
1848 /* Issue device init to initialize the vnic-to-switch link.
1849 * We'll start with carrier off and wait for link UP
1850 * notification later to turn on carrier. We don't need
1851 * to wait here for the vnic-to-switch link initialization
1852 * to complete; link UP notification is the indication that
1853 * the process is complete.
1856 err = vnic_dev_init(enic->vdev, 0);
1858 dev_err(enic, "vNIC dev init failed, aborting\n");
1859 goto err_out_dev_close;
1862 err = enic_dev_init(enic);
1864 dev_err(enic, "Device initialization failed, aborting\n");
1865 goto err_out_dev_close;
1871 vnic_dev_close(enic->vdev);
1873 vnic_dev_unregister(enic->vdev);
1878 void enic_remove(struct enic *enic)
1880 enic_dev_deinit(enic);
1881 vnic_dev_close(enic->vdev);
1882 vnic_dev_unregister(enic->vdev);