1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
13 #include <rte_bus_pci.h>
14 #include <rte_memzone.h>
15 #include <rte_malloc.h>
17 #include <rte_string_fns.h>
18 #include <rte_ethdev_driver.h>
20 #include "enic_compat.h"
22 #include "wq_enet_desc.h"
23 #include "rq_enet_desc.h"
24 #include "cq_enet_desc.h"
25 #include "vnic_enet.h"
30 #include "vnic_intr.h"
33 static inline int enic_is_sriov_vf(struct enic *enic)
35 return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
38 static int is_zero_addr(uint8_t *addr)
40 return !(addr[0] | addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
43 static int is_mcast_addr(uint8_t *addr)
48 static int is_eth_addr_valid(uint8_t *addr)
50 return !is_mcast_addr(addr) && !is_zero_addr(addr);
54 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
58 if (!rq || !rq->mbuf_ring) {
59 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
63 for (i = 0; i < rq->ring.desc_count; i++) {
64 if (rq->mbuf_ring[i]) {
65 rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
66 rq->mbuf_ring[i] = NULL;
71 static void enic_free_wq_buf(struct rte_mbuf **buf)
73 struct rte_mbuf *mbuf = *buf;
75 rte_pktmbuf_free_seg(mbuf);
79 static void enic_log_q_error(struct enic *enic)
84 for (i = 0; i < enic->wq_count; i++) {
85 error_status = vnic_wq_error_status(&enic->wq[i]);
87 dev_err(enic, "WQ[%d] error_status %d\n", i,
91 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
92 if (!enic->rq[i].in_use)
94 error_status = vnic_rq_error_status(&enic->rq[i]);
96 dev_err(enic, "RQ[%d] error_status %d\n", i,
101 static void enic_clear_soft_stats(struct enic *enic)
103 struct enic_soft_stats *soft_stats = &enic->soft_stats;
104 rte_atomic64_clear(&soft_stats->rx_nombuf);
105 rte_atomic64_clear(&soft_stats->rx_packet_errors);
106 rte_atomic64_clear(&soft_stats->tx_oversized);
109 static void enic_init_soft_stats(struct enic *enic)
111 struct enic_soft_stats *soft_stats = &enic->soft_stats;
112 rte_atomic64_init(&soft_stats->rx_nombuf);
113 rte_atomic64_init(&soft_stats->rx_packet_errors);
114 rte_atomic64_init(&soft_stats->tx_oversized);
115 enic_clear_soft_stats(enic);
118 int enic_dev_stats_clear(struct enic *enic)
122 ret = vnic_dev_stats_clear(enic->vdev);
124 dev_err(enic, "Error in clearing stats\n");
127 enic_clear_soft_stats(enic);
132 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
134 struct vnic_stats *stats;
135 struct enic_soft_stats *soft_stats = &enic->soft_stats;
136 int64_t rx_truncated;
137 uint64_t rx_packet_errors;
138 int ret = vnic_dev_stats_dump(enic->vdev, &stats);
141 dev_err(enic, "Error in getting stats\n");
145 /* The number of truncated packets can only be calculated by
146 * subtracting a hardware counter from error packets received by
147 * the driver. Note: this causes transient inaccuracies in the
148 * ipackets count. Also, the length of truncated packets are
149 * counted in ibytes even though truncated packets are dropped
150 * which can make ibytes be slightly higher than it should be.
152 rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
153 rx_truncated = rx_packet_errors - stats->rx.rx_errors;
155 r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
156 r_stats->opackets = stats->tx.tx_frames_ok;
158 r_stats->ibytes = stats->rx.rx_bytes_ok;
159 r_stats->obytes = stats->tx.tx_bytes_ok;
161 r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
162 r_stats->oerrors = stats->tx.tx_errors
163 + rte_atomic64_read(&soft_stats->tx_oversized);
165 r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
167 r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
171 int enic_del_mac_address(struct enic *enic, int mac_index)
173 struct rte_eth_dev *eth_dev = enic->rte_dev;
174 uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
176 return vnic_dev_del_addr(enic->vdev, mac_addr);
179 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
183 if (!is_eth_addr_valid(mac_addr)) {
184 dev_err(enic, "invalid mac address\n");
188 err = vnic_dev_add_addr(enic->vdev, mac_addr);
190 dev_err(enic, "add mac addr failed\n");
195 enic_free_rq_buf(struct rte_mbuf **mbuf)
200 rte_pktmbuf_free(*mbuf);
204 void enic_init_vnic_resources(struct enic *enic)
206 unsigned int error_interrupt_enable = 1;
207 unsigned int error_interrupt_offset = 0;
208 unsigned int rxq_interrupt_enable = 0;
209 unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
210 unsigned int index = 0;
212 struct vnic_rq *data_rq;
214 if (enic->rte_dev->data->dev_conf.intr_conf.rxq)
215 rxq_interrupt_enable = 1;
217 for (index = 0; index < enic->rq_count; index++) {
218 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
220 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
222 error_interrupt_enable,
223 error_interrupt_offset);
225 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
227 vnic_rq_init(data_rq,
229 error_interrupt_enable,
230 error_interrupt_offset);
232 vnic_cq_init(&enic->cq[cq_idx],
233 0 /* flow_control_enable */,
234 1 /* color_enable */,
237 1 /* cq_tail_color */,
238 rxq_interrupt_enable,
239 1 /* cq_entry_enable */,
240 0 /* cq_message_enable */,
241 rxq_interrupt_offset,
242 0 /* cq_message_addr */);
243 if (rxq_interrupt_enable)
244 rxq_interrupt_offset++;
247 for (index = 0; index < enic->wq_count; index++) {
248 vnic_wq_init(&enic->wq[index],
249 enic_cq_wq(enic, index),
250 error_interrupt_enable,
251 error_interrupt_offset);
252 /* Compute unsupported ol flags for enic_prep_pkts() */
253 enic->wq[index].tx_offload_notsup_mask =
254 PKT_TX_OFFLOAD_MASK ^ enic->tx_offload_mask;
256 cq_idx = enic_cq_wq(enic, index);
257 vnic_cq_init(&enic->cq[cq_idx],
258 0 /* flow_control_enable */,
259 1 /* color_enable */,
262 1 /* cq_tail_color */,
263 0 /* interrupt_enable */,
264 0 /* cq_entry_enable */,
265 1 /* cq_message_enable */,
266 0 /* interrupt offset */,
267 (u64)enic->wq[index].cqmsg_rz->iova);
270 for (index = 0; index < enic->intr_count; index++) {
271 vnic_intr_init(&enic->intr[index],
272 enic->config.intr_timer_usec,
273 enic->config.intr_timer_type,
274 /*mask_on_assertion*/1);
280 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
283 struct rq_enet_desc *rqd = rq->ring.descs;
286 uint32_t max_rx_pkt_len;
292 dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
293 rq->ring.desc_count);
296 * If *not* using scatter and the mbuf size is greater than the
297 * requested max packet size (max_rx_pkt_len), then reduce the
298 * posted buffer size to max_rx_pkt_len. HW still receives packets
299 * larger than max_rx_pkt_len, but they will be truncated, which we
300 * drop in the rx handler. Not ideal, but better than returning
301 * large packets when the user is not expecting them.
303 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
304 rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
305 if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
306 rq_buf_len = max_rx_pkt_len;
307 for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
308 mb = rte_mbuf_raw_alloc(rq->mp);
310 dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
311 (unsigned)rq->index);
315 mb->data_off = RTE_PKTMBUF_HEADROOM;
316 dma_addr = (dma_addr_t)(mb->buf_iova
317 + RTE_PKTMBUF_HEADROOM);
318 rq_enet_desc_enc(rqd, dma_addr,
319 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
320 : RQ_ENET_TYPE_NOT_SOP),
322 rq->mbuf_ring[i] = mb;
325 * Do not post the buffers to the NIC until we enable the RQ via
328 rq->need_initial_post = true;
329 /* Initialize fetch index while RQ is disabled */
330 iowrite32(0, &rq->ctrl->fetch_index);
335 * Post the Rx buffers for the first time. enic_alloc_rx_queue_mbufs() has
336 * allocated the buffers and filled the RQ descriptor ring. Just need to push
337 * the post index to the NIC.
340 enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)
342 if (!rq->in_use || !rq->need_initial_post)
345 /* make sure all prior writes are complete before doing the PIO write */
348 /* Post all but the last buffer to VIC. */
349 rq->posted_index = rq->ring.desc_count - 1;
353 dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
354 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
355 iowrite32(rq->posted_index, &rq->ctrl->posted_index);
357 rq->need_initial_post = false;
361 enic_alloc_consistent(void *priv, size_t size,
362 dma_addr_t *dma_handle, u8 *name)
365 const struct rte_memzone *rz;
367 struct enic *enic = (struct enic *)priv;
368 struct enic_memzone_entry *mze;
370 rz = rte_memzone_reserve_aligned((const char *)name, size,
371 SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
373 pr_err("%s : Failed to allocate memory requested for %s\n",
379 *dma_handle = (dma_addr_t)rz->iova;
381 mze = rte_malloc("enic memzone entry",
382 sizeof(struct enic_memzone_entry), 0);
385 pr_err("%s : Failed to allocate memory for memzone list\n",
387 rte_memzone_free(rz);
393 rte_spinlock_lock(&enic->memzone_list_lock);
394 LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
395 rte_spinlock_unlock(&enic->memzone_list_lock);
401 enic_free_consistent(void *priv,
402 __rte_unused size_t size,
404 dma_addr_t dma_handle)
406 struct enic_memzone_entry *mze;
407 struct enic *enic = (struct enic *)priv;
409 rte_spinlock_lock(&enic->memzone_list_lock);
410 LIST_FOREACH(mze, &enic->memzone_list, entries) {
411 if (mze->rz->addr == vaddr &&
412 mze->rz->iova == dma_handle)
416 rte_spinlock_unlock(&enic->memzone_list_lock);
418 "Tried to free memory, but couldn't find it in the memzone list\n");
421 LIST_REMOVE(mze, entries);
422 rte_spinlock_unlock(&enic->memzone_list_lock);
423 rte_memzone_free(mze->rz);
427 int enic_link_update(struct rte_eth_dev *eth_dev)
429 struct enic *enic = pmd_priv(eth_dev);
430 struct rte_eth_link link;
432 memset(&link, 0, sizeof(link));
433 link.link_status = enic_get_link_status(enic);
434 link.link_duplex = ETH_LINK_FULL_DUPLEX;
435 link.link_speed = vnic_dev_port_speed(enic->vdev);
437 return rte_eth_linkstatus_set(eth_dev, &link);
441 enic_intr_handler(void *arg)
443 struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
444 struct enic *enic = pmd_priv(dev);
446 vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
448 enic_link_update(dev);
449 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
450 enic_log_q_error(enic);
453 static int enic_rxq_intr_init(struct enic *enic)
455 struct rte_intr_handle *intr_handle;
456 uint32_t rxq_intr_count, i;
459 intr_handle = enic->rte_dev->intr_handle;
460 if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
463 * Rx queue interrupts only work when we have MSI-X interrupts,
464 * one per queue. Sharing one interrupt is technically
465 * possible with VIC, but it is not worth the complications it brings.
467 if (!rte_intr_cap_multiple(intr_handle)) {
468 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
469 " (vfio-pci driver)\n");
472 rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
473 err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
475 dev_err(enic, "Failed to enable event fds for Rx queue"
479 intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
480 rxq_intr_count * sizeof(int), 0);
481 if (intr_handle->intr_vec == NULL) {
482 dev_err(enic, "Failed to allocate intr_vec\n");
485 for (i = 0; i < rxq_intr_count; i++)
486 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
490 static void enic_rxq_intr_deinit(struct enic *enic)
492 struct rte_intr_handle *intr_handle;
494 intr_handle = enic->rte_dev->intr_handle;
495 rte_intr_efd_disable(intr_handle);
496 if (intr_handle->intr_vec != NULL) {
497 rte_free(intr_handle->intr_vec);
498 intr_handle->intr_vec = NULL;
502 static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
504 struct wq_enet_desc *desc;
509 * Fill WQ descriptor fields that never change. Every descriptor is
510 * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
511 * descriptors (i.e. request one completion update every 32 packets).
513 wq = &enic->wq[queue_idx];
514 desc = (struct wq_enet_desc *)wq->ring.descs;
515 for (i = 0; i < wq->ring.desc_count; i++, desc++) {
516 desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
517 if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
518 desc->header_length_flags |=
519 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
524 * The 'strong' version is in enic_rxtx_vec_avx2.c. This weak version is used
525 * used when that file is not compiled.
528 enic_use_vector_rx_handler(__rte_unused struct rte_eth_dev *eth_dev)
533 void enic_pick_rx_handler(struct rte_eth_dev *eth_dev)
535 struct enic *enic = pmd_priv(eth_dev);
539 * 1. The vectorized handler if possible and requested.
540 * 2. The non-scatter, simplified handler if scatter Rx is not used.
541 * 3. The default handler as a fallback.
543 if (enic_use_vector_rx_handler(eth_dev))
545 if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
546 ENICPMD_LOG(DEBUG, " use the non-scatter Rx handler");
547 eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
549 ENICPMD_LOG(DEBUG, " use the normal Rx handler");
550 eth_dev->rx_pkt_burst = &enic_recv_pkts;
554 /* Secondary process uses this to set the Tx handler */
555 void enic_pick_tx_handler(struct rte_eth_dev *eth_dev)
557 struct enic *enic = pmd_priv(eth_dev);
559 if (enic->use_simple_tx_handler) {
560 ENICPMD_LOG(DEBUG, " use the simple tx handler");
561 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
563 ENICPMD_LOG(DEBUG, " use the default tx handler");
564 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
568 int enic_enable(struct enic *enic)
572 struct rte_eth_dev *eth_dev = enic->rte_dev;
573 uint64_t simple_tx_offloads;
576 if (enic->enable_avx2_rx) {
577 struct rte_mbuf mb_def = { .buf_addr = 0 };
580 * mbuf_initializer contains const-after-init fields of
581 * receive mbufs (i.e. 64 bits of fields from rearm_data).
582 * It is currently used by the vectorized handler.
585 mb_def.data_off = RTE_PKTMBUF_HEADROOM;
586 mb_def.port = enic->port_id;
587 rte_mbuf_refcnt_set(&mb_def, 1);
588 rte_compiler_barrier();
589 p = (uintptr_t)&mb_def.rearm_data;
590 enic->mbuf_initializer = *(uint64_t *)p;
593 eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
594 eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
596 /* vnic notification of link status has already been turned on in
597 * enic_dev_init() which is called during probe time. Here we are
598 * just turning on interrupt vector 0 if needed.
600 if (eth_dev->data->dev_conf.intr_conf.lsc)
601 vnic_dev_notify_set(enic->vdev, 0);
603 err = enic_rxq_intr_init(enic);
606 if (enic_clsf_init(enic))
607 dev_warning(enic, "Init of hash table for clsf failed."\
608 "Flow director feature will not work\n");
610 if (enic_fm_init(enic))
611 dev_warning(enic, "Init of flowman failed.\n");
613 for (index = 0; index < enic->rq_count; index++) {
614 err = enic_alloc_rx_queue_mbufs(enic,
615 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
617 dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
620 err = enic_alloc_rx_queue_mbufs(enic,
621 &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
623 /* release the allocated mbufs for the sop rq*/
624 enic_rxmbuf_queue_release(enic,
625 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
627 dev_err(enic, "Failed to alloc data RX queue mbufs\n");
633 * Use the simple TX handler if possible. Only checksum offloads
634 * and vlan insertion are supported.
636 simple_tx_offloads = enic->tx_offload_capa &
637 (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
638 DEV_TX_OFFLOAD_VLAN_INSERT |
639 DEV_TX_OFFLOAD_IPV4_CKSUM |
640 DEV_TX_OFFLOAD_UDP_CKSUM |
641 DEV_TX_OFFLOAD_TCP_CKSUM);
642 if ((eth_dev->data->dev_conf.txmode.offloads &
643 ~simple_tx_offloads) == 0) {
644 ENICPMD_LOG(DEBUG, " use the simple tx handler");
645 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
646 for (index = 0; index < enic->wq_count; index++)
647 enic_prep_wq_for_simple_tx(enic, index);
648 enic->use_simple_tx_handler = 1;
650 ENICPMD_LOG(DEBUG, " use the default tx handler");
651 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
654 enic_pick_rx_handler(eth_dev);
656 for (index = 0; index < enic->wq_count; index++)
657 enic_start_wq(enic, index);
658 for (index = 0; index < enic->rq_count; index++)
659 enic_start_rq(enic, index);
661 vnic_dev_add_addr(enic->vdev, enic->mac_addr);
663 vnic_dev_enable_wait(enic->vdev);
665 /* Register and enable error interrupt */
666 rte_intr_callback_register(&(enic->pdev->intr_handle),
667 enic_intr_handler, (void *)enic->rte_dev);
669 rte_intr_enable(&(enic->pdev->intr_handle));
670 /* Unmask LSC interrupt */
671 vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
676 int enic_alloc_intr_resources(struct enic *enic)
681 dev_info(enic, "vNIC resources used: "\
682 "wq %d rq %d cq %d intr %d\n",
683 enic->wq_count, enic_vnic_rq_count(enic),
684 enic->cq_count, enic->intr_count);
686 for (i = 0; i < enic->intr_count; i++) {
687 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
689 enic_free_vnic_resources(enic);
696 void enic_free_rq(void *rxq)
698 struct vnic_rq *rq_sop, *rq_data;
704 rq_sop = (struct vnic_rq *)rxq;
705 enic = vnic_dev_priv(rq_sop->vdev);
706 rq_data = &enic->rq[rq_sop->data_queue_idx];
708 if (rq_sop->free_mbufs) {
709 struct rte_mbuf **mb;
712 mb = rq_sop->free_mbufs;
713 for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs;
714 i < ENIC_RX_BURST_MAX; i++)
715 rte_pktmbuf_free(mb[i]);
716 rte_free(rq_sop->free_mbufs);
717 rq_sop->free_mbufs = NULL;
718 rq_sop->num_free_mbufs = 0;
721 enic_rxmbuf_queue_release(enic, rq_sop);
723 enic_rxmbuf_queue_release(enic, rq_data);
725 rte_free(rq_sop->mbuf_ring);
727 rte_free(rq_data->mbuf_ring);
729 rq_sop->mbuf_ring = NULL;
730 rq_data->mbuf_ring = NULL;
732 vnic_rq_free(rq_sop);
734 vnic_rq_free(rq_data);
736 vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
742 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
744 struct rte_eth_dev_data *data = enic->dev_data;
745 vnic_wq_enable(&enic->wq[queue_idx]);
746 data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
749 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
751 struct rte_eth_dev_data *data = enic->dev_data;
754 ret = vnic_wq_disable(&enic->wq[queue_idx]);
758 data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
762 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
764 struct rte_eth_dev_data *data = enic->dev_data;
765 struct vnic_rq *rq_sop;
766 struct vnic_rq *rq_data;
767 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
768 rq_data = &enic->rq[rq_sop->data_queue_idx];
770 if (rq_data->in_use) {
771 vnic_rq_enable(rq_data);
772 enic_initial_post_rx(enic, rq_data);
775 vnic_rq_enable(rq_sop);
776 enic_initial_post_rx(enic, rq_sop);
777 data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
780 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
782 struct rte_eth_dev_data *data = enic->dev_data;
783 int ret1 = 0, ret2 = 0;
784 struct vnic_rq *rq_sop;
785 struct vnic_rq *rq_data;
786 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
787 rq_data = &enic->rq[rq_sop->data_queue_idx];
789 ret2 = vnic_rq_disable(rq_sop);
792 ret1 = vnic_rq_disable(rq_data);
799 data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
803 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
804 unsigned int socket_id, struct rte_mempool *mp,
805 uint16_t nb_desc, uint16_t free_thresh)
808 uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
809 uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
810 struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
811 struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
812 unsigned int mbuf_size, mbufs_per_pkt;
813 unsigned int nb_sop_desc, nb_data_desc;
814 uint16_t min_sop, max_sop, min_data, max_data;
815 uint32_t max_rx_pkt_len;
818 rq_sop->data_queue_idx = data_queue_idx;
820 rq_data->data_queue_idx = 0;
821 rq_sop->socket_id = socket_id;
823 rq_data->socket_id = socket_id;
826 rq_sop->rx_free_thresh = free_thresh;
827 rq_data->rx_free_thresh = free_thresh;
828 dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
831 mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
832 RTE_PKTMBUF_HEADROOM);
833 /* max_rx_pkt_len includes the ethernet header and CRC. */
834 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
836 if (enic->rte_dev->data->dev_conf.rxmode.offloads &
837 DEV_RX_OFFLOAD_SCATTER) {
838 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
839 /* ceil((max pkt len)/mbuf_size) */
840 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
842 dev_info(enic, "Scatter rx mode disabled\n");
844 if (max_rx_pkt_len > mbuf_size) {
845 dev_warning(enic, "The maximum Rx packet size (%u) is"
846 " larger than the mbuf size (%u), and"
847 " scatter is disabled. Larger packets will"
849 max_rx_pkt_len, mbuf_size);
853 if (mbufs_per_pkt > 1) {
854 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
855 rq_sop->data_queue_enable = 1;
858 * HW does not directly support rxmode.max_rx_pkt_len. HW always
859 * receives packet sizes up to the "max" MTU.
860 * If not using scatter, we can achieve the effect of dropping
861 * larger packets by reducing the size of posted buffers.
862 * See enic_alloc_rx_queue_mbufs().
865 enic_mtu_to_max_rx_pktlen(enic->max_mtu)) {
866 dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
867 " when scatter rx mode is in use.\n");
870 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
872 rq_sop->data_queue_enable = 0;
876 /* number of descriptors have to be a multiple of 32 */
877 nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK;
878 nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK;
880 rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
881 rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
883 if (mbufs_per_pkt > 1) {
884 min_sop = ENIC_RX_BURST_MAX;
885 max_sop = ((enic->config.rq_desc_count /
886 (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK);
887 min_data = min_sop * (mbufs_per_pkt - 1);
888 max_data = enic->config.rq_desc_count;
890 min_sop = ENIC_RX_BURST_MAX;
891 max_sop = enic->config.rq_desc_count;
896 if (nb_desc < (min_sop + min_data)) {
898 "Number of rx descs too low, adjusting to minimum\n");
899 nb_sop_desc = min_sop;
900 nb_data_desc = min_data;
901 } else if (nb_desc > (max_sop + max_data)) {
903 "Number of rx_descs too high, adjusting to maximum\n");
904 nb_sop_desc = max_sop;
905 nb_data_desc = max_data;
907 if (mbufs_per_pkt > 1) {
908 dev_info(enic, "For max packet size %u and mbuf size %u valid"
909 " rx descriptor range is %u to %u\n",
910 max_rx_pkt_len, mbuf_size, min_sop + min_data,
913 dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
914 nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
916 /* Allocate sop queue resources */
917 rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
918 nb_sop_desc, sizeof(struct rq_enet_desc));
920 dev_err(enic, "error in allocation of sop rq\n");
923 nb_sop_desc = rq_sop->ring.desc_count;
925 if (rq_data->in_use) {
926 /* Allocate data queue resources */
927 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
929 sizeof(struct rq_enet_desc));
931 dev_err(enic, "error in allocation of data rq\n");
932 goto err_free_rq_sop;
934 nb_data_desc = rq_data->ring.desc_count;
936 rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
937 socket_id, nb_sop_desc + nb_data_desc,
938 sizeof(struct cq_enet_rq_desc));
940 dev_err(enic, "error in allocation of cq for rq\n");
941 goto err_free_rq_data;
944 /* Allocate the mbuf rings */
945 rq_sop->mbuf_ring = (struct rte_mbuf **)
946 rte_zmalloc_socket("rq->mbuf_ring",
947 sizeof(struct rte_mbuf *) * nb_sop_desc,
948 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
949 if (rq_sop->mbuf_ring == NULL)
952 if (rq_data->in_use) {
953 rq_data->mbuf_ring = (struct rte_mbuf **)
954 rte_zmalloc_socket("rq->mbuf_ring",
955 sizeof(struct rte_mbuf *) * nb_data_desc,
956 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
957 if (rq_data->mbuf_ring == NULL)
958 goto err_free_sop_mbuf;
961 rq_sop->free_mbufs = (struct rte_mbuf **)
962 rte_zmalloc_socket("rq->free_mbufs",
963 sizeof(struct rte_mbuf *) *
965 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
966 if (rq_sop->free_mbufs == NULL)
967 goto err_free_data_mbuf;
968 rq_sop->num_free_mbufs = 0;
970 rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
975 rte_free(rq_data->mbuf_ring);
977 rte_free(rq_sop->mbuf_ring);
979 /* cleanup on error */
980 vnic_cq_free(&enic->cq[queue_idx]);
983 vnic_rq_free(rq_data);
985 vnic_rq_free(rq_sop);
990 void enic_free_wq(void *txq)
998 wq = (struct vnic_wq *)txq;
999 enic = vnic_dev_priv(wq->vdev);
1000 rte_memzone_free(wq->cqmsg_rz);
1002 vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
1005 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
1006 unsigned int socket_id, uint16_t nb_desc)
1009 struct vnic_wq *wq = &enic->wq[queue_idx];
1010 unsigned int cq_index = enic_cq_wq(enic, queue_idx);
1011 char name[NAME_MAX];
1012 static int instance;
1014 wq->socket_id = socket_id;
1016 * rte_eth_tx_queue_setup() checks min, max, and alignment. So just
1017 * print an info message for diagnostics.
1019 dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc);
1021 /* Allocate queue resources */
1022 err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
1024 sizeof(struct wq_enet_desc));
1026 dev_err(enic, "error in allocation of wq\n");
1030 err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
1032 sizeof(struct cq_enet_wq_desc));
1035 dev_err(enic, "error in allocation of cq for wq\n");
1038 /* setup up CQ message */
1039 snprintf((char *)name, sizeof(name),
1040 "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
1043 wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
1044 sizeof(uint32_t), SOCKET_ID_ANY,
1045 RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
1052 int enic_disable(struct enic *enic)
1057 for (i = 0; i < enic->intr_count; i++) {
1058 vnic_intr_mask(&enic->intr[i]);
1059 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
1061 enic_rxq_intr_deinit(enic);
1062 rte_intr_disable(&enic->pdev->intr_handle);
1063 rte_intr_callback_unregister(&enic->pdev->intr_handle,
1065 (void *)enic->rte_dev);
1067 vnic_dev_disable(enic->vdev);
1069 enic_clsf_destroy(enic);
1070 enic_fm_destroy(enic);
1072 if (!enic_is_sriov_vf(enic))
1073 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
1075 for (i = 0; i < enic->wq_count; i++) {
1076 err = vnic_wq_disable(&enic->wq[i]);
1080 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
1081 if (enic->rq[i].in_use) {
1082 err = vnic_rq_disable(&enic->rq[i]);
1088 /* If we were using interrupts, set the interrupt vector to -1
1089 * to disable interrupts. We are not disabling link notifcations,
1090 * though, as we want the polling of link status to continue working.
1092 if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
1093 vnic_dev_notify_set(enic->vdev, -1);
1095 vnic_dev_set_reset_flag(enic->vdev, 1);
1097 for (i = 0; i < enic->wq_count; i++)
1098 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
1100 for (i = 0; i < enic_vnic_rq_count(enic); i++)
1101 if (enic->rq[i].in_use)
1102 vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
1103 for (i = 0; i < enic->cq_count; i++)
1104 vnic_cq_clean(&enic->cq[i]);
1105 for (i = 0; i < enic->intr_count; i++)
1106 vnic_intr_clean(&enic->intr[i]);
1111 static int enic_dev_wait(struct vnic_dev *vdev,
1112 int (*start)(struct vnic_dev *, int),
1113 int (*finished)(struct vnic_dev *, int *),
1120 err = start(vdev, arg);
1124 /* Wait for func to complete...2 seconds max */
1125 for (i = 0; i < 2000; i++) {
1126 err = finished(vdev, &done);
1136 static int enic_dev_open(struct enic *enic)
1139 int flags = CMD_OPENF_IG_DESCCACHE;
1141 err = enic_dev_wait(enic->vdev, vnic_dev_open,
1142 vnic_dev_open_done, flags);
1144 dev_err(enic_get_dev(enic),
1145 "vNIC device open failed, err %d\n", err);
1150 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
1152 dma_addr_t rss_key_buf_pa;
1153 union vnic_rss_key *rss_key_buf_va = NULL;
1157 RTE_ASSERT(user_key != NULL);
1158 snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
1159 rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1160 &rss_key_buf_pa, name);
1161 if (!rss_key_buf_va)
1164 for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1165 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1167 err = enic_set_rss_key(enic,
1169 sizeof(union vnic_rss_key));
1171 /* Save for later queries */
1173 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1174 sizeof(union vnic_rss_key));
1176 enic_free_consistent(enic, sizeof(union vnic_rss_key),
1177 rss_key_buf_va, rss_key_buf_pa);
1182 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1184 dma_addr_t rss_cpu_buf_pa;
1185 union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1189 snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
1190 rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1191 &rss_cpu_buf_pa, name);
1192 if (!rss_cpu_buf_va)
1195 rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1197 err = enic_set_rss_cpu(enic,
1199 sizeof(union vnic_rss_cpu));
1201 enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1202 rss_cpu_buf_va, rss_cpu_buf_pa);
1204 /* Save for later queries */
1206 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1210 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
1211 u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
1213 const u8 tso_ipid_split_en = 0;
1216 err = enic_set_nic_cfg(enic,
1217 rss_default_cpu, rss_hash_type,
1218 rss_hash_bits, rss_base_cpu,
1219 rss_enable, tso_ipid_split_en,
1220 enic->ig_vlan_strip_en);
1225 /* Initialize RSS with defaults, called from dev_configure */
1226 int enic_init_rss_nic_cfg(struct enic *enic)
1228 static uint8_t default_rss_key[] = {
1229 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1230 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1231 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1232 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1234 struct rte_eth_rss_conf rss_conf;
1235 union vnic_rss_cpu rss_cpu;
1238 rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1240 * If setting key for the first time, and the user gives us none, then
1241 * push the default key to NIC.
1243 if (rss_conf.rss_key == NULL) {
1244 rss_conf.rss_key = default_rss_key;
1245 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1247 ret = enic_set_rss_conf(enic, &rss_conf);
1249 dev_err(enic, "Failed to configure RSS\n");
1252 if (enic->rss_enable) {
1253 /* If enabling RSS, use the default reta */
1254 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1255 rss_cpu.cpu[i / 4].b[i % 4] =
1256 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1258 ret = enic_set_rss_reta(enic, &rss_cpu);
1260 dev_err(enic, "Failed to set RSS indirection table\n");
1265 int enic_setup_finish(struct enic *enic)
1267 enic_init_soft_stats(enic);
1270 vnic_dev_packet_filter(enic->vdev,
1283 static int enic_rss_conf_valid(struct enic *enic,
1284 struct rte_eth_rss_conf *rss_conf)
1286 /* RSS is disabled per VIC settings. Ignore rss_conf. */
1287 if (enic->flow_type_rss_offloads == 0)
1289 if (rss_conf->rss_key != NULL &&
1290 rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1291 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1292 rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1295 if (rss_conf->rss_hf != 0 &&
1296 (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1297 dev_err(enic, "Given rss_hf contains none of the supported"
1304 /* Set hash type and key according to rss_conf */
1305 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1307 struct rte_eth_dev *eth_dev;
1313 RTE_ASSERT(rss_conf != NULL);
1314 ret = enic_rss_conf_valid(enic, rss_conf);
1316 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1320 eth_dev = enic->rte_dev;
1322 rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1323 if (enic->rq_count > 1 &&
1324 (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1327 if (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1328 ETH_RSS_NONFRAG_IPV4_OTHER))
1329 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1330 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1331 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1332 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
1333 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4;
1334 if (enic->udp_rss_weak) {
1336 * 'TCP' is not a typo. The "weak" version of
1337 * UDP RSS requires both the TCP and UDP bits
1338 * be set. It does enable TCP RSS as well.
1340 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1343 if (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_IPV6_EX |
1344 ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER))
1345 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1346 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX))
1347 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1348 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_UDP_EX)) {
1349 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6;
1350 if (enic->udp_rss_weak)
1351 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1358 /* Set the hash key if provided */
1359 if (rss_enable && rss_conf->rss_key) {
1360 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1362 dev_err(enic, "Failed to set RSS key\n");
1367 ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1368 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1371 enic->rss_hf = rss_hf;
1372 enic->rss_hash_type = rss_hash_type;
1373 enic->rss_enable = rss_enable;
1375 dev_err(enic, "Failed to update RSS configurations."
1376 " hash=0x%x\n", rss_hash_type);
1381 int enic_set_vlan_strip(struct enic *enic)
1384 * Unfortunately, VLAN strip on/off and RSS on/off are configured
1385 * together. So, re-do niccfg, preserving the current RSS settings.
1387 return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1388 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1392 int enic_add_packet_filter(struct enic *enic)
1394 /* Args -> directed, multicast, broadcast, promisc, allmulti */
1395 return vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1396 enic->promisc, enic->allmulti);
1399 int enic_get_link_status(struct enic *enic)
1401 return vnic_dev_link_status(enic->vdev);
1404 static void enic_dev_deinit(struct enic *enic)
1406 /* stop link status checking */
1407 vnic_dev_notify_unset(enic->vdev);
1409 /* mac_addrs is freed by rte_eth_dev_release_port() */
1411 rte_free(enic->intr);
1417 int enic_set_vnic_res(struct enic *enic)
1419 struct rte_eth_dev *eth_dev = enic->rte_dev;
1421 unsigned int required_rq, required_wq, required_cq, required_intr;
1423 /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1424 required_rq = eth_dev->data->nb_rx_queues * 2;
1425 required_wq = eth_dev->data->nb_tx_queues;
1426 required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1427 required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1428 if (eth_dev->data->dev_conf.intr_conf.rxq) {
1429 required_intr += eth_dev->data->nb_rx_queues;
1432 if (enic->conf_rq_count < required_rq) {
1433 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1434 eth_dev->data->nb_rx_queues,
1435 required_rq, enic->conf_rq_count);
1438 if (enic->conf_wq_count < required_wq) {
1439 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1440 eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1444 if (enic->conf_cq_count < required_cq) {
1445 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1446 required_cq, enic->conf_cq_count);
1449 if (enic->conf_intr_count < required_intr) {
1450 dev_err(dev, "Not enough Interrupts to support Rx queue"
1451 " interrupts. Required:%u, Configured:%u\n",
1452 required_intr, enic->conf_intr_count);
1457 enic->rq_count = eth_dev->data->nb_rx_queues;
1458 enic->wq_count = eth_dev->data->nb_tx_queues;
1459 enic->cq_count = enic->rq_count + enic->wq_count;
1460 enic->intr_count = required_intr;
1466 /* Initialize the completion queue for an RQ */
1468 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1470 struct vnic_rq *sop_rq, *data_rq;
1471 unsigned int cq_idx;
1474 sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1475 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1478 vnic_cq_clean(&enic->cq[cq_idx]);
1479 vnic_cq_init(&enic->cq[cq_idx],
1480 0 /* flow_control_enable */,
1481 1 /* color_enable */,
1484 1 /* cq_tail_color */,
1485 0 /* interrupt_enable */,
1486 1 /* cq_entry_enable */,
1487 0 /* cq_message_enable */,
1488 0 /* interrupt offset */,
1489 0 /* cq_message_addr */);
1492 vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1493 enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1494 sop_rq->ring.desc_count - 1, 1, 0);
1495 if (data_rq->in_use) {
1496 vnic_rq_init_start(data_rq,
1498 enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1499 data_rq->ring.desc_count - 1, 1, 0);
1502 rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1506 if (data_rq->in_use) {
1507 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1509 enic_rxmbuf_queue_release(enic, sop_rq);
1517 /* The Cisco NIC can send and receive packets up to a max packet size
1518 * determined by the NIC type and firmware. There is also an MTU
1519 * configured into the NIC via the CIMC/UCSM management interface
1520 * which can be overridden by this function (up to the max packet size).
1521 * Depending on the network setup, doing so may cause packet drops
1522 * and unexpected behavior.
1524 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1526 unsigned int rq_idx;
1529 uint16_t old_mtu; /* previous setting */
1530 uint16_t config_mtu; /* Value configured into NIC via CIMC/UCSM */
1531 struct rte_eth_dev *eth_dev = enic->rte_dev;
1533 old_mtu = eth_dev->data->mtu;
1534 config_mtu = enic->config.mtu;
1536 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1537 return -E_RTE_SECONDARY;
1539 if (new_mtu > enic->max_mtu) {
1541 "MTU not updated: requested (%u) greater than max (%u)\n",
1542 new_mtu, enic->max_mtu);
1545 if (new_mtu < ENIC_MIN_MTU) {
1547 "MTU not updated: requested (%u) less than min (%u)\n",
1548 new_mtu, ENIC_MIN_MTU);
1551 if (new_mtu > config_mtu)
1553 "MTU (%u) is greater than value configured in NIC (%u)\n",
1554 new_mtu, config_mtu);
1556 /* Update the MTU and maximum packet length */
1557 eth_dev->data->mtu = new_mtu;
1558 eth_dev->data->dev_conf.rxmode.max_rx_pkt_len =
1559 enic_mtu_to_max_rx_pktlen(new_mtu);
1562 * If the device has not started (enic_enable), nothing to do.
1563 * Later, enic_enable() will set up RQs reflecting the new maximum
1566 if (!eth_dev->data->dev_started)
1570 * The device has started, re-do RQs on the fly. In the process, we
1571 * pick up the new maximum packet length.
1573 * Some applications rely on the ability to change MTU without stopping
1574 * the device. So keep this behavior for now.
1576 rte_spinlock_lock(&enic->mtu_lock);
1578 /* Stop traffic on all RQs */
1579 for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1580 rq = &enic->rq[rq_idx];
1581 if (rq->is_sop && rq->in_use) {
1582 rc = enic_stop_rq(enic,
1583 enic_sop_rq_idx_to_rte_idx(rq_idx));
1585 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1591 /* replace Rx function with a no-op to avoid getting stale pkts */
1592 eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1595 /* Allow time for threads to exit the real Rx function. */
1598 /* now it is safe to reconfigure the RQs */
1601 /* free and reallocate RQs with the new MTU */
1602 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1603 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1608 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1609 rq->tot_nb_desc, rq->rx_free_thresh);
1612 "Fatal MTU alloc error- No traffic will pass\n");
1616 rc = enic_reinit_rq(enic, rq_idx);
1619 "Fatal MTU RQ reinit- No traffic will pass\n");
1624 /* put back the real receive function */
1626 enic_pick_rx_handler(eth_dev);
1629 /* restart Rx traffic */
1630 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1631 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1632 if (rq->is_sop && rq->in_use)
1633 enic_start_rq(enic, rq_idx);
1637 dev_info(enic, "MTU changed from %u to %u\n", old_mtu, new_mtu);
1638 rte_spinlock_unlock(&enic->mtu_lock);
1642 static int enic_dev_init(struct enic *enic)
1645 struct rte_eth_dev *eth_dev = enic->rte_dev;
1647 vnic_dev_intr_coal_timer_info_default(enic->vdev);
1649 /* Get vNIC configuration
1651 err = enic_get_vnic_config(enic);
1653 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1657 /* Get available resource counts */
1658 enic_get_res_counts(enic);
1659 if (enic->conf_rq_count == 1) {
1660 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1661 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1662 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1665 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1666 enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1667 enic->conf_cq_count, 8);
1668 enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1669 enic->conf_intr_count, 8);
1670 enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1671 enic->conf_rq_count, 8);
1672 enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1673 enic->conf_wq_count, 8);
1674 if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1675 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1678 if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1679 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1682 if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1683 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1686 if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1687 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1691 /* Get the supported filters */
1692 enic_fdir_info(enic);
1694 eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr",
1695 sizeof(struct rte_ether_addr) *
1696 ENIC_UNICAST_PERFECT_FILTERS, 0);
1697 if (!eth_dev->data->mac_addrs) {
1698 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1701 rte_ether_addr_copy((struct rte_ether_addr *)enic->mac_addr,
1702 eth_dev->data->mac_addrs);
1704 vnic_dev_set_reset_flag(enic->vdev, 0);
1706 LIST_INIT(&enic->flows);
1708 /* set up link status checking */
1709 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1712 * When Geneve with options offload is available, always disable it
1713 * first as it can interfere with user flow rules.
1715 if (enic->geneve_opt_avail &&
1716 vnic_dev_overlay_offload_ctrl(enic->vdev,
1717 OVERLAY_FEATURE_GENEVE,
1718 OVERLAY_OFFLOAD_DISABLE)) {
1719 dev_err(enic, "failed to disable geneve+option\n");
1721 enic->overlay_offload = false;
1722 if (enic->disable_overlay && enic->vxlan) {
1724 * Explicitly disable overlay offload as the setting is
1725 * sticky, and resetting vNIC does not disable it.
1727 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1728 OVERLAY_FEATURE_VXLAN,
1729 OVERLAY_OFFLOAD_DISABLE)) {
1730 dev_err(enic, "failed to disable overlay offload\n");
1732 dev_info(enic, "Overlay offload is disabled\n");
1735 if (!enic->disable_overlay && enic->vxlan &&
1736 /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1737 vnic_dev_overlay_offload_ctrl(enic->vdev,
1738 OVERLAY_FEATURE_VXLAN,
1739 OVERLAY_OFFLOAD_ENABLE) == 0) {
1740 enic->tx_offload_capa |=
1741 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1742 DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
1743 DEV_TX_OFFLOAD_VXLAN_TNL_TSO;
1744 enic->tx_offload_mask |=
1747 PKT_TX_OUTER_IP_CKSUM |
1749 enic->overlay_offload = true;
1750 dev_info(enic, "Overlay offload is enabled\n");
1752 /* Geneve with options offload requires overlay offload */
1753 if (enic->overlay_offload && enic->geneve_opt_avail &&
1754 enic->geneve_opt_request) {
1755 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1756 OVERLAY_FEATURE_GENEVE,
1757 OVERLAY_OFFLOAD_ENABLE)) {
1758 dev_err(enic, "failed to enable geneve+option\n");
1760 enic->geneve_opt_enabled = 1;
1761 dev_info(enic, "Geneve with options is enabled\n");
1765 * Reset the vxlan port if HW vxlan parsing is available. It
1766 * is always enabled regardless of overlay offload
1770 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT;
1772 * Reset the vxlan port to the default, as the NIC firmware
1773 * does not reset it automatically and keeps the old setting.
1775 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1776 OVERLAY_CFG_VXLAN_PORT_UPDATE,
1777 ENIC_DEFAULT_VXLAN_PORT)) {
1778 dev_err(enic, "failed to update vxlan port\n");
1787 int enic_probe(struct enic *enic)
1789 struct rte_pci_device *pdev = enic->pdev;
1792 dev_debug(enic, "Initializing ENIC PMD\n");
1794 /* if this is a secondary process the hardware is already initialized */
1795 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1798 enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1799 enic->bar0.len = pdev->mem_resource[0].len;
1801 /* Register vNIC device */
1802 enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1804 dev_err(enic, "vNIC registration failed, aborting\n");
1808 LIST_INIT(&enic->memzone_list);
1809 rte_spinlock_init(&enic->memzone_list_lock);
1811 vnic_register_cbacks(enic->vdev,
1812 enic_alloc_consistent,
1813 enic_free_consistent);
1816 * Allocate the consistent memory for stats upfront so both primary and
1817 * secondary processes can dump stats.
1819 err = vnic_dev_alloc_stats_mem(enic->vdev);
1821 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1822 goto err_out_unregister;
1824 /* Issue device open to get device in known state */
1825 err = enic_dev_open(enic);
1827 dev_err(enic, "vNIC dev open failed, aborting\n");
1828 goto err_out_unregister;
1831 /* Set ingress vlan rewrite mode before vnic initialization */
1832 dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n",
1833 enic->ig_vlan_rewrite_mode);
1834 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1835 enic->ig_vlan_rewrite_mode);
1838 "Failed to set ingress vlan rewrite mode, aborting.\n");
1839 goto err_out_dev_close;
1842 /* Issue device init to initialize the vnic-to-switch link.
1843 * We'll start with carrier off and wait for link UP
1844 * notification later to turn on carrier. We don't need
1845 * to wait here for the vnic-to-switch link initialization
1846 * to complete; link UP notification is the indication that
1847 * the process is complete.
1850 err = vnic_dev_init(enic->vdev, 0);
1852 dev_err(enic, "vNIC dev init failed, aborting\n");
1853 goto err_out_dev_close;
1856 err = enic_dev_init(enic);
1858 dev_err(enic, "Device initialization failed, aborting\n");
1859 goto err_out_dev_close;
1865 vnic_dev_close(enic->vdev);
1867 vnic_dev_unregister(enic->vdev);
1872 void enic_remove(struct enic *enic)
1874 enic_dev_deinit(enic);
1875 vnic_dev_close(enic->vdev);
1876 vnic_dev_unregister(enic->vdev);