1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
14 #include <rte_bus_pci.h>
15 #include <rte_memzone.h>
16 #include <rte_malloc.h>
18 #include <rte_string_fns.h>
19 #include <rte_ethdev_driver.h>
21 #include "enic_compat.h"
23 #include "wq_enet_desc.h"
24 #include "rq_enet_desc.h"
25 #include "cq_enet_desc.h"
26 #include "vnic_enet.h"
31 #include "vnic_intr.h"
34 static inline int enic_is_sriov_vf(struct enic *enic)
36 return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
39 static int is_zero_addr(uint8_t *addr)
41 return !(addr[0] | addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
44 static int is_mcast_addr(uint8_t *addr)
49 static int is_eth_addr_valid(uint8_t *addr)
51 return !is_mcast_addr(addr) && !is_zero_addr(addr);
55 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
59 if (!rq || !rq->mbuf_ring) {
60 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
64 for (i = 0; i < rq->ring.desc_count; i++) {
65 if (rq->mbuf_ring[i]) {
66 rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
67 rq->mbuf_ring[i] = NULL;
72 static void enic_free_wq_buf(struct rte_mbuf **buf)
74 struct rte_mbuf *mbuf = *buf;
76 rte_pktmbuf_free_seg(mbuf);
80 static void enic_log_q_error(struct enic *enic)
85 for (i = 0; i < enic->wq_count; i++) {
86 error_status = vnic_wq_error_status(&enic->wq[i]);
88 dev_err(enic, "WQ[%d] error_status %d\n", i,
92 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
93 if (!enic->rq[i].in_use)
95 error_status = vnic_rq_error_status(&enic->rq[i]);
97 dev_err(enic, "RQ[%d] error_status %d\n", i,
102 static void enic_clear_soft_stats(struct enic *enic)
104 struct enic_soft_stats *soft_stats = &enic->soft_stats;
105 rte_atomic64_clear(&soft_stats->rx_nombuf);
106 rte_atomic64_clear(&soft_stats->rx_packet_errors);
107 rte_atomic64_clear(&soft_stats->tx_oversized);
110 static void enic_init_soft_stats(struct enic *enic)
112 struct enic_soft_stats *soft_stats = &enic->soft_stats;
113 rte_atomic64_init(&soft_stats->rx_nombuf);
114 rte_atomic64_init(&soft_stats->rx_packet_errors);
115 rte_atomic64_init(&soft_stats->tx_oversized);
116 enic_clear_soft_stats(enic);
119 void enic_dev_stats_clear(struct enic *enic)
121 if (vnic_dev_stats_clear(enic->vdev))
122 dev_err(enic, "Error in clearing stats\n");
123 enic_clear_soft_stats(enic);
126 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
128 struct vnic_stats *stats;
129 struct enic_soft_stats *soft_stats = &enic->soft_stats;
130 int64_t rx_truncated;
131 uint64_t rx_packet_errors;
132 int ret = vnic_dev_stats_dump(enic->vdev, &stats);
135 dev_err(enic, "Error in getting stats\n");
139 /* The number of truncated packets can only be calculated by
140 * subtracting a hardware counter from error packets received by
141 * the driver. Note: this causes transient inaccuracies in the
142 * ipackets count. Also, the length of truncated packets are
143 * counted in ibytes even though truncated packets are dropped
144 * which can make ibytes be slightly higher than it should be.
146 rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
147 rx_truncated = rx_packet_errors - stats->rx.rx_errors;
149 r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
150 r_stats->opackets = stats->tx.tx_frames_ok;
152 r_stats->ibytes = stats->rx.rx_bytes_ok;
153 r_stats->obytes = stats->tx.tx_bytes_ok;
155 r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
156 r_stats->oerrors = stats->tx.tx_errors
157 + rte_atomic64_read(&soft_stats->tx_oversized);
159 r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
161 r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
165 int enic_del_mac_address(struct enic *enic, int mac_index)
167 struct rte_eth_dev *eth_dev = enic->rte_dev;
168 uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
170 return vnic_dev_del_addr(enic->vdev, mac_addr);
173 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
177 if (!is_eth_addr_valid(mac_addr)) {
178 dev_err(enic, "invalid mac address\n");
182 err = vnic_dev_add_addr(enic->vdev, mac_addr);
184 dev_err(enic, "add mac addr failed\n");
189 enic_free_rq_buf(struct rte_mbuf **mbuf)
194 rte_pktmbuf_free(*mbuf);
198 void enic_init_vnic_resources(struct enic *enic)
200 unsigned int error_interrupt_enable = 1;
201 unsigned int error_interrupt_offset = 0;
202 unsigned int rxq_interrupt_enable = 0;
203 unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
204 unsigned int index = 0;
206 struct vnic_rq *data_rq;
208 if (enic->rte_dev->data->dev_conf.intr_conf.rxq)
209 rxq_interrupt_enable = 1;
211 for (index = 0; index < enic->rq_count; index++) {
212 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
214 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
216 error_interrupt_enable,
217 error_interrupt_offset);
219 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
221 vnic_rq_init(data_rq,
223 error_interrupt_enable,
224 error_interrupt_offset);
226 vnic_cq_init(&enic->cq[cq_idx],
227 0 /* flow_control_enable */,
228 1 /* color_enable */,
231 1 /* cq_tail_color */,
232 rxq_interrupt_enable,
233 1 /* cq_entry_enable */,
234 0 /* cq_message_enable */,
235 rxq_interrupt_offset,
236 0 /* cq_message_addr */);
237 if (rxq_interrupt_enable)
238 rxq_interrupt_offset++;
241 for (index = 0; index < enic->wq_count; index++) {
242 vnic_wq_init(&enic->wq[index],
243 enic_cq_wq(enic, index),
244 error_interrupt_enable,
245 error_interrupt_offset);
246 /* Compute unsupported ol flags for enic_prep_pkts() */
247 enic->wq[index].tx_offload_notsup_mask =
248 PKT_TX_OFFLOAD_MASK ^ enic->tx_offload_mask;
250 cq_idx = enic_cq_wq(enic, index);
251 vnic_cq_init(&enic->cq[cq_idx],
252 0 /* flow_control_enable */,
253 1 /* color_enable */,
256 1 /* cq_tail_color */,
257 0 /* interrupt_enable */,
258 0 /* cq_entry_enable */,
259 1 /* cq_message_enable */,
260 0 /* interrupt offset */,
261 (u64)enic->wq[index].cqmsg_rz->iova);
264 for (index = 0; index < enic->intr_count; index++) {
265 vnic_intr_init(&enic->intr[index],
266 enic->config.intr_timer_usec,
267 enic->config.intr_timer_type,
268 /*mask_on_assertion*/1);
274 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
277 struct rq_enet_desc *rqd = rq->ring.descs;
280 uint32_t max_rx_pkt_len;
286 dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
287 rq->ring.desc_count);
290 * If *not* using scatter and the mbuf size is greater than the
291 * requested max packet size (max_rx_pkt_len), then reduce the
292 * posted buffer size to max_rx_pkt_len. HW still receives packets
293 * larger than max_rx_pkt_len, but they will be truncated, which we
294 * drop in the rx handler. Not ideal, but better than returning
295 * large packets when the user is not expecting them.
297 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
298 rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
299 if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
300 rq_buf_len = max_rx_pkt_len;
301 for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
302 mb = rte_mbuf_raw_alloc(rq->mp);
304 dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
305 (unsigned)rq->index);
309 mb->data_off = RTE_PKTMBUF_HEADROOM;
310 dma_addr = (dma_addr_t)(mb->buf_iova
311 + RTE_PKTMBUF_HEADROOM);
312 rq_enet_desc_enc(rqd, dma_addr,
313 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
314 : RQ_ENET_TYPE_NOT_SOP),
316 rq->mbuf_ring[i] = mb;
319 * Do not post the buffers to the NIC until we enable the RQ via
322 rq->need_initial_post = true;
323 /* Initialize fetch index while RQ is disabled */
324 iowrite32(0, &rq->ctrl->fetch_index);
329 * Post the Rx buffers for the first time. enic_alloc_rx_queue_mbufs() has
330 * allocated the buffers and filled the RQ descriptor ring. Just need to push
331 * the post index to the NIC.
334 enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)
336 if (!rq->in_use || !rq->need_initial_post)
339 /* make sure all prior writes are complete before doing the PIO write */
342 /* Post all but the last buffer to VIC. */
343 rq->posted_index = rq->ring.desc_count - 1;
347 dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
348 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
349 iowrite32(rq->posted_index, &rq->ctrl->posted_index);
351 rq->need_initial_post = false;
355 enic_alloc_consistent(void *priv, size_t size,
356 dma_addr_t *dma_handle, u8 *name)
359 const struct rte_memzone *rz;
361 struct enic *enic = (struct enic *)priv;
362 struct enic_memzone_entry *mze;
364 rz = rte_memzone_reserve_aligned((const char *)name, size,
365 SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
367 pr_err("%s : Failed to allocate memory requested for %s\n",
373 *dma_handle = (dma_addr_t)rz->iova;
375 mze = rte_malloc("enic memzone entry",
376 sizeof(struct enic_memzone_entry), 0);
379 pr_err("%s : Failed to allocate memory for memzone list\n",
381 rte_memzone_free(rz);
387 rte_spinlock_lock(&enic->memzone_list_lock);
388 LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
389 rte_spinlock_unlock(&enic->memzone_list_lock);
395 enic_free_consistent(void *priv,
396 __rte_unused size_t size,
398 dma_addr_t dma_handle)
400 struct enic_memzone_entry *mze;
401 struct enic *enic = (struct enic *)priv;
403 rte_spinlock_lock(&enic->memzone_list_lock);
404 LIST_FOREACH(mze, &enic->memzone_list, entries) {
405 if (mze->rz->addr == vaddr &&
406 mze->rz->iova == dma_handle)
410 rte_spinlock_unlock(&enic->memzone_list_lock);
412 "Tried to free memory, but couldn't find it in the memzone list\n");
415 LIST_REMOVE(mze, entries);
416 rte_spinlock_unlock(&enic->memzone_list_lock);
417 rte_memzone_free(mze->rz);
421 int enic_link_update(struct enic *enic)
423 struct rte_eth_dev *eth_dev = enic->rte_dev;
424 struct rte_eth_link link;
426 memset(&link, 0, sizeof(link));
427 link.link_status = enic_get_link_status(enic);
428 link.link_duplex = ETH_LINK_FULL_DUPLEX;
429 link.link_speed = vnic_dev_port_speed(enic->vdev);
431 return rte_eth_linkstatus_set(eth_dev, &link);
435 enic_intr_handler(void *arg)
437 struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
438 struct enic *enic = pmd_priv(dev);
440 vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
442 enic_link_update(enic);
443 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
444 enic_log_q_error(enic);
447 static int enic_rxq_intr_init(struct enic *enic)
449 struct rte_intr_handle *intr_handle;
450 uint32_t rxq_intr_count, i;
453 intr_handle = enic->rte_dev->intr_handle;
454 if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
457 * Rx queue interrupts only work when we have MSI-X interrupts,
458 * one per queue. Sharing one interrupt is technically
459 * possible with VIC, but it is not worth the complications it brings.
461 if (!rte_intr_cap_multiple(intr_handle)) {
462 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
463 " (vfio-pci driver)\n");
466 rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
467 err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
469 dev_err(enic, "Failed to enable event fds for Rx queue"
473 intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
474 rxq_intr_count * sizeof(int), 0);
475 if (intr_handle->intr_vec == NULL) {
476 dev_err(enic, "Failed to allocate intr_vec\n");
479 for (i = 0; i < rxq_intr_count; i++)
480 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
484 static void enic_rxq_intr_deinit(struct enic *enic)
486 struct rte_intr_handle *intr_handle;
488 intr_handle = enic->rte_dev->intr_handle;
489 rte_intr_efd_disable(intr_handle);
490 if (intr_handle->intr_vec != NULL) {
491 rte_free(intr_handle->intr_vec);
492 intr_handle->intr_vec = NULL;
496 static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
498 struct wq_enet_desc *desc;
503 * Fill WQ descriptor fields that never change. Every descriptor is
504 * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
505 * descriptors (i.e. request one completion update every 32 packets).
507 wq = &enic->wq[queue_idx];
508 desc = (struct wq_enet_desc *)wq->ring.descs;
509 for (i = 0; i < wq->ring.desc_count; i++, desc++) {
510 desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
511 if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
512 desc->header_length_flags |=
513 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
518 * The 'strong' version is in enic_rxtx_vec_avx2.c. This weak version is used
519 * used when that file is not compiled.
522 enic_use_vector_rx_handler(__rte_unused struct enic *enic)
527 static void pick_rx_handler(struct enic *enic)
529 struct rte_eth_dev *eth_dev;
533 * 1. The vectorized handler if possible and requested.
534 * 2. The non-scatter, simplified handler if scatter Rx is not used.
535 * 3. The default handler as a fallback.
537 eth_dev = enic->rte_dev;
538 if (enic_use_vector_rx_handler(enic))
540 if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
541 PMD_INIT_LOG(DEBUG, " use the non-scatter Rx handler");
542 eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
544 PMD_INIT_LOG(DEBUG, " use the normal Rx handler");
545 eth_dev->rx_pkt_burst = &enic_recv_pkts;
549 int enic_enable(struct enic *enic)
553 struct rte_eth_dev *eth_dev = enic->rte_dev;
554 uint64_t simple_tx_offloads;
557 if (enic->enable_avx2_rx) {
558 struct rte_mbuf mb_def = { .buf_addr = 0 };
561 * mbuf_initializer contains const-after-init fields of
562 * receive mbufs (i.e. 64 bits of fields from rearm_data).
563 * It is currently used by the vectorized handler.
566 mb_def.data_off = RTE_PKTMBUF_HEADROOM;
567 mb_def.port = enic->port_id;
568 rte_mbuf_refcnt_set(&mb_def, 1);
569 rte_compiler_barrier();
570 p = (uintptr_t)&mb_def.rearm_data;
571 enic->mbuf_initializer = *(uint64_t *)p;
574 eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
575 eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
577 /* vnic notification of link status has already been turned on in
578 * enic_dev_init() which is called during probe time. Here we are
579 * just turning on interrupt vector 0 if needed.
581 if (eth_dev->data->dev_conf.intr_conf.lsc)
582 vnic_dev_notify_set(enic->vdev, 0);
584 err = enic_rxq_intr_init(enic);
587 if (enic_clsf_init(enic))
588 dev_warning(enic, "Init of hash table for clsf failed."\
589 "Flow director feature will not work\n");
591 for (index = 0; index < enic->rq_count; index++) {
592 err = enic_alloc_rx_queue_mbufs(enic,
593 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
595 dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
598 err = enic_alloc_rx_queue_mbufs(enic,
599 &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
601 /* release the allocated mbufs for the sop rq*/
602 enic_rxmbuf_queue_release(enic,
603 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
605 dev_err(enic, "Failed to alloc data RX queue mbufs\n");
611 * Use the simple TX handler if possible. Only checksum offloads
612 * and vlan insertion are supported.
614 simple_tx_offloads = enic->tx_offload_capa &
615 (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
616 DEV_TX_OFFLOAD_VLAN_INSERT |
617 DEV_TX_OFFLOAD_IPV4_CKSUM |
618 DEV_TX_OFFLOAD_UDP_CKSUM |
619 DEV_TX_OFFLOAD_TCP_CKSUM);
620 if ((eth_dev->data->dev_conf.txmode.offloads &
621 ~simple_tx_offloads) == 0) {
622 PMD_INIT_LOG(DEBUG, " use the simple tx handler");
623 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
624 for (index = 0; index < enic->wq_count; index++)
625 enic_prep_wq_for_simple_tx(enic, index);
627 PMD_INIT_LOG(DEBUG, " use the default tx handler");
628 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
631 pick_rx_handler(enic);
633 for (index = 0; index < enic->wq_count; index++)
634 enic_start_wq(enic, index);
635 for (index = 0; index < enic->rq_count; index++)
636 enic_start_rq(enic, index);
638 vnic_dev_add_addr(enic->vdev, enic->mac_addr);
640 vnic_dev_enable_wait(enic->vdev);
642 /* Register and enable error interrupt */
643 rte_intr_callback_register(&(enic->pdev->intr_handle),
644 enic_intr_handler, (void *)enic->rte_dev);
646 rte_intr_enable(&(enic->pdev->intr_handle));
647 /* Unmask LSC interrupt */
648 vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
653 int enic_alloc_intr_resources(struct enic *enic)
658 dev_info(enic, "vNIC resources used: "\
659 "wq %d rq %d cq %d intr %d\n",
660 enic->wq_count, enic_vnic_rq_count(enic),
661 enic->cq_count, enic->intr_count);
663 for (i = 0; i < enic->intr_count; i++) {
664 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
666 enic_free_vnic_resources(enic);
673 void enic_free_rq(void *rxq)
675 struct vnic_rq *rq_sop, *rq_data;
681 rq_sop = (struct vnic_rq *)rxq;
682 enic = vnic_dev_priv(rq_sop->vdev);
683 rq_data = &enic->rq[rq_sop->data_queue_idx];
685 if (rq_sop->free_mbufs) {
686 struct rte_mbuf **mb;
689 mb = rq_sop->free_mbufs;
690 for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs;
691 i < ENIC_RX_BURST_MAX; i++)
692 rte_pktmbuf_free(mb[i]);
693 rte_free(rq_sop->free_mbufs);
694 rq_sop->free_mbufs = NULL;
695 rq_sop->num_free_mbufs = 0;
698 enic_rxmbuf_queue_release(enic, rq_sop);
700 enic_rxmbuf_queue_release(enic, rq_data);
702 rte_free(rq_sop->mbuf_ring);
704 rte_free(rq_data->mbuf_ring);
706 rq_sop->mbuf_ring = NULL;
707 rq_data->mbuf_ring = NULL;
709 vnic_rq_free(rq_sop);
711 vnic_rq_free(rq_data);
713 vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
719 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
721 struct rte_eth_dev *eth_dev = enic->rte_dev;
722 vnic_wq_enable(&enic->wq[queue_idx]);
723 eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
726 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
728 struct rte_eth_dev *eth_dev = enic->rte_dev;
731 ret = vnic_wq_disable(&enic->wq[queue_idx]);
735 eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
739 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
741 struct vnic_rq *rq_sop;
742 struct vnic_rq *rq_data;
743 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
744 rq_data = &enic->rq[rq_sop->data_queue_idx];
745 struct rte_eth_dev *eth_dev = enic->rte_dev;
747 if (rq_data->in_use) {
748 vnic_rq_enable(rq_data);
749 enic_initial_post_rx(enic, rq_data);
752 vnic_rq_enable(rq_sop);
753 enic_initial_post_rx(enic, rq_sop);
754 eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
757 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
759 int ret1 = 0, ret2 = 0;
760 struct rte_eth_dev *eth_dev = enic->rte_dev;
761 struct vnic_rq *rq_sop;
762 struct vnic_rq *rq_data;
763 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
764 rq_data = &enic->rq[rq_sop->data_queue_idx];
766 ret2 = vnic_rq_disable(rq_sop);
769 ret1 = vnic_rq_disable(rq_data);
776 eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
780 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
781 unsigned int socket_id, struct rte_mempool *mp,
782 uint16_t nb_desc, uint16_t free_thresh)
785 uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
786 uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
787 struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
788 struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
789 unsigned int mbuf_size, mbufs_per_pkt;
790 unsigned int nb_sop_desc, nb_data_desc;
791 uint16_t min_sop, max_sop, min_data, max_data;
792 uint32_t max_rx_pkt_len;
795 rq_sop->data_queue_idx = data_queue_idx;
797 rq_data->data_queue_idx = 0;
798 rq_sop->socket_id = socket_id;
800 rq_data->socket_id = socket_id;
803 rq_sop->rx_free_thresh = free_thresh;
804 rq_data->rx_free_thresh = free_thresh;
805 dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
808 mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
809 RTE_PKTMBUF_HEADROOM);
810 /* max_rx_pkt_len includes the ethernet header and CRC. */
811 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
813 if (enic->rte_dev->data->dev_conf.rxmode.offloads &
814 DEV_RX_OFFLOAD_SCATTER) {
815 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
816 /* ceil((max pkt len)/mbuf_size) */
817 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
819 dev_info(enic, "Scatter rx mode disabled\n");
821 if (max_rx_pkt_len > mbuf_size) {
822 dev_warning(enic, "The maximum Rx packet size (%u) is"
823 " larger than the mbuf size (%u), and"
824 " scatter is disabled. Larger packets will"
826 max_rx_pkt_len, mbuf_size);
830 if (mbufs_per_pkt > 1) {
831 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
832 rq_sop->data_queue_enable = 1;
835 * HW does not directly support rxmode.max_rx_pkt_len. HW always
836 * receives packet sizes up to the "max" MTU.
837 * If not using scatter, we can achieve the effect of dropping
838 * larger packets by reducing the size of posted buffers.
839 * See enic_alloc_rx_queue_mbufs().
842 enic_mtu_to_max_rx_pktlen(enic->max_mtu)) {
843 dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
844 " when scatter rx mode is in use.\n");
847 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
849 rq_sop->data_queue_enable = 0;
853 /* number of descriptors have to be a multiple of 32 */
854 nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK;
855 nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK;
857 rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
858 rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
860 if (mbufs_per_pkt > 1) {
861 min_sop = ENIC_RX_BURST_MAX;
862 max_sop = ((enic->config.rq_desc_count /
863 (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK);
864 min_data = min_sop * (mbufs_per_pkt - 1);
865 max_data = enic->config.rq_desc_count;
867 min_sop = ENIC_RX_BURST_MAX;
868 max_sop = enic->config.rq_desc_count;
873 if (nb_desc < (min_sop + min_data)) {
875 "Number of rx descs too low, adjusting to minimum\n");
876 nb_sop_desc = min_sop;
877 nb_data_desc = min_data;
878 } else if (nb_desc > (max_sop + max_data)) {
880 "Number of rx_descs too high, adjusting to maximum\n");
881 nb_sop_desc = max_sop;
882 nb_data_desc = max_data;
884 if (mbufs_per_pkt > 1) {
885 dev_info(enic, "For max packet size %u and mbuf size %u valid"
886 " rx descriptor range is %u to %u\n",
887 max_rx_pkt_len, mbuf_size, min_sop + min_data,
890 dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
891 nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
893 /* Allocate sop queue resources */
894 rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
895 nb_sop_desc, sizeof(struct rq_enet_desc));
897 dev_err(enic, "error in allocation of sop rq\n");
900 nb_sop_desc = rq_sop->ring.desc_count;
902 if (rq_data->in_use) {
903 /* Allocate data queue resources */
904 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
906 sizeof(struct rq_enet_desc));
908 dev_err(enic, "error in allocation of data rq\n");
909 goto err_free_rq_sop;
911 nb_data_desc = rq_data->ring.desc_count;
913 rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
914 socket_id, nb_sop_desc + nb_data_desc,
915 sizeof(struct cq_enet_rq_desc));
917 dev_err(enic, "error in allocation of cq for rq\n");
918 goto err_free_rq_data;
921 /* Allocate the mbuf rings */
922 rq_sop->mbuf_ring = (struct rte_mbuf **)
923 rte_zmalloc_socket("rq->mbuf_ring",
924 sizeof(struct rte_mbuf *) * nb_sop_desc,
925 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
926 if (rq_sop->mbuf_ring == NULL)
929 if (rq_data->in_use) {
930 rq_data->mbuf_ring = (struct rte_mbuf **)
931 rte_zmalloc_socket("rq->mbuf_ring",
932 sizeof(struct rte_mbuf *) * nb_data_desc,
933 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
934 if (rq_data->mbuf_ring == NULL)
935 goto err_free_sop_mbuf;
938 rq_sop->free_mbufs = (struct rte_mbuf **)
939 rte_zmalloc_socket("rq->free_mbufs",
940 sizeof(struct rte_mbuf *) *
942 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
943 if (rq_sop->free_mbufs == NULL)
944 goto err_free_data_mbuf;
945 rq_sop->num_free_mbufs = 0;
947 rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
952 rte_free(rq_data->mbuf_ring);
954 rte_free(rq_sop->mbuf_ring);
956 /* cleanup on error */
957 vnic_cq_free(&enic->cq[queue_idx]);
960 vnic_rq_free(rq_data);
962 vnic_rq_free(rq_sop);
967 void enic_free_wq(void *txq)
975 wq = (struct vnic_wq *)txq;
976 enic = vnic_dev_priv(wq->vdev);
977 rte_memzone_free(wq->cqmsg_rz);
979 vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
982 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
983 unsigned int socket_id, uint16_t nb_desc)
986 struct vnic_wq *wq = &enic->wq[queue_idx];
987 unsigned int cq_index = enic_cq_wq(enic, queue_idx);
991 wq->socket_id = socket_id;
993 * rte_eth_tx_queue_setup() checks min, max, and alignment. So just
994 * print an info message for diagnostics.
996 dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc);
998 /* Allocate queue resources */
999 err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
1001 sizeof(struct wq_enet_desc));
1003 dev_err(enic, "error in allocation of wq\n");
1007 err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
1009 sizeof(struct cq_enet_wq_desc));
1012 dev_err(enic, "error in allocation of cq for wq\n");
1015 /* setup up CQ message */
1016 snprintf((char *)name, sizeof(name),
1017 "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
1020 wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
1021 sizeof(uint32_t), SOCKET_ID_ANY,
1022 RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
1029 int enic_disable(struct enic *enic)
1034 for (i = 0; i < enic->intr_count; i++) {
1035 vnic_intr_mask(&enic->intr[i]);
1036 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
1038 enic_rxq_intr_deinit(enic);
1039 rte_intr_disable(&enic->pdev->intr_handle);
1040 rte_intr_callback_unregister(&enic->pdev->intr_handle,
1042 (void *)enic->rte_dev);
1044 vnic_dev_disable(enic->vdev);
1046 enic_clsf_destroy(enic);
1048 if (!enic_is_sriov_vf(enic))
1049 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
1051 for (i = 0; i < enic->wq_count; i++) {
1052 err = vnic_wq_disable(&enic->wq[i]);
1056 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
1057 if (enic->rq[i].in_use) {
1058 err = vnic_rq_disable(&enic->rq[i]);
1064 /* If we were using interrupts, set the interrupt vector to -1
1065 * to disable interrupts. We are not disabling link notifcations,
1066 * though, as we want the polling of link status to continue working.
1068 if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
1069 vnic_dev_notify_set(enic->vdev, -1);
1071 vnic_dev_set_reset_flag(enic->vdev, 1);
1073 for (i = 0; i < enic->wq_count; i++)
1074 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
1076 for (i = 0; i < enic_vnic_rq_count(enic); i++)
1077 if (enic->rq[i].in_use)
1078 vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
1079 for (i = 0; i < enic->cq_count; i++)
1080 vnic_cq_clean(&enic->cq[i]);
1081 for (i = 0; i < enic->intr_count; i++)
1082 vnic_intr_clean(&enic->intr[i]);
1087 static int enic_dev_wait(struct vnic_dev *vdev,
1088 int (*start)(struct vnic_dev *, int),
1089 int (*finished)(struct vnic_dev *, int *),
1096 err = start(vdev, arg);
1100 /* Wait for func to complete...2 seconds max */
1101 for (i = 0; i < 2000; i++) {
1102 err = finished(vdev, &done);
1112 static int enic_dev_open(struct enic *enic)
1115 int flags = CMD_OPENF_IG_DESCCACHE;
1117 err = enic_dev_wait(enic->vdev, vnic_dev_open,
1118 vnic_dev_open_done, flags);
1120 dev_err(enic_get_dev(enic),
1121 "vNIC device open failed, err %d\n", err);
1126 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
1128 dma_addr_t rss_key_buf_pa;
1129 union vnic_rss_key *rss_key_buf_va = NULL;
1133 RTE_ASSERT(user_key != NULL);
1134 snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
1135 rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1136 &rss_key_buf_pa, name);
1137 if (!rss_key_buf_va)
1140 for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1141 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1143 err = enic_set_rss_key(enic,
1145 sizeof(union vnic_rss_key));
1147 /* Save for later queries */
1149 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1150 sizeof(union vnic_rss_key));
1152 enic_free_consistent(enic, sizeof(union vnic_rss_key),
1153 rss_key_buf_va, rss_key_buf_pa);
1158 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1160 dma_addr_t rss_cpu_buf_pa;
1161 union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1165 snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
1166 rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1167 &rss_cpu_buf_pa, name);
1168 if (!rss_cpu_buf_va)
1171 rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1173 err = enic_set_rss_cpu(enic,
1175 sizeof(union vnic_rss_cpu));
1177 enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1178 rss_cpu_buf_va, rss_cpu_buf_pa);
1180 /* Save for later queries */
1182 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1186 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
1187 u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
1189 const u8 tso_ipid_split_en = 0;
1192 err = enic_set_nic_cfg(enic,
1193 rss_default_cpu, rss_hash_type,
1194 rss_hash_bits, rss_base_cpu,
1195 rss_enable, tso_ipid_split_en,
1196 enic->ig_vlan_strip_en);
1201 /* Initialize RSS with defaults, called from dev_configure */
1202 int enic_init_rss_nic_cfg(struct enic *enic)
1204 static uint8_t default_rss_key[] = {
1205 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1206 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1207 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1208 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1210 struct rte_eth_rss_conf rss_conf;
1211 union vnic_rss_cpu rss_cpu;
1214 rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1216 * If setting key for the first time, and the user gives us none, then
1217 * push the default key to NIC.
1219 if (rss_conf.rss_key == NULL) {
1220 rss_conf.rss_key = default_rss_key;
1221 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1223 ret = enic_set_rss_conf(enic, &rss_conf);
1225 dev_err(enic, "Failed to configure RSS\n");
1228 if (enic->rss_enable) {
1229 /* If enabling RSS, use the default reta */
1230 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1231 rss_cpu.cpu[i / 4].b[i % 4] =
1232 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1234 ret = enic_set_rss_reta(enic, &rss_cpu);
1236 dev_err(enic, "Failed to set RSS indirection table\n");
1241 int enic_setup_finish(struct enic *enic)
1243 enic_init_soft_stats(enic);
1246 vnic_dev_packet_filter(enic->vdev,
1259 static int enic_rss_conf_valid(struct enic *enic,
1260 struct rte_eth_rss_conf *rss_conf)
1262 /* RSS is disabled per VIC settings. Ignore rss_conf. */
1263 if (enic->flow_type_rss_offloads == 0)
1265 if (rss_conf->rss_key != NULL &&
1266 rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1267 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1268 rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1271 if (rss_conf->rss_hf != 0 &&
1272 (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1273 dev_err(enic, "Given rss_hf contains none of the supported"
1280 /* Set hash type and key according to rss_conf */
1281 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1283 struct rte_eth_dev *eth_dev;
1289 RTE_ASSERT(rss_conf != NULL);
1290 ret = enic_rss_conf_valid(enic, rss_conf);
1292 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1296 eth_dev = enic->rte_dev;
1298 rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1299 if (enic->rq_count > 1 &&
1300 (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1303 if (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1304 ETH_RSS_NONFRAG_IPV4_OTHER))
1305 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1306 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1307 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1308 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
1309 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4;
1310 if (enic->udp_rss_weak) {
1312 * 'TCP' is not a typo. The "weak" version of
1313 * UDP RSS requires both the TCP and UDP bits
1314 * be set. It does enable TCP RSS as well.
1316 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1319 if (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_IPV6_EX |
1320 ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER))
1321 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1322 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX))
1323 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1324 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_UDP_EX)) {
1325 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6;
1326 if (enic->udp_rss_weak)
1327 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1334 /* Set the hash key if provided */
1335 if (rss_enable && rss_conf->rss_key) {
1336 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1338 dev_err(enic, "Failed to set RSS key\n");
1343 ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1344 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1347 enic->rss_hf = rss_hf;
1348 enic->rss_hash_type = rss_hash_type;
1349 enic->rss_enable = rss_enable;
1351 dev_err(enic, "Failed to update RSS configurations."
1352 " hash=0x%x\n", rss_hash_type);
1357 int enic_set_vlan_strip(struct enic *enic)
1360 * Unfortunately, VLAN strip on/off and RSS on/off are configured
1361 * together. So, re-do niccfg, preserving the current RSS settings.
1363 return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1364 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1368 void enic_add_packet_filter(struct enic *enic)
1370 /* Args -> directed, multicast, broadcast, promisc, allmulti */
1371 vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1372 enic->promisc, enic->allmulti);
1375 int enic_get_link_status(struct enic *enic)
1377 return vnic_dev_link_status(enic->vdev);
1380 static void enic_dev_deinit(struct enic *enic)
1382 /* stop link status checking */
1383 vnic_dev_notify_unset(enic->vdev);
1385 /* mac_addrs is freed by rte_eth_dev_release_port() */
1387 rte_free(enic->intr);
1393 int enic_set_vnic_res(struct enic *enic)
1395 struct rte_eth_dev *eth_dev = enic->rte_dev;
1397 unsigned int required_rq, required_wq, required_cq, required_intr;
1399 /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1400 required_rq = eth_dev->data->nb_rx_queues * 2;
1401 required_wq = eth_dev->data->nb_tx_queues;
1402 required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1403 required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1404 if (eth_dev->data->dev_conf.intr_conf.rxq) {
1405 required_intr += eth_dev->data->nb_rx_queues;
1408 if (enic->conf_rq_count < required_rq) {
1409 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1410 eth_dev->data->nb_rx_queues,
1411 required_rq, enic->conf_rq_count);
1414 if (enic->conf_wq_count < required_wq) {
1415 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1416 eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1420 if (enic->conf_cq_count < required_cq) {
1421 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1422 required_cq, enic->conf_cq_count);
1425 if (enic->conf_intr_count < required_intr) {
1426 dev_err(dev, "Not enough Interrupts to support Rx queue"
1427 " interrupts. Required:%u, Configured:%u\n",
1428 required_intr, enic->conf_intr_count);
1433 enic->rq_count = eth_dev->data->nb_rx_queues;
1434 enic->wq_count = eth_dev->data->nb_tx_queues;
1435 enic->cq_count = enic->rq_count + enic->wq_count;
1436 enic->intr_count = required_intr;
1442 /* Initialize the completion queue for an RQ */
1444 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1446 struct vnic_rq *sop_rq, *data_rq;
1447 unsigned int cq_idx;
1450 sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1451 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1454 vnic_cq_clean(&enic->cq[cq_idx]);
1455 vnic_cq_init(&enic->cq[cq_idx],
1456 0 /* flow_control_enable */,
1457 1 /* color_enable */,
1460 1 /* cq_tail_color */,
1461 0 /* interrupt_enable */,
1462 1 /* cq_entry_enable */,
1463 0 /* cq_message_enable */,
1464 0 /* interrupt offset */,
1465 0 /* cq_message_addr */);
1468 vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1469 enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1470 sop_rq->ring.desc_count - 1, 1, 0);
1471 if (data_rq->in_use) {
1472 vnic_rq_init_start(data_rq,
1474 enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1475 data_rq->ring.desc_count - 1, 1, 0);
1478 rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1482 if (data_rq->in_use) {
1483 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1485 enic_rxmbuf_queue_release(enic, sop_rq);
1493 /* The Cisco NIC can send and receive packets up to a max packet size
1494 * determined by the NIC type and firmware. There is also an MTU
1495 * configured into the NIC via the CIMC/UCSM management interface
1496 * which can be overridden by this function (up to the max packet size).
1497 * Depending on the network setup, doing so may cause packet drops
1498 * and unexpected behavior.
1500 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1502 unsigned int rq_idx;
1505 uint16_t old_mtu; /* previous setting */
1506 uint16_t config_mtu; /* Value configured into NIC via CIMC/UCSM */
1507 struct rte_eth_dev *eth_dev = enic->rte_dev;
1509 old_mtu = eth_dev->data->mtu;
1510 config_mtu = enic->config.mtu;
1512 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1513 return -E_RTE_SECONDARY;
1515 if (new_mtu > enic->max_mtu) {
1517 "MTU not updated: requested (%u) greater than max (%u)\n",
1518 new_mtu, enic->max_mtu);
1521 if (new_mtu < ENIC_MIN_MTU) {
1523 "MTU not updated: requested (%u) less than min (%u)\n",
1524 new_mtu, ENIC_MIN_MTU);
1527 if (new_mtu > config_mtu)
1529 "MTU (%u) is greater than value configured in NIC (%u)\n",
1530 new_mtu, config_mtu);
1532 /* Update the MTU and maximum packet length */
1533 eth_dev->data->mtu = new_mtu;
1534 eth_dev->data->dev_conf.rxmode.max_rx_pkt_len =
1535 enic_mtu_to_max_rx_pktlen(new_mtu);
1538 * If the device has not started (enic_enable), nothing to do.
1539 * Later, enic_enable() will set up RQs reflecting the new maximum
1542 if (!eth_dev->data->dev_started)
1546 * The device has started, re-do RQs on the fly. In the process, we
1547 * pick up the new maximum packet length.
1549 * Some applications rely on the ability to change MTU without stopping
1550 * the device. So keep this behavior for now.
1552 rte_spinlock_lock(&enic->mtu_lock);
1554 /* Stop traffic on all RQs */
1555 for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1556 rq = &enic->rq[rq_idx];
1557 if (rq->is_sop && rq->in_use) {
1558 rc = enic_stop_rq(enic,
1559 enic_sop_rq_idx_to_rte_idx(rq_idx));
1561 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1567 /* replace Rx function with a no-op to avoid getting stale pkts */
1568 eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1571 /* Allow time for threads to exit the real Rx function. */
1574 /* now it is safe to reconfigure the RQs */
1577 /* free and reallocate RQs with the new MTU */
1578 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1579 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1584 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1585 rq->tot_nb_desc, rq->rx_free_thresh);
1588 "Fatal MTU alloc error- No traffic will pass\n");
1592 rc = enic_reinit_rq(enic, rq_idx);
1595 "Fatal MTU RQ reinit- No traffic will pass\n");
1600 /* put back the real receive function */
1602 pick_rx_handler(enic);
1605 /* restart Rx traffic */
1606 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1607 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1608 if (rq->is_sop && rq->in_use)
1609 enic_start_rq(enic, rq_idx);
1613 dev_info(enic, "MTU changed from %u to %u\n", old_mtu, new_mtu);
1614 rte_spinlock_unlock(&enic->mtu_lock);
1618 static int enic_dev_init(struct enic *enic)
1621 struct rte_eth_dev *eth_dev = enic->rte_dev;
1623 vnic_dev_intr_coal_timer_info_default(enic->vdev);
1625 /* Get vNIC configuration
1627 err = enic_get_vnic_config(enic);
1629 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1633 /* Get available resource counts */
1634 enic_get_res_counts(enic);
1635 if (enic->conf_rq_count == 1) {
1636 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1637 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1638 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1641 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1642 enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1643 enic->conf_cq_count, 8);
1644 enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1645 enic->conf_intr_count, 8);
1646 enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1647 enic->conf_rq_count, 8);
1648 enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1649 enic->conf_wq_count, 8);
1650 if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1651 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1654 if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1655 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1658 if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1659 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1662 if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1663 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1667 /* Get the supported filters */
1668 enic_fdir_info(enic);
1670 eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr",
1671 sizeof(struct ether_addr) *
1672 ENIC_UNICAST_PERFECT_FILTERS, 0);
1673 if (!eth_dev->data->mac_addrs) {
1674 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1677 ether_addr_copy((struct ether_addr *) enic->mac_addr,
1678 eth_dev->data->mac_addrs);
1680 vnic_dev_set_reset_flag(enic->vdev, 0);
1682 LIST_INIT(&enic->flows);
1683 rte_spinlock_init(&enic->flows_lock);
1684 enic->max_flow_counter = -1;
1686 /* set up link status checking */
1687 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1689 enic->overlay_offload = false;
1690 if (enic->disable_overlay && enic->vxlan) {
1692 * Explicitly disable overlay offload as the setting is
1693 * sticky, and resetting vNIC does not disable it.
1695 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1696 OVERLAY_FEATURE_VXLAN,
1697 OVERLAY_OFFLOAD_DISABLE)) {
1698 dev_err(enic, "failed to disable overlay offload\n");
1700 dev_info(enic, "Overlay offload is disabled\n");
1703 if (!enic->disable_overlay && enic->vxlan &&
1704 /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1705 vnic_dev_overlay_offload_ctrl(enic->vdev,
1706 OVERLAY_FEATURE_VXLAN,
1707 OVERLAY_OFFLOAD_ENABLE) == 0) {
1708 enic->tx_offload_capa |=
1709 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1710 DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
1711 DEV_TX_OFFLOAD_VXLAN_TNL_TSO;
1712 enic->tx_offload_mask |=
1715 PKT_TX_OUTER_IP_CKSUM |
1717 enic->overlay_offload = true;
1718 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT;
1719 dev_info(enic, "Overlay offload is enabled\n");
1721 * Reset the vxlan port to the default, as the NIC firmware
1722 * does not reset it automatically and keeps the old setting.
1724 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1725 OVERLAY_CFG_VXLAN_PORT_UPDATE,
1726 ENIC_DEFAULT_VXLAN_PORT)) {
1727 dev_err(enic, "failed to update vxlan port\n");
1736 int enic_probe(struct enic *enic)
1738 struct rte_pci_device *pdev = enic->pdev;
1741 dev_debug(enic, " Initializing ENIC PMD\n");
1743 /* if this is a secondary process the hardware is already initialized */
1744 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1747 enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1748 enic->bar0.len = pdev->mem_resource[0].len;
1750 /* Register vNIC device */
1751 enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1753 dev_err(enic, "vNIC registration failed, aborting\n");
1757 LIST_INIT(&enic->memzone_list);
1758 rte_spinlock_init(&enic->memzone_list_lock);
1760 vnic_register_cbacks(enic->vdev,
1761 enic_alloc_consistent,
1762 enic_free_consistent);
1765 * Allocate the consistent memory for stats and counters upfront so
1766 * both primary and secondary processes can access them.
1768 err = vnic_dev_alloc_stats_mem(enic->vdev);
1770 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1771 goto err_out_unregister;
1773 err = vnic_dev_alloc_counter_mem(enic->vdev);
1775 dev_err(enic, "Failed to allocate counter memory, aborting\n");
1776 goto err_out_unregister;
1779 /* Issue device open to get device in known state */
1780 err = enic_dev_open(enic);
1782 dev_err(enic, "vNIC dev open failed, aborting\n");
1783 goto err_out_unregister;
1786 /* Set ingress vlan rewrite mode before vnic initialization */
1787 dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n",
1788 enic->ig_vlan_rewrite_mode);
1789 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1790 enic->ig_vlan_rewrite_mode);
1793 "Failed to set ingress vlan rewrite mode, aborting.\n");
1794 goto err_out_dev_close;
1797 /* Issue device init to initialize the vnic-to-switch link.
1798 * We'll start with carrier off and wait for link UP
1799 * notification later to turn on carrier. We don't need
1800 * to wait here for the vnic-to-switch link initialization
1801 * to complete; link UP notification is the indication that
1802 * the process is complete.
1805 err = vnic_dev_init(enic->vdev, 0);
1807 dev_err(enic, "vNIC dev init failed, aborting\n");
1808 goto err_out_dev_close;
1811 err = enic_dev_init(enic);
1813 dev_err(enic, "Device initialization failed, aborting\n");
1814 goto err_out_dev_close;
1820 vnic_dev_close(enic->vdev);
1822 vnic_dev_unregister(enic->vdev);
1827 void enic_remove(struct enic *enic)
1829 enic_dev_deinit(enic);
1830 vnic_dev_close(enic->vdev);
1831 vnic_dev_unregister(enic->vdev);