1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
13 #include <rte_bus_pci.h>
14 #include <rte_memzone.h>
15 #include <rte_malloc.h>
17 #include <rte_string_fns.h>
18 #include <rte_ethdev_driver.h>
20 #include "enic_compat.h"
22 #include "wq_enet_desc.h"
23 #include "rq_enet_desc.h"
24 #include "cq_enet_desc.h"
25 #include "vnic_enet.h"
30 #include "vnic_intr.h"
33 static inline int enic_is_sriov_vf(struct enic *enic)
35 return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
38 static int is_zero_addr(uint8_t *addr)
40 return !(addr[0] | addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
43 static int is_mcast_addr(uint8_t *addr)
48 static int is_eth_addr_valid(uint8_t *addr)
50 return !is_mcast_addr(addr) && !is_zero_addr(addr);
54 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
58 if (!rq || !rq->mbuf_ring) {
59 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
63 for (i = 0; i < rq->ring.desc_count; i++) {
64 if (rq->mbuf_ring[i]) {
65 rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
66 rq->mbuf_ring[i] = NULL;
71 static void enic_free_wq_buf(struct rte_mbuf **buf)
73 struct rte_mbuf *mbuf = *buf;
75 rte_pktmbuf_free_seg(mbuf);
79 static void enic_log_q_error(struct enic *enic)
84 for (i = 0; i < enic->wq_count; i++) {
85 error_status = vnic_wq_error_status(&enic->wq[i]);
87 dev_err(enic, "WQ[%d] error_status %d\n", i,
91 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
92 if (!enic->rq[i].in_use)
94 error_status = vnic_rq_error_status(&enic->rq[i]);
96 dev_err(enic, "RQ[%d] error_status %d\n", i,
101 static void enic_clear_soft_stats(struct enic *enic)
103 struct enic_soft_stats *soft_stats = &enic->soft_stats;
104 rte_atomic64_clear(&soft_stats->rx_nombuf);
105 rte_atomic64_clear(&soft_stats->rx_packet_errors);
106 rte_atomic64_clear(&soft_stats->tx_oversized);
109 static void enic_init_soft_stats(struct enic *enic)
111 struct enic_soft_stats *soft_stats = &enic->soft_stats;
112 rte_atomic64_init(&soft_stats->rx_nombuf);
113 rte_atomic64_init(&soft_stats->rx_packet_errors);
114 rte_atomic64_init(&soft_stats->tx_oversized);
115 enic_clear_soft_stats(enic);
118 int enic_dev_stats_clear(struct enic *enic)
122 ret = vnic_dev_stats_clear(enic->vdev);
124 dev_err(enic, "Error in clearing stats\n");
127 enic_clear_soft_stats(enic);
132 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
134 struct vnic_stats *stats;
135 struct enic_soft_stats *soft_stats = &enic->soft_stats;
136 int64_t rx_truncated;
137 uint64_t rx_packet_errors;
138 int ret = vnic_dev_stats_dump(enic->vdev, &stats);
141 dev_err(enic, "Error in getting stats\n");
145 /* The number of truncated packets can only be calculated by
146 * subtracting a hardware counter from error packets received by
147 * the driver. Note: this causes transient inaccuracies in the
148 * ipackets count. Also, the length of truncated packets are
149 * counted in ibytes even though truncated packets are dropped
150 * which can make ibytes be slightly higher than it should be.
152 rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
153 rx_truncated = rx_packet_errors - stats->rx.rx_errors;
155 r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
156 r_stats->opackets = stats->tx.tx_frames_ok;
158 r_stats->ibytes = stats->rx.rx_bytes_ok;
159 r_stats->obytes = stats->tx.tx_bytes_ok;
161 r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
162 r_stats->oerrors = stats->tx.tx_errors
163 + rte_atomic64_read(&soft_stats->tx_oversized);
165 r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
167 r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
171 int enic_del_mac_address(struct enic *enic, int mac_index)
173 struct rte_eth_dev *eth_dev = enic->rte_dev;
174 uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
176 return vnic_dev_del_addr(enic->vdev, mac_addr);
179 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
183 if (!is_eth_addr_valid(mac_addr)) {
184 dev_err(enic, "invalid mac address\n");
188 err = vnic_dev_add_addr(enic->vdev, mac_addr);
190 dev_err(enic, "add mac addr failed\n");
195 enic_free_rq_buf(struct rte_mbuf **mbuf)
200 rte_pktmbuf_free(*mbuf);
204 void enic_init_vnic_resources(struct enic *enic)
206 unsigned int error_interrupt_enable = 1;
207 unsigned int error_interrupt_offset = 0;
208 unsigned int rxq_interrupt_enable = 0;
209 unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
210 unsigned int index = 0;
212 struct vnic_rq *data_rq;
214 if (enic->rte_dev->data->dev_conf.intr_conf.rxq)
215 rxq_interrupt_enable = 1;
217 for (index = 0; index < enic->rq_count; index++) {
218 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
220 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
222 error_interrupt_enable,
223 error_interrupt_offset);
225 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
227 vnic_rq_init(data_rq,
229 error_interrupt_enable,
230 error_interrupt_offset);
232 vnic_cq_init(&enic->cq[cq_idx],
233 0 /* flow_control_enable */,
234 1 /* color_enable */,
237 1 /* cq_tail_color */,
238 rxq_interrupt_enable,
239 1 /* cq_entry_enable */,
240 0 /* cq_message_enable */,
241 rxq_interrupt_offset,
242 0 /* cq_message_addr */);
243 if (rxq_interrupt_enable)
244 rxq_interrupt_offset++;
247 for (index = 0; index < enic->wq_count; index++) {
248 vnic_wq_init(&enic->wq[index],
249 enic_cq_wq(enic, index),
250 error_interrupt_enable,
251 error_interrupt_offset);
252 /* Compute unsupported ol flags for enic_prep_pkts() */
253 enic->wq[index].tx_offload_notsup_mask =
254 PKT_TX_OFFLOAD_MASK ^ enic->tx_offload_mask;
256 cq_idx = enic_cq_wq(enic, index);
257 vnic_cq_init(&enic->cq[cq_idx],
258 0 /* flow_control_enable */,
259 1 /* color_enable */,
262 1 /* cq_tail_color */,
263 0 /* interrupt_enable */,
264 0 /* cq_entry_enable */,
265 1 /* cq_message_enable */,
266 0 /* interrupt offset */,
267 (u64)enic->wq[index].cqmsg_rz->iova);
270 for (index = 0; index < enic->intr_count; index++) {
271 vnic_intr_init(&enic->intr[index],
272 enic->config.intr_timer_usec,
273 enic->config.intr_timer_type,
274 /*mask_on_assertion*/1);
280 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
283 struct rq_enet_desc *rqd = rq->ring.descs;
286 uint32_t max_rx_pkt_len;
292 dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
293 rq->ring.desc_count);
296 * If *not* using scatter and the mbuf size is greater than the
297 * requested max packet size (max_rx_pkt_len), then reduce the
298 * posted buffer size to max_rx_pkt_len. HW still receives packets
299 * larger than max_rx_pkt_len, but they will be truncated, which we
300 * drop in the rx handler. Not ideal, but better than returning
301 * large packets when the user is not expecting them.
303 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
304 rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
305 if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
306 rq_buf_len = max_rx_pkt_len;
307 for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
308 mb = rte_mbuf_raw_alloc(rq->mp);
310 dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
311 (unsigned)rq->index);
315 mb->data_off = RTE_PKTMBUF_HEADROOM;
316 dma_addr = (dma_addr_t)(mb->buf_iova
317 + RTE_PKTMBUF_HEADROOM);
318 rq_enet_desc_enc(rqd, dma_addr,
319 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
320 : RQ_ENET_TYPE_NOT_SOP),
322 rq->mbuf_ring[i] = mb;
325 * Do not post the buffers to the NIC until we enable the RQ via
328 rq->need_initial_post = true;
329 /* Initialize fetch index while RQ is disabled */
330 iowrite32(0, &rq->ctrl->fetch_index);
335 * Post the Rx buffers for the first time. enic_alloc_rx_queue_mbufs() has
336 * allocated the buffers and filled the RQ descriptor ring. Just need to push
337 * the post index to the NIC.
340 enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)
342 if (!rq->in_use || !rq->need_initial_post)
345 /* make sure all prior writes are complete before doing the PIO write */
348 /* Post all but the last buffer to VIC. */
349 rq->posted_index = rq->ring.desc_count - 1;
353 dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
354 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
355 iowrite32(rq->posted_index, &rq->ctrl->posted_index);
357 rq->need_initial_post = false;
361 enic_alloc_consistent(void *priv, size_t size,
362 dma_addr_t *dma_handle, u8 *name)
365 const struct rte_memzone *rz;
367 struct enic *enic = (struct enic *)priv;
368 struct enic_memzone_entry *mze;
370 rz = rte_memzone_reserve_aligned((const char *)name, size,
371 SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
373 pr_err("%s : Failed to allocate memory requested for %s\n",
379 *dma_handle = (dma_addr_t)rz->iova;
381 mze = rte_malloc("enic memzone entry",
382 sizeof(struct enic_memzone_entry), 0);
385 pr_err("%s : Failed to allocate memory for memzone list\n",
387 rte_memzone_free(rz);
393 rte_spinlock_lock(&enic->memzone_list_lock);
394 LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
395 rte_spinlock_unlock(&enic->memzone_list_lock);
401 enic_free_consistent(void *priv,
402 __rte_unused size_t size,
404 dma_addr_t dma_handle)
406 struct enic_memzone_entry *mze;
407 struct enic *enic = (struct enic *)priv;
409 rte_spinlock_lock(&enic->memzone_list_lock);
410 LIST_FOREACH(mze, &enic->memzone_list, entries) {
411 if (mze->rz->addr == vaddr &&
412 mze->rz->iova == dma_handle)
416 rte_spinlock_unlock(&enic->memzone_list_lock);
418 "Tried to free memory, but couldn't find it in the memzone list\n");
421 LIST_REMOVE(mze, entries);
422 rte_spinlock_unlock(&enic->memzone_list_lock);
423 rte_memzone_free(mze->rz);
427 int enic_link_update(struct enic *enic)
429 struct rte_eth_dev *eth_dev = enic->rte_dev;
430 struct rte_eth_link link;
432 memset(&link, 0, sizeof(link));
433 link.link_status = enic_get_link_status(enic);
434 link.link_duplex = ETH_LINK_FULL_DUPLEX;
435 link.link_speed = vnic_dev_port_speed(enic->vdev);
437 return rte_eth_linkstatus_set(eth_dev, &link);
441 enic_intr_handler(void *arg)
443 struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
444 struct enic *enic = pmd_priv(dev);
446 vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
448 enic_link_update(enic);
449 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
450 enic_log_q_error(enic);
453 static int enic_rxq_intr_init(struct enic *enic)
455 struct rte_intr_handle *intr_handle;
456 uint32_t rxq_intr_count, i;
459 intr_handle = enic->rte_dev->intr_handle;
460 if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
463 * Rx queue interrupts only work when we have MSI-X interrupts,
464 * one per queue. Sharing one interrupt is technically
465 * possible with VIC, but it is not worth the complications it brings.
467 if (!rte_intr_cap_multiple(intr_handle)) {
468 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
469 " (vfio-pci driver)\n");
472 rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
473 err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
475 dev_err(enic, "Failed to enable event fds for Rx queue"
479 intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
480 rxq_intr_count * sizeof(int), 0);
481 if (intr_handle->intr_vec == NULL) {
482 dev_err(enic, "Failed to allocate intr_vec\n");
485 for (i = 0; i < rxq_intr_count; i++)
486 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
490 static void enic_rxq_intr_deinit(struct enic *enic)
492 struct rte_intr_handle *intr_handle;
494 intr_handle = enic->rte_dev->intr_handle;
495 rte_intr_efd_disable(intr_handle);
496 if (intr_handle->intr_vec != NULL) {
497 rte_free(intr_handle->intr_vec);
498 intr_handle->intr_vec = NULL;
502 static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
504 struct wq_enet_desc *desc;
509 * Fill WQ descriptor fields that never change. Every descriptor is
510 * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
511 * descriptors (i.e. request one completion update every 32 packets).
513 wq = &enic->wq[queue_idx];
514 desc = (struct wq_enet_desc *)wq->ring.descs;
515 for (i = 0; i < wq->ring.desc_count; i++, desc++) {
516 desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
517 if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
518 desc->header_length_flags |=
519 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
524 * The 'strong' version is in enic_rxtx_vec_avx2.c. This weak version is used
525 * used when that file is not compiled.
528 enic_use_vector_rx_handler(__rte_unused struct enic *enic)
533 static void pick_rx_handler(struct enic *enic)
535 struct rte_eth_dev *eth_dev;
539 * 1. The vectorized handler if possible and requested.
540 * 2. The non-scatter, simplified handler if scatter Rx is not used.
541 * 3. The default handler as a fallback.
543 eth_dev = enic->rte_dev;
544 if (enic_use_vector_rx_handler(enic))
546 if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
547 ENICPMD_LOG(DEBUG, " use the non-scatter Rx handler");
548 eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
550 ENICPMD_LOG(DEBUG, " use the normal Rx handler");
551 eth_dev->rx_pkt_burst = &enic_recv_pkts;
555 int enic_enable(struct enic *enic)
559 struct rte_eth_dev *eth_dev = enic->rte_dev;
560 uint64_t simple_tx_offloads;
563 if (enic->enable_avx2_rx) {
564 struct rte_mbuf mb_def = { .buf_addr = 0 };
567 * mbuf_initializer contains const-after-init fields of
568 * receive mbufs (i.e. 64 bits of fields from rearm_data).
569 * It is currently used by the vectorized handler.
572 mb_def.data_off = RTE_PKTMBUF_HEADROOM;
573 mb_def.port = enic->port_id;
574 rte_mbuf_refcnt_set(&mb_def, 1);
575 rte_compiler_barrier();
576 p = (uintptr_t)&mb_def.rearm_data;
577 enic->mbuf_initializer = *(uint64_t *)p;
580 eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
581 eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
583 /* vnic notification of link status has already been turned on in
584 * enic_dev_init() which is called during probe time. Here we are
585 * just turning on interrupt vector 0 if needed.
587 if (eth_dev->data->dev_conf.intr_conf.lsc)
588 vnic_dev_notify_set(enic->vdev, 0);
590 err = enic_rxq_intr_init(enic);
593 if (enic_clsf_init(enic))
594 dev_warning(enic, "Init of hash table for clsf failed."\
595 "Flow director feature will not work\n");
597 for (index = 0; index < enic->rq_count; index++) {
598 err = enic_alloc_rx_queue_mbufs(enic,
599 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
601 dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
604 err = enic_alloc_rx_queue_mbufs(enic,
605 &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
607 /* release the allocated mbufs for the sop rq*/
608 enic_rxmbuf_queue_release(enic,
609 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
611 dev_err(enic, "Failed to alloc data RX queue mbufs\n");
617 * Use the simple TX handler if possible. Only checksum offloads
618 * and vlan insertion are supported.
620 simple_tx_offloads = enic->tx_offload_capa &
621 (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
622 DEV_TX_OFFLOAD_VLAN_INSERT |
623 DEV_TX_OFFLOAD_IPV4_CKSUM |
624 DEV_TX_OFFLOAD_UDP_CKSUM |
625 DEV_TX_OFFLOAD_TCP_CKSUM);
626 if ((eth_dev->data->dev_conf.txmode.offloads &
627 ~simple_tx_offloads) == 0) {
628 ENICPMD_LOG(DEBUG, " use the simple tx handler");
629 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
630 for (index = 0; index < enic->wq_count; index++)
631 enic_prep_wq_for_simple_tx(enic, index);
633 ENICPMD_LOG(DEBUG, " use the default tx handler");
634 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
637 pick_rx_handler(enic);
639 for (index = 0; index < enic->wq_count; index++)
640 enic_start_wq(enic, index);
641 for (index = 0; index < enic->rq_count; index++)
642 enic_start_rq(enic, index);
644 vnic_dev_add_addr(enic->vdev, enic->mac_addr);
646 vnic_dev_enable_wait(enic->vdev);
648 /* Register and enable error interrupt */
649 rte_intr_callback_register(&(enic->pdev->intr_handle),
650 enic_intr_handler, (void *)enic->rte_dev);
652 rte_intr_enable(&(enic->pdev->intr_handle));
653 /* Unmask LSC interrupt */
654 vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
659 int enic_alloc_intr_resources(struct enic *enic)
664 dev_info(enic, "vNIC resources used: "\
665 "wq %d rq %d cq %d intr %d\n",
666 enic->wq_count, enic_vnic_rq_count(enic),
667 enic->cq_count, enic->intr_count);
669 for (i = 0; i < enic->intr_count; i++) {
670 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
672 enic_free_vnic_resources(enic);
679 void enic_free_rq(void *rxq)
681 struct vnic_rq *rq_sop, *rq_data;
687 rq_sop = (struct vnic_rq *)rxq;
688 enic = vnic_dev_priv(rq_sop->vdev);
689 rq_data = &enic->rq[rq_sop->data_queue_idx];
691 if (rq_sop->free_mbufs) {
692 struct rte_mbuf **mb;
695 mb = rq_sop->free_mbufs;
696 for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs;
697 i < ENIC_RX_BURST_MAX; i++)
698 rte_pktmbuf_free(mb[i]);
699 rte_free(rq_sop->free_mbufs);
700 rq_sop->free_mbufs = NULL;
701 rq_sop->num_free_mbufs = 0;
704 enic_rxmbuf_queue_release(enic, rq_sop);
706 enic_rxmbuf_queue_release(enic, rq_data);
708 rte_free(rq_sop->mbuf_ring);
710 rte_free(rq_data->mbuf_ring);
712 rq_sop->mbuf_ring = NULL;
713 rq_data->mbuf_ring = NULL;
715 vnic_rq_free(rq_sop);
717 vnic_rq_free(rq_data);
719 vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
725 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
727 struct rte_eth_dev *eth_dev = enic->rte_dev;
728 vnic_wq_enable(&enic->wq[queue_idx]);
729 eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
732 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
734 struct rte_eth_dev *eth_dev = enic->rte_dev;
737 ret = vnic_wq_disable(&enic->wq[queue_idx]);
741 eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
745 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
747 struct vnic_rq *rq_sop;
748 struct vnic_rq *rq_data;
749 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
750 rq_data = &enic->rq[rq_sop->data_queue_idx];
751 struct rte_eth_dev *eth_dev = enic->rte_dev;
753 if (rq_data->in_use) {
754 vnic_rq_enable(rq_data);
755 enic_initial_post_rx(enic, rq_data);
758 vnic_rq_enable(rq_sop);
759 enic_initial_post_rx(enic, rq_sop);
760 eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
763 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
765 int ret1 = 0, ret2 = 0;
766 struct rte_eth_dev *eth_dev = enic->rte_dev;
767 struct vnic_rq *rq_sop;
768 struct vnic_rq *rq_data;
769 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
770 rq_data = &enic->rq[rq_sop->data_queue_idx];
772 ret2 = vnic_rq_disable(rq_sop);
775 ret1 = vnic_rq_disable(rq_data);
782 eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
786 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
787 unsigned int socket_id, struct rte_mempool *mp,
788 uint16_t nb_desc, uint16_t free_thresh)
791 uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
792 uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
793 struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
794 struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
795 unsigned int mbuf_size, mbufs_per_pkt;
796 unsigned int nb_sop_desc, nb_data_desc;
797 uint16_t min_sop, max_sop, min_data, max_data;
798 uint32_t max_rx_pkt_len;
801 rq_sop->data_queue_idx = data_queue_idx;
803 rq_data->data_queue_idx = 0;
804 rq_sop->socket_id = socket_id;
806 rq_data->socket_id = socket_id;
809 rq_sop->rx_free_thresh = free_thresh;
810 rq_data->rx_free_thresh = free_thresh;
811 dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
814 mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
815 RTE_PKTMBUF_HEADROOM);
816 /* max_rx_pkt_len includes the ethernet header and CRC. */
817 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
819 if (enic->rte_dev->data->dev_conf.rxmode.offloads &
820 DEV_RX_OFFLOAD_SCATTER) {
821 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
822 /* ceil((max pkt len)/mbuf_size) */
823 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
825 dev_info(enic, "Scatter rx mode disabled\n");
827 if (max_rx_pkt_len > mbuf_size) {
828 dev_warning(enic, "The maximum Rx packet size (%u) is"
829 " larger than the mbuf size (%u), and"
830 " scatter is disabled. Larger packets will"
832 max_rx_pkt_len, mbuf_size);
836 if (mbufs_per_pkt > 1) {
837 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
838 rq_sop->data_queue_enable = 1;
841 * HW does not directly support rxmode.max_rx_pkt_len. HW always
842 * receives packet sizes up to the "max" MTU.
843 * If not using scatter, we can achieve the effect of dropping
844 * larger packets by reducing the size of posted buffers.
845 * See enic_alloc_rx_queue_mbufs().
848 enic_mtu_to_max_rx_pktlen(enic->max_mtu)) {
849 dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
850 " when scatter rx mode is in use.\n");
853 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
855 rq_sop->data_queue_enable = 0;
859 /* number of descriptors have to be a multiple of 32 */
860 nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK;
861 nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK;
863 rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
864 rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
866 if (mbufs_per_pkt > 1) {
867 min_sop = ENIC_RX_BURST_MAX;
868 max_sop = ((enic->config.rq_desc_count /
869 (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK);
870 min_data = min_sop * (mbufs_per_pkt - 1);
871 max_data = enic->config.rq_desc_count;
873 min_sop = ENIC_RX_BURST_MAX;
874 max_sop = enic->config.rq_desc_count;
879 if (nb_desc < (min_sop + min_data)) {
881 "Number of rx descs too low, adjusting to minimum\n");
882 nb_sop_desc = min_sop;
883 nb_data_desc = min_data;
884 } else if (nb_desc > (max_sop + max_data)) {
886 "Number of rx_descs too high, adjusting to maximum\n");
887 nb_sop_desc = max_sop;
888 nb_data_desc = max_data;
890 if (mbufs_per_pkt > 1) {
891 dev_info(enic, "For max packet size %u and mbuf size %u valid"
892 " rx descriptor range is %u to %u\n",
893 max_rx_pkt_len, mbuf_size, min_sop + min_data,
896 dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
897 nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
899 /* Allocate sop queue resources */
900 rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
901 nb_sop_desc, sizeof(struct rq_enet_desc));
903 dev_err(enic, "error in allocation of sop rq\n");
906 nb_sop_desc = rq_sop->ring.desc_count;
908 if (rq_data->in_use) {
909 /* Allocate data queue resources */
910 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
912 sizeof(struct rq_enet_desc));
914 dev_err(enic, "error in allocation of data rq\n");
915 goto err_free_rq_sop;
917 nb_data_desc = rq_data->ring.desc_count;
919 rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
920 socket_id, nb_sop_desc + nb_data_desc,
921 sizeof(struct cq_enet_rq_desc));
923 dev_err(enic, "error in allocation of cq for rq\n");
924 goto err_free_rq_data;
927 /* Allocate the mbuf rings */
928 rq_sop->mbuf_ring = (struct rte_mbuf **)
929 rte_zmalloc_socket("rq->mbuf_ring",
930 sizeof(struct rte_mbuf *) * nb_sop_desc,
931 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
932 if (rq_sop->mbuf_ring == NULL)
935 if (rq_data->in_use) {
936 rq_data->mbuf_ring = (struct rte_mbuf **)
937 rte_zmalloc_socket("rq->mbuf_ring",
938 sizeof(struct rte_mbuf *) * nb_data_desc,
939 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
940 if (rq_data->mbuf_ring == NULL)
941 goto err_free_sop_mbuf;
944 rq_sop->free_mbufs = (struct rte_mbuf **)
945 rte_zmalloc_socket("rq->free_mbufs",
946 sizeof(struct rte_mbuf *) *
948 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
949 if (rq_sop->free_mbufs == NULL)
950 goto err_free_data_mbuf;
951 rq_sop->num_free_mbufs = 0;
953 rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
958 rte_free(rq_data->mbuf_ring);
960 rte_free(rq_sop->mbuf_ring);
962 /* cleanup on error */
963 vnic_cq_free(&enic->cq[queue_idx]);
966 vnic_rq_free(rq_data);
968 vnic_rq_free(rq_sop);
973 void enic_free_wq(void *txq)
981 wq = (struct vnic_wq *)txq;
982 enic = vnic_dev_priv(wq->vdev);
983 rte_memzone_free(wq->cqmsg_rz);
985 vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
988 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
989 unsigned int socket_id, uint16_t nb_desc)
992 struct vnic_wq *wq = &enic->wq[queue_idx];
993 unsigned int cq_index = enic_cq_wq(enic, queue_idx);
997 wq->socket_id = socket_id;
999 * rte_eth_tx_queue_setup() checks min, max, and alignment. So just
1000 * print an info message for diagnostics.
1002 dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc);
1004 /* Allocate queue resources */
1005 err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
1007 sizeof(struct wq_enet_desc));
1009 dev_err(enic, "error in allocation of wq\n");
1013 err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
1015 sizeof(struct cq_enet_wq_desc));
1018 dev_err(enic, "error in allocation of cq for wq\n");
1021 /* setup up CQ message */
1022 snprintf((char *)name, sizeof(name),
1023 "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
1026 wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
1027 sizeof(uint32_t), SOCKET_ID_ANY,
1028 RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
1035 int enic_disable(struct enic *enic)
1040 for (i = 0; i < enic->intr_count; i++) {
1041 vnic_intr_mask(&enic->intr[i]);
1042 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
1044 enic_rxq_intr_deinit(enic);
1045 rte_intr_disable(&enic->pdev->intr_handle);
1046 rte_intr_callback_unregister(&enic->pdev->intr_handle,
1048 (void *)enic->rte_dev);
1050 vnic_dev_disable(enic->vdev);
1052 enic_clsf_destroy(enic);
1054 if (!enic_is_sriov_vf(enic))
1055 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
1057 for (i = 0; i < enic->wq_count; i++) {
1058 err = vnic_wq_disable(&enic->wq[i]);
1062 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
1063 if (enic->rq[i].in_use) {
1064 err = vnic_rq_disable(&enic->rq[i]);
1070 /* If we were using interrupts, set the interrupt vector to -1
1071 * to disable interrupts. We are not disabling link notifcations,
1072 * though, as we want the polling of link status to continue working.
1074 if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
1075 vnic_dev_notify_set(enic->vdev, -1);
1077 vnic_dev_set_reset_flag(enic->vdev, 1);
1079 for (i = 0; i < enic->wq_count; i++)
1080 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
1082 for (i = 0; i < enic_vnic_rq_count(enic); i++)
1083 if (enic->rq[i].in_use)
1084 vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
1085 for (i = 0; i < enic->cq_count; i++)
1086 vnic_cq_clean(&enic->cq[i]);
1087 for (i = 0; i < enic->intr_count; i++)
1088 vnic_intr_clean(&enic->intr[i]);
1093 static int enic_dev_wait(struct vnic_dev *vdev,
1094 int (*start)(struct vnic_dev *, int),
1095 int (*finished)(struct vnic_dev *, int *),
1102 err = start(vdev, arg);
1106 /* Wait for func to complete...2 seconds max */
1107 for (i = 0; i < 2000; i++) {
1108 err = finished(vdev, &done);
1118 static int enic_dev_open(struct enic *enic)
1121 int flags = CMD_OPENF_IG_DESCCACHE;
1123 err = enic_dev_wait(enic->vdev, vnic_dev_open,
1124 vnic_dev_open_done, flags);
1126 dev_err(enic_get_dev(enic),
1127 "vNIC device open failed, err %d\n", err);
1132 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
1134 dma_addr_t rss_key_buf_pa;
1135 union vnic_rss_key *rss_key_buf_va = NULL;
1139 RTE_ASSERT(user_key != NULL);
1140 snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
1141 rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1142 &rss_key_buf_pa, name);
1143 if (!rss_key_buf_va)
1146 for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1147 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1149 err = enic_set_rss_key(enic,
1151 sizeof(union vnic_rss_key));
1153 /* Save for later queries */
1155 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1156 sizeof(union vnic_rss_key));
1158 enic_free_consistent(enic, sizeof(union vnic_rss_key),
1159 rss_key_buf_va, rss_key_buf_pa);
1164 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1166 dma_addr_t rss_cpu_buf_pa;
1167 union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1171 snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
1172 rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1173 &rss_cpu_buf_pa, name);
1174 if (!rss_cpu_buf_va)
1177 rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1179 err = enic_set_rss_cpu(enic,
1181 sizeof(union vnic_rss_cpu));
1183 enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1184 rss_cpu_buf_va, rss_cpu_buf_pa);
1186 /* Save for later queries */
1188 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1192 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
1193 u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
1195 const u8 tso_ipid_split_en = 0;
1198 err = enic_set_nic_cfg(enic,
1199 rss_default_cpu, rss_hash_type,
1200 rss_hash_bits, rss_base_cpu,
1201 rss_enable, tso_ipid_split_en,
1202 enic->ig_vlan_strip_en);
1207 /* Initialize RSS with defaults, called from dev_configure */
1208 int enic_init_rss_nic_cfg(struct enic *enic)
1210 static uint8_t default_rss_key[] = {
1211 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1212 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1213 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1214 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1216 struct rte_eth_rss_conf rss_conf;
1217 union vnic_rss_cpu rss_cpu;
1220 rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1222 * If setting key for the first time, and the user gives us none, then
1223 * push the default key to NIC.
1225 if (rss_conf.rss_key == NULL) {
1226 rss_conf.rss_key = default_rss_key;
1227 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1229 ret = enic_set_rss_conf(enic, &rss_conf);
1231 dev_err(enic, "Failed to configure RSS\n");
1234 if (enic->rss_enable) {
1235 /* If enabling RSS, use the default reta */
1236 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1237 rss_cpu.cpu[i / 4].b[i % 4] =
1238 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1240 ret = enic_set_rss_reta(enic, &rss_cpu);
1242 dev_err(enic, "Failed to set RSS indirection table\n");
1247 int enic_setup_finish(struct enic *enic)
1249 enic_init_soft_stats(enic);
1252 vnic_dev_packet_filter(enic->vdev,
1265 static int enic_rss_conf_valid(struct enic *enic,
1266 struct rte_eth_rss_conf *rss_conf)
1268 /* RSS is disabled per VIC settings. Ignore rss_conf. */
1269 if (enic->flow_type_rss_offloads == 0)
1271 if (rss_conf->rss_key != NULL &&
1272 rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1273 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1274 rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1277 if (rss_conf->rss_hf != 0 &&
1278 (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1279 dev_err(enic, "Given rss_hf contains none of the supported"
1286 /* Set hash type and key according to rss_conf */
1287 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1289 struct rte_eth_dev *eth_dev;
1295 RTE_ASSERT(rss_conf != NULL);
1296 ret = enic_rss_conf_valid(enic, rss_conf);
1298 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1302 eth_dev = enic->rte_dev;
1304 rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1305 if (enic->rq_count > 1 &&
1306 (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1309 if (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1310 ETH_RSS_NONFRAG_IPV4_OTHER))
1311 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1312 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1313 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1314 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
1315 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4;
1316 if (enic->udp_rss_weak) {
1318 * 'TCP' is not a typo. The "weak" version of
1319 * UDP RSS requires both the TCP and UDP bits
1320 * be set. It does enable TCP RSS as well.
1322 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1325 if (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_IPV6_EX |
1326 ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER))
1327 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1328 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX))
1329 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1330 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_UDP_EX)) {
1331 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6;
1332 if (enic->udp_rss_weak)
1333 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1340 /* Set the hash key if provided */
1341 if (rss_enable && rss_conf->rss_key) {
1342 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1344 dev_err(enic, "Failed to set RSS key\n");
1349 ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1350 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1353 enic->rss_hf = rss_hf;
1354 enic->rss_hash_type = rss_hash_type;
1355 enic->rss_enable = rss_enable;
1357 dev_err(enic, "Failed to update RSS configurations."
1358 " hash=0x%x\n", rss_hash_type);
1363 int enic_set_vlan_strip(struct enic *enic)
1366 * Unfortunately, VLAN strip on/off and RSS on/off are configured
1367 * together. So, re-do niccfg, preserving the current RSS settings.
1369 return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1370 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1374 int enic_add_packet_filter(struct enic *enic)
1376 /* Args -> directed, multicast, broadcast, promisc, allmulti */
1377 return vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1378 enic->promisc, enic->allmulti);
1381 int enic_get_link_status(struct enic *enic)
1383 return vnic_dev_link_status(enic->vdev);
1386 static void enic_dev_deinit(struct enic *enic)
1388 /* stop link status checking */
1389 vnic_dev_notify_unset(enic->vdev);
1391 /* mac_addrs is freed by rte_eth_dev_release_port() */
1393 rte_free(enic->intr);
1399 int enic_set_vnic_res(struct enic *enic)
1401 struct rte_eth_dev *eth_dev = enic->rte_dev;
1403 unsigned int required_rq, required_wq, required_cq, required_intr;
1405 /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1406 required_rq = eth_dev->data->nb_rx_queues * 2;
1407 required_wq = eth_dev->data->nb_tx_queues;
1408 required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1409 required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1410 if (eth_dev->data->dev_conf.intr_conf.rxq) {
1411 required_intr += eth_dev->data->nb_rx_queues;
1414 if (enic->conf_rq_count < required_rq) {
1415 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1416 eth_dev->data->nb_rx_queues,
1417 required_rq, enic->conf_rq_count);
1420 if (enic->conf_wq_count < required_wq) {
1421 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1422 eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1426 if (enic->conf_cq_count < required_cq) {
1427 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1428 required_cq, enic->conf_cq_count);
1431 if (enic->conf_intr_count < required_intr) {
1432 dev_err(dev, "Not enough Interrupts to support Rx queue"
1433 " interrupts. Required:%u, Configured:%u\n",
1434 required_intr, enic->conf_intr_count);
1439 enic->rq_count = eth_dev->data->nb_rx_queues;
1440 enic->wq_count = eth_dev->data->nb_tx_queues;
1441 enic->cq_count = enic->rq_count + enic->wq_count;
1442 enic->intr_count = required_intr;
1448 /* Initialize the completion queue for an RQ */
1450 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1452 struct vnic_rq *sop_rq, *data_rq;
1453 unsigned int cq_idx;
1456 sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1457 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1460 vnic_cq_clean(&enic->cq[cq_idx]);
1461 vnic_cq_init(&enic->cq[cq_idx],
1462 0 /* flow_control_enable */,
1463 1 /* color_enable */,
1466 1 /* cq_tail_color */,
1467 0 /* interrupt_enable */,
1468 1 /* cq_entry_enable */,
1469 0 /* cq_message_enable */,
1470 0 /* interrupt offset */,
1471 0 /* cq_message_addr */);
1474 vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1475 enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1476 sop_rq->ring.desc_count - 1, 1, 0);
1477 if (data_rq->in_use) {
1478 vnic_rq_init_start(data_rq,
1480 enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1481 data_rq->ring.desc_count - 1, 1, 0);
1484 rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1488 if (data_rq->in_use) {
1489 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1491 enic_rxmbuf_queue_release(enic, sop_rq);
1499 /* The Cisco NIC can send and receive packets up to a max packet size
1500 * determined by the NIC type and firmware. There is also an MTU
1501 * configured into the NIC via the CIMC/UCSM management interface
1502 * which can be overridden by this function (up to the max packet size).
1503 * Depending on the network setup, doing so may cause packet drops
1504 * and unexpected behavior.
1506 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1508 unsigned int rq_idx;
1511 uint16_t old_mtu; /* previous setting */
1512 uint16_t config_mtu; /* Value configured into NIC via CIMC/UCSM */
1513 struct rte_eth_dev *eth_dev = enic->rte_dev;
1515 old_mtu = eth_dev->data->mtu;
1516 config_mtu = enic->config.mtu;
1518 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1519 return -E_RTE_SECONDARY;
1521 if (new_mtu > enic->max_mtu) {
1523 "MTU not updated: requested (%u) greater than max (%u)\n",
1524 new_mtu, enic->max_mtu);
1527 if (new_mtu < ENIC_MIN_MTU) {
1529 "MTU not updated: requested (%u) less than min (%u)\n",
1530 new_mtu, ENIC_MIN_MTU);
1533 if (new_mtu > config_mtu)
1535 "MTU (%u) is greater than value configured in NIC (%u)\n",
1536 new_mtu, config_mtu);
1538 /* Update the MTU and maximum packet length */
1539 eth_dev->data->mtu = new_mtu;
1540 eth_dev->data->dev_conf.rxmode.max_rx_pkt_len =
1541 enic_mtu_to_max_rx_pktlen(new_mtu);
1544 * If the device has not started (enic_enable), nothing to do.
1545 * Later, enic_enable() will set up RQs reflecting the new maximum
1548 if (!eth_dev->data->dev_started)
1552 * The device has started, re-do RQs on the fly. In the process, we
1553 * pick up the new maximum packet length.
1555 * Some applications rely on the ability to change MTU without stopping
1556 * the device. So keep this behavior for now.
1558 rte_spinlock_lock(&enic->mtu_lock);
1560 /* Stop traffic on all RQs */
1561 for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1562 rq = &enic->rq[rq_idx];
1563 if (rq->is_sop && rq->in_use) {
1564 rc = enic_stop_rq(enic,
1565 enic_sop_rq_idx_to_rte_idx(rq_idx));
1567 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1573 /* replace Rx function with a no-op to avoid getting stale pkts */
1574 eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1577 /* Allow time for threads to exit the real Rx function. */
1580 /* now it is safe to reconfigure the RQs */
1583 /* free and reallocate RQs with the new MTU */
1584 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1585 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1590 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1591 rq->tot_nb_desc, rq->rx_free_thresh);
1594 "Fatal MTU alloc error- No traffic will pass\n");
1598 rc = enic_reinit_rq(enic, rq_idx);
1601 "Fatal MTU RQ reinit- No traffic will pass\n");
1606 /* put back the real receive function */
1608 pick_rx_handler(enic);
1611 /* restart Rx traffic */
1612 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1613 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1614 if (rq->is_sop && rq->in_use)
1615 enic_start_rq(enic, rq_idx);
1619 dev_info(enic, "MTU changed from %u to %u\n", old_mtu, new_mtu);
1620 rte_spinlock_unlock(&enic->mtu_lock);
1624 static int enic_dev_init(struct enic *enic)
1627 struct rte_eth_dev *eth_dev = enic->rte_dev;
1629 vnic_dev_intr_coal_timer_info_default(enic->vdev);
1631 /* Get vNIC configuration
1633 err = enic_get_vnic_config(enic);
1635 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1639 /* Get available resource counts */
1640 enic_get_res_counts(enic);
1641 if (enic->conf_rq_count == 1) {
1642 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1643 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1644 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1647 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1648 enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1649 enic->conf_cq_count, 8);
1650 enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1651 enic->conf_intr_count, 8);
1652 enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1653 enic->conf_rq_count, 8);
1654 enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1655 enic->conf_wq_count, 8);
1656 if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1657 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1660 if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1661 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1664 if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1665 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1668 if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1669 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1673 /* Get the supported filters */
1674 enic_fdir_info(enic);
1676 eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr",
1677 sizeof(struct rte_ether_addr) *
1678 ENIC_UNICAST_PERFECT_FILTERS, 0);
1679 if (!eth_dev->data->mac_addrs) {
1680 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1683 rte_ether_addr_copy((struct rte_ether_addr *)enic->mac_addr,
1684 eth_dev->data->mac_addrs);
1686 vnic_dev_set_reset_flag(enic->vdev, 0);
1688 LIST_INIT(&enic->flows);
1690 /* set up link status checking */
1691 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1693 enic->overlay_offload = false;
1694 if (enic->disable_overlay && enic->vxlan) {
1696 * Explicitly disable overlay offload as the setting is
1697 * sticky, and resetting vNIC does not disable it.
1699 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1700 OVERLAY_FEATURE_VXLAN,
1701 OVERLAY_OFFLOAD_DISABLE)) {
1702 dev_err(enic, "failed to disable overlay offload\n");
1704 dev_info(enic, "Overlay offload is disabled\n");
1707 if (!enic->disable_overlay && enic->vxlan &&
1708 /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1709 vnic_dev_overlay_offload_ctrl(enic->vdev,
1710 OVERLAY_FEATURE_VXLAN,
1711 OVERLAY_OFFLOAD_ENABLE) == 0) {
1712 enic->tx_offload_capa |=
1713 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1714 DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
1715 DEV_TX_OFFLOAD_VXLAN_TNL_TSO;
1716 enic->tx_offload_mask |=
1719 PKT_TX_OUTER_IP_CKSUM |
1721 enic->overlay_offload = true;
1722 dev_info(enic, "Overlay offload is enabled\n");
1725 * Reset the vxlan port if HW vxlan parsing is available. It
1726 * is always enabled regardless of overlay offload
1730 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT;
1732 * Reset the vxlan port to the default, as the NIC firmware
1733 * does not reset it automatically and keeps the old setting.
1735 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1736 OVERLAY_CFG_VXLAN_PORT_UPDATE,
1737 ENIC_DEFAULT_VXLAN_PORT)) {
1738 dev_err(enic, "failed to update vxlan port\n");
1747 int enic_probe(struct enic *enic)
1749 struct rte_pci_device *pdev = enic->pdev;
1752 dev_debug(enic, "Initializing ENIC PMD\n");
1754 /* if this is a secondary process the hardware is already initialized */
1755 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1758 enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1759 enic->bar0.len = pdev->mem_resource[0].len;
1761 /* Register vNIC device */
1762 enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1764 dev_err(enic, "vNIC registration failed, aborting\n");
1768 LIST_INIT(&enic->memzone_list);
1769 rte_spinlock_init(&enic->memzone_list_lock);
1771 vnic_register_cbacks(enic->vdev,
1772 enic_alloc_consistent,
1773 enic_free_consistent);
1776 * Allocate the consistent memory for stats upfront so both primary and
1777 * secondary processes can dump stats.
1779 err = vnic_dev_alloc_stats_mem(enic->vdev);
1781 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1782 goto err_out_unregister;
1784 /* Issue device open to get device in known state */
1785 err = enic_dev_open(enic);
1787 dev_err(enic, "vNIC dev open failed, aborting\n");
1788 goto err_out_unregister;
1791 /* Set ingress vlan rewrite mode before vnic initialization */
1792 dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n",
1793 enic->ig_vlan_rewrite_mode);
1794 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1795 enic->ig_vlan_rewrite_mode);
1798 "Failed to set ingress vlan rewrite mode, aborting.\n");
1799 goto err_out_dev_close;
1802 /* Issue device init to initialize the vnic-to-switch link.
1803 * We'll start with carrier off and wait for link UP
1804 * notification later to turn on carrier. We don't need
1805 * to wait here for the vnic-to-switch link initialization
1806 * to complete; link UP notification is the indication that
1807 * the process is complete.
1810 err = vnic_dev_init(enic->vdev, 0);
1812 dev_err(enic, "vNIC dev init failed, aborting\n");
1813 goto err_out_dev_close;
1816 err = enic_dev_init(enic);
1818 dev_err(enic, "Device initialization failed, aborting\n");
1819 goto err_out_dev_close;
1825 vnic_dev_close(enic->vdev);
1827 vnic_dev_unregister(enic->vdev);
1832 void enic_remove(struct enic *enic)
1834 enic_dev_deinit(enic);
1835 vnic_dev_close(enic->vdev);
1836 vnic_dev_unregister(enic->vdev);