1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
14 #include <rte_bus_pci.h>
15 #include <rte_memzone.h>
16 #include <rte_malloc.h>
18 #include <rte_string_fns.h>
19 #include <rte_ethdev_driver.h>
21 #include "enic_compat.h"
23 #include "wq_enet_desc.h"
24 #include "rq_enet_desc.h"
25 #include "cq_enet_desc.h"
26 #include "vnic_enet.h"
31 #include "vnic_intr.h"
34 static inline int enic_is_sriov_vf(struct enic *enic)
36 return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
39 static int is_zero_addr(uint8_t *addr)
41 return !(addr[0] | addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
44 static int is_mcast_addr(uint8_t *addr)
49 static int is_eth_addr_valid(uint8_t *addr)
51 return !is_mcast_addr(addr) && !is_zero_addr(addr);
55 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
59 if (!rq || !rq->mbuf_ring) {
60 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
64 for (i = 0; i < rq->ring.desc_count; i++) {
65 if (rq->mbuf_ring[i]) {
66 rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
67 rq->mbuf_ring[i] = NULL;
72 static void enic_free_wq_buf(struct rte_mbuf **buf)
74 struct rte_mbuf *mbuf = *buf;
76 rte_pktmbuf_free_seg(mbuf);
80 static void enic_log_q_error(struct enic *enic)
85 for (i = 0; i < enic->wq_count; i++) {
86 error_status = vnic_wq_error_status(&enic->wq[i]);
88 dev_err(enic, "WQ[%d] error_status %d\n", i,
92 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
93 if (!enic->rq[i].in_use)
95 error_status = vnic_rq_error_status(&enic->rq[i]);
97 dev_err(enic, "RQ[%d] error_status %d\n", i,
102 static void enic_clear_soft_stats(struct enic *enic)
104 struct enic_soft_stats *soft_stats = &enic->soft_stats;
105 rte_atomic64_clear(&soft_stats->rx_nombuf);
106 rte_atomic64_clear(&soft_stats->rx_packet_errors);
107 rte_atomic64_clear(&soft_stats->tx_oversized);
110 static void enic_init_soft_stats(struct enic *enic)
112 struct enic_soft_stats *soft_stats = &enic->soft_stats;
113 rte_atomic64_init(&soft_stats->rx_nombuf);
114 rte_atomic64_init(&soft_stats->rx_packet_errors);
115 rte_atomic64_init(&soft_stats->tx_oversized);
116 enic_clear_soft_stats(enic);
119 void enic_dev_stats_clear(struct enic *enic)
121 if (vnic_dev_stats_clear(enic->vdev))
122 dev_err(enic, "Error in clearing stats\n");
123 enic_clear_soft_stats(enic);
126 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
128 struct vnic_stats *stats;
129 struct enic_soft_stats *soft_stats = &enic->soft_stats;
130 int64_t rx_truncated;
131 uint64_t rx_packet_errors;
132 int ret = vnic_dev_stats_dump(enic->vdev, &stats);
135 dev_err(enic, "Error in getting stats\n");
139 /* The number of truncated packets can only be calculated by
140 * subtracting a hardware counter from error packets received by
141 * the driver. Note: this causes transient inaccuracies in the
142 * ipackets count. Also, the length of truncated packets are
143 * counted in ibytes even though truncated packets are dropped
144 * which can make ibytes be slightly higher than it should be.
146 rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
147 rx_truncated = rx_packet_errors - stats->rx.rx_errors;
149 r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
150 r_stats->opackets = stats->tx.tx_frames_ok;
152 r_stats->ibytes = stats->rx.rx_bytes_ok;
153 r_stats->obytes = stats->tx.tx_bytes_ok;
155 r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
156 r_stats->oerrors = stats->tx.tx_errors
157 + rte_atomic64_read(&soft_stats->tx_oversized);
159 r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
161 r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
165 int enic_del_mac_address(struct enic *enic, int mac_index)
167 struct rte_eth_dev *eth_dev = enic->rte_dev;
168 uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
170 return vnic_dev_del_addr(enic->vdev, mac_addr);
173 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
177 if (!is_eth_addr_valid(mac_addr)) {
178 dev_err(enic, "invalid mac address\n");
182 err = vnic_dev_add_addr(enic->vdev, mac_addr);
184 dev_err(enic, "add mac addr failed\n");
189 enic_free_rq_buf(struct rte_mbuf **mbuf)
194 rte_pktmbuf_free(*mbuf);
198 void enic_init_vnic_resources(struct enic *enic)
200 unsigned int error_interrupt_enable = 1;
201 unsigned int error_interrupt_offset = 0;
202 unsigned int rxq_interrupt_enable = 0;
203 unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
204 unsigned int index = 0;
206 struct vnic_rq *data_rq;
208 if (enic->rte_dev->data->dev_conf.intr_conf.rxq)
209 rxq_interrupt_enable = 1;
211 for (index = 0; index < enic->rq_count; index++) {
212 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
214 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
216 error_interrupt_enable,
217 error_interrupt_offset);
219 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
221 vnic_rq_init(data_rq,
223 error_interrupt_enable,
224 error_interrupt_offset);
226 vnic_cq_init(&enic->cq[cq_idx],
227 0 /* flow_control_enable */,
228 1 /* color_enable */,
231 1 /* cq_tail_color */,
232 rxq_interrupt_enable,
233 1 /* cq_entry_enable */,
234 0 /* cq_message_enable */,
235 rxq_interrupt_offset,
236 0 /* cq_message_addr */);
237 if (rxq_interrupt_enable)
238 rxq_interrupt_offset++;
241 for (index = 0; index < enic->wq_count; index++) {
242 vnic_wq_init(&enic->wq[index],
243 enic_cq_wq(enic, index),
244 error_interrupt_enable,
245 error_interrupt_offset);
246 /* Compute unsupported ol flags for enic_prep_pkts() */
247 enic->wq[index].tx_offload_notsup_mask =
248 PKT_TX_OFFLOAD_MASK ^ enic->tx_offload_mask;
250 cq_idx = enic_cq_wq(enic, index);
251 vnic_cq_init(&enic->cq[cq_idx],
252 0 /* flow_control_enable */,
253 1 /* color_enable */,
256 1 /* cq_tail_color */,
257 0 /* interrupt_enable */,
258 0 /* cq_entry_enable */,
259 1 /* cq_message_enable */,
260 0 /* interrupt offset */,
261 (u64)enic->wq[index].cqmsg_rz->iova);
264 for (index = 0; index < enic->intr_count; index++) {
265 vnic_intr_init(&enic->intr[index],
266 enic->config.intr_timer_usec,
267 enic->config.intr_timer_type,
268 /*mask_on_assertion*/1);
274 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
277 struct rq_enet_desc *rqd = rq->ring.descs;
280 uint32_t max_rx_pkt_len;
286 dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
287 rq->ring.desc_count);
290 * If *not* using scatter and the mbuf size is greater than the
291 * requested max packet size (max_rx_pkt_len), then reduce the
292 * posted buffer size to max_rx_pkt_len. HW still receives packets
293 * larger than max_rx_pkt_len, but they will be truncated, which we
294 * drop in the rx handler. Not ideal, but better than returning
295 * large packets when the user is not expecting them.
297 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
298 rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
299 if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
300 rq_buf_len = max_rx_pkt_len;
301 for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
302 mb = rte_mbuf_raw_alloc(rq->mp);
304 dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
305 (unsigned)rq->index);
309 mb->data_off = RTE_PKTMBUF_HEADROOM;
310 dma_addr = (dma_addr_t)(mb->buf_iova
311 + RTE_PKTMBUF_HEADROOM);
312 rq_enet_desc_enc(rqd, dma_addr,
313 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
314 : RQ_ENET_TYPE_NOT_SOP),
316 rq->mbuf_ring[i] = mb;
319 * Do not post the buffers to the NIC until we enable the RQ via
322 rq->need_initial_post = true;
323 /* Initialize fetch index while RQ is disabled */
324 iowrite32(0, &rq->ctrl->fetch_index);
329 * Post the Rx buffers for the first time. enic_alloc_rx_queue_mbufs() has
330 * allocated the buffers and filled the RQ descriptor ring. Just need to push
331 * the post index to the NIC.
334 enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)
336 if (!rq->in_use || !rq->need_initial_post)
339 /* make sure all prior writes are complete before doing the PIO write */
342 /* Post all but the last buffer to VIC. */
343 rq->posted_index = rq->ring.desc_count - 1;
347 dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
348 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
349 iowrite32(rq->posted_index, &rq->ctrl->posted_index);
351 rq->need_initial_post = false;
355 enic_alloc_consistent(void *priv, size_t size,
356 dma_addr_t *dma_handle, u8 *name)
359 const struct rte_memzone *rz;
361 struct enic *enic = (struct enic *)priv;
362 struct enic_memzone_entry *mze;
364 rz = rte_memzone_reserve_aligned((const char *)name, size,
365 SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
367 pr_err("%s : Failed to allocate memory requested for %s\n",
373 *dma_handle = (dma_addr_t)rz->iova;
375 mze = rte_malloc("enic memzone entry",
376 sizeof(struct enic_memzone_entry), 0);
379 pr_err("%s : Failed to allocate memory for memzone list\n",
381 rte_memzone_free(rz);
387 rte_spinlock_lock(&enic->memzone_list_lock);
388 LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
389 rte_spinlock_unlock(&enic->memzone_list_lock);
395 enic_free_consistent(void *priv,
396 __rte_unused size_t size,
398 dma_addr_t dma_handle)
400 struct enic_memzone_entry *mze;
401 struct enic *enic = (struct enic *)priv;
403 rte_spinlock_lock(&enic->memzone_list_lock);
404 LIST_FOREACH(mze, &enic->memzone_list, entries) {
405 if (mze->rz->addr == vaddr &&
406 mze->rz->iova == dma_handle)
410 rte_spinlock_unlock(&enic->memzone_list_lock);
412 "Tried to free memory, but couldn't find it in the memzone list\n");
415 LIST_REMOVE(mze, entries);
416 rte_spinlock_unlock(&enic->memzone_list_lock);
417 rte_memzone_free(mze->rz);
421 int enic_link_update(struct enic *enic)
423 struct rte_eth_dev *eth_dev = enic->rte_dev;
424 struct rte_eth_link link;
426 memset(&link, 0, sizeof(link));
427 link.link_status = enic_get_link_status(enic);
428 link.link_duplex = ETH_LINK_FULL_DUPLEX;
429 link.link_speed = vnic_dev_port_speed(enic->vdev);
431 return rte_eth_linkstatus_set(eth_dev, &link);
435 enic_intr_handler(void *arg)
437 struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
438 struct enic *enic = pmd_priv(dev);
440 vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
442 enic_link_update(enic);
443 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
444 enic_log_q_error(enic);
447 static int enic_rxq_intr_init(struct enic *enic)
449 struct rte_intr_handle *intr_handle;
450 uint32_t rxq_intr_count, i;
453 intr_handle = enic->rte_dev->intr_handle;
454 if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
457 * Rx queue interrupts only work when we have MSI-X interrupts,
458 * one per queue. Sharing one interrupt is technically
459 * possible with VIC, but it is not worth the complications it brings.
461 if (!rte_intr_cap_multiple(intr_handle)) {
462 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
463 " (vfio-pci driver)\n");
466 rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
467 err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
469 dev_err(enic, "Failed to enable event fds for Rx queue"
473 intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
474 rxq_intr_count * sizeof(int), 0);
475 if (intr_handle->intr_vec == NULL) {
476 dev_err(enic, "Failed to allocate intr_vec\n");
479 for (i = 0; i < rxq_intr_count; i++)
480 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
484 static void enic_rxq_intr_deinit(struct enic *enic)
486 struct rte_intr_handle *intr_handle;
488 intr_handle = enic->rte_dev->intr_handle;
489 rte_intr_efd_disable(intr_handle);
490 if (intr_handle->intr_vec != NULL) {
491 rte_free(intr_handle->intr_vec);
492 intr_handle->intr_vec = NULL;
496 static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
498 struct wq_enet_desc *desc;
503 * Fill WQ descriptor fields that never change. Every descriptor is
504 * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
505 * descriptors (i.e. request one completion update every 32 packets).
507 wq = &enic->wq[queue_idx];
508 desc = (struct wq_enet_desc *)wq->ring.descs;
509 for (i = 0; i < wq->ring.desc_count; i++, desc++) {
510 desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
511 if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
512 desc->header_length_flags |=
513 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
517 static void pick_rx_handler(struct enic *enic)
519 struct rte_eth_dev *eth_dev;
521 /* Use the non-scatter, simplified RX handler if possible. */
522 eth_dev = enic->rte_dev;
523 if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
524 PMD_INIT_LOG(DEBUG, " use the non-scatter Rx handler");
525 eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
527 PMD_INIT_LOG(DEBUG, " use the normal Rx handler");
528 eth_dev->rx_pkt_burst = &enic_recv_pkts;
532 int enic_enable(struct enic *enic)
536 struct rte_eth_dev *eth_dev = enic->rte_dev;
537 uint64_t simple_tx_offloads;
539 eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
540 eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
542 /* vnic notification of link status has already been turned on in
543 * enic_dev_init() which is called during probe time. Here we are
544 * just turning on interrupt vector 0 if needed.
546 if (eth_dev->data->dev_conf.intr_conf.lsc)
547 vnic_dev_notify_set(enic->vdev, 0);
549 err = enic_rxq_intr_init(enic);
552 if (enic_clsf_init(enic))
553 dev_warning(enic, "Init of hash table for clsf failed."\
554 "Flow director feature will not work\n");
556 for (index = 0; index < enic->rq_count; index++) {
557 err = enic_alloc_rx_queue_mbufs(enic,
558 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
560 dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
563 err = enic_alloc_rx_queue_mbufs(enic,
564 &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
566 /* release the allocated mbufs for the sop rq*/
567 enic_rxmbuf_queue_release(enic,
568 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
570 dev_err(enic, "Failed to alloc data RX queue mbufs\n");
576 * Use the simple TX handler if possible. Only checksum offloads
577 * and vlan insertion are supported.
579 simple_tx_offloads = enic->tx_offload_capa &
580 (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
581 DEV_TX_OFFLOAD_VLAN_INSERT |
582 DEV_TX_OFFLOAD_IPV4_CKSUM |
583 DEV_TX_OFFLOAD_UDP_CKSUM |
584 DEV_TX_OFFLOAD_TCP_CKSUM);
585 if ((eth_dev->data->dev_conf.txmode.offloads &
586 ~simple_tx_offloads) == 0) {
587 PMD_INIT_LOG(DEBUG, " use the simple tx handler");
588 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
589 for (index = 0; index < enic->wq_count; index++)
590 enic_prep_wq_for_simple_tx(enic, index);
592 PMD_INIT_LOG(DEBUG, " use the default tx handler");
593 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
596 pick_rx_handler(enic);
598 for (index = 0; index < enic->wq_count; index++)
599 enic_start_wq(enic, index);
600 for (index = 0; index < enic->rq_count; index++)
601 enic_start_rq(enic, index);
603 vnic_dev_add_addr(enic->vdev, enic->mac_addr);
605 vnic_dev_enable_wait(enic->vdev);
607 /* Register and enable error interrupt */
608 rte_intr_callback_register(&(enic->pdev->intr_handle),
609 enic_intr_handler, (void *)enic->rte_dev);
611 rte_intr_enable(&(enic->pdev->intr_handle));
612 /* Unmask LSC interrupt */
613 vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
618 int enic_alloc_intr_resources(struct enic *enic)
623 dev_info(enic, "vNIC resources used: "\
624 "wq %d rq %d cq %d intr %d\n",
625 enic->wq_count, enic_vnic_rq_count(enic),
626 enic->cq_count, enic->intr_count);
628 for (i = 0; i < enic->intr_count; i++) {
629 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
631 enic_free_vnic_resources(enic);
638 void enic_free_rq(void *rxq)
640 struct vnic_rq *rq_sop, *rq_data;
646 rq_sop = (struct vnic_rq *)rxq;
647 enic = vnic_dev_priv(rq_sop->vdev);
648 rq_data = &enic->rq[rq_sop->data_queue_idx];
650 if (rq_sop->free_mbufs) {
651 struct rte_mbuf **mb;
654 mb = rq_sop->free_mbufs;
655 for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs;
656 i < ENIC_RX_BURST_MAX; i++)
657 rte_pktmbuf_free(mb[i]);
658 rte_free(rq_sop->free_mbufs);
659 rq_sop->free_mbufs = NULL;
660 rq_sop->num_free_mbufs = 0;
663 enic_rxmbuf_queue_release(enic, rq_sop);
665 enic_rxmbuf_queue_release(enic, rq_data);
667 rte_free(rq_sop->mbuf_ring);
669 rte_free(rq_data->mbuf_ring);
671 rq_sop->mbuf_ring = NULL;
672 rq_data->mbuf_ring = NULL;
674 vnic_rq_free(rq_sop);
676 vnic_rq_free(rq_data);
678 vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
684 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
686 struct rte_eth_dev *eth_dev = enic->rte_dev;
687 vnic_wq_enable(&enic->wq[queue_idx]);
688 eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
691 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
693 struct rte_eth_dev *eth_dev = enic->rte_dev;
696 ret = vnic_wq_disable(&enic->wq[queue_idx]);
700 eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
704 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
706 struct vnic_rq *rq_sop;
707 struct vnic_rq *rq_data;
708 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
709 rq_data = &enic->rq[rq_sop->data_queue_idx];
710 struct rte_eth_dev *eth_dev = enic->rte_dev;
712 if (rq_data->in_use) {
713 vnic_rq_enable(rq_data);
714 enic_initial_post_rx(enic, rq_data);
717 vnic_rq_enable(rq_sop);
718 enic_initial_post_rx(enic, rq_sop);
719 eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
722 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
724 int ret1 = 0, ret2 = 0;
725 struct rte_eth_dev *eth_dev = enic->rte_dev;
726 struct vnic_rq *rq_sop;
727 struct vnic_rq *rq_data;
728 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
729 rq_data = &enic->rq[rq_sop->data_queue_idx];
731 ret2 = vnic_rq_disable(rq_sop);
734 ret1 = vnic_rq_disable(rq_data);
741 eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
745 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
746 unsigned int socket_id, struct rte_mempool *mp,
747 uint16_t nb_desc, uint16_t free_thresh)
750 uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
751 uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
752 struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
753 struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
754 unsigned int mbuf_size, mbufs_per_pkt;
755 unsigned int nb_sop_desc, nb_data_desc;
756 uint16_t min_sop, max_sop, min_data, max_data;
757 uint32_t max_rx_pkt_len;
760 rq_sop->data_queue_idx = data_queue_idx;
762 rq_data->data_queue_idx = 0;
763 rq_sop->socket_id = socket_id;
765 rq_data->socket_id = socket_id;
768 rq_sop->rx_free_thresh = free_thresh;
769 rq_data->rx_free_thresh = free_thresh;
770 dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
773 mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
774 RTE_PKTMBUF_HEADROOM);
775 /* max_rx_pkt_len includes the ethernet header and CRC. */
776 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
778 if (enic->rte_dev->data->dev_conf.rxmode.offloads &
779 DEV_RX_OFFLOAD_SCATTER) {
780 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
781 /* ceil((max pkt len)/mbuf_size) */
782 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
784 dev_info(enic, "Scatter rx mode disabled\n");
786 if (max_rx_pkt_len > mbuf_size) {
787 dev_warning(enic, "The maximum Rx packet size (%u) is"
788 " larger than the mbuf size (%u), and"
789 " scatter is disabled. Larger packets will"
791 max_rx_pkt_len, mbuf_size);
795 if (mbufs_per_pkt > 1) {
796 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
797 rq_sop->data_queue_enable = 1;
800 * HW does not directly support rxmode.max_rx_pkt_len. HW always
801 * receives packet sizes up to the "max" MTU.
802 * If not using scatter, we can achieve the effect of dropping
803 * larger packets by reducing the size of posted buffers.
804 * See enic_alloc_rx_queue_mbufs().
807 enic_mtu_to_max_rx_pktlen(enic->max_mtu)) {
808 dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
809 " when scatter rx mode is in use.\n");
812 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
814 rq_sop->data_queue_enable = 0;
818 /* number of descriptors have to be a multiple of 32 */
819 nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK;
820 nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK;
822 rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
823 rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
825 if (mbufs_per_pkt > 1) {
826 min_sop = ENIC_RX_BURST_MAX;
827 max_sop = ((enic->config.rq_desc_count /
828 (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK);
829 min_data = min_sop * (mbufs_per_pkt - 1);
830 max_data = enic->config.rq_desc_count;
832 min_sop = ENIC_RX_BURST_MAX;
833 max_sop = enic->config.rq_desc_count;
838 if (nb_desc < (min_sop + min_data)) {
840 "Number of rx descs too low, adjusting to minimum\n");
841 nb_sop_desc = min_sop;
842 nb_data_desc = min_data;
843 } else if (nb_desc > (max_sop + max_data)) {
845 "Number of rx_descs too high, adjusting to maximum\n");
846 nb_sop_desc = max_sop;
847 nb_data_desc = max_data;
849 if (mbufs_per_pkt > 1) {
850 dev_info(enic, "For max packet size %u and mbuf size %u valid"
851 " rx descriptor range is %u to %u\n",
852 max_rx_pkt_len, mbuf_size, min_sop + min_data,
855 dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
856 nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
858 /* Allocate sop queue resources */
859 rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
860 nb_sop_desc, sizeof(struct rq_enet_desc));
862 dev_err(enic, "error in allocation of sop rq\n");
865 nb_sop_desc = rq_sop->ring.desc_count;
867 if (rq_data->in_use) {
868 /* Allocate data queue resources */
869 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
871 sizeof(struct rq_enet_desc));
873 dev_err(enic, "error in allocation of data rq\n");
874 goto err_free_rq_sop;
876 nb_data_desc = rq_data->ring.desc_count;
878 rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
879 socket_id, nb_sop_desc + nb_data_desc,
880 sizeof(struct cq_enet_rq_desc));
882 dev_err(enic, "error in allocation of cq for rq\n");
883 goto err_free_rq_data;
886 /* Allocate the mbuf rings */
887 rq_sop->mbuf_ring = (struct rte_mbuf **)
888 rte_zmalloc_socket("rq->mbuf_ring",
889 sizeof(struct rte_mbuf *) * nb_sop_desc,
890 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
891 if (rq_sop->mbuf_ring == NULL)
894 if (rq_data->in_use) {
895 rq_data->mbuf_ring = (struct rte_mbuf **)
896 rte_zmalloc_socket("rq->mbuf_ring",
897 sizeof(struct rte_mbuf *) * nb_data_desc,
898 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
899 if (rq_data->mbuf_ring == NULL)
900 goto err_free_sop_mbuf;
903 rq_sop->free_mbufs = (struct rte_mbuf **)
904 rte_zmalloc_socket("rq->free_mbufs",
905 sizeof(struct rte_mbuf *) *
907 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
908 if (rq_sop->free_mbufs == NULL)
909 goto err_free_data_mbuf;
910 rq_sop->num_free_mbufs = 0;
912 rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
917 rte_free(rq_data->mbuf_ring);
919 rte_free(rq_sop->mbuf_ring);
921 /* cleanup on error */
922 vnic_cq_free(&enic->cq[queue_idx]);
925 vnic_rq_free(rq_data);
927 vnic_rq_free(rq_sop);
932 void enic_free_wq(void *txq)
940 wq = (struct vnic_wq *)txq;
941 enic = vnic_dev_priv(wq->vdev);
942 rte_memzone_free(wq->cqmsg_rz);
944 vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
947 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
948 unsigned int socket_id, uint16_t nb_desc)
951 struct vnic_wq *wq = &enic->wq[queue_idx];
952 unsigned int cq_index = enic_cq_wq(enic, queue_idx);
956 wq->socket_id = socket_id;
958 * rte_eth_tx_queue_setup() checks min, max, and alignment. So just
959 * print an info message for diagnostics.
961 dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc);
963 /* Allocate queue resources */
964 err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
966 sizeof(struct wq_enet_desc));
968 dev_err(enic, "error in allocation of wq\n");
972 err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
974 sizeof(struct cq_enet_wq_desc));
977 dev_err(enic, "error in allocation of cq for wq\n");
980 /* setup up CQ message */
981 snprintf((char *)name, sizeof(name),
982 "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
985 wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
986 sizeof(uint32_t), SOCKET_ID_ANY,
987 RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
994 int enic_disable(struct enic *enic)
999 for (i = 0; i < enic->intr_count; i++) {
1000 vnic_intr_mask(&enic->intr[i]);
1001 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
1003 enic_rxq_intr_deinit(enic);
1004 rte_intr_disable(&enic->pdev->intr_handle);
1005 rte_intr_callback_unregister(&enic->pdev->intr_handle,
1007 (void *)enic->rte_dev);
1009 vnic_dev_disable(enic->vdev);
1011 enic_clsf_destroy(enic);
1013 if (!enic_is_sriov_vf(enic))
1014 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
1016 for (i = 0; i < enic->wq_count; i++) {
1017 err = vnic_wq_disable(&enic->wq[i]);
1021 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
1022 if (enic->rq[i].in_use) {
1023 err = vnic_rq_disable(&enic->rq[i]);
1029 /* If we were using interrupts, set the interrupt vector to -1
1030 * to disable interrupts. We are not disabling link notifcations,
1031 * though, as we want the polling of link status to continue working.
1033 if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
1034 vnic_dev_notify_set(enic->vdev, -1);
1036 vnic_dev_set_reset_flag(enic->vdev, 1);
1038 for (i = 0; i < enic->wq_count; i++)
1039 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
1041 for (i = 0; i < enic_vnic_rq_count(enic); i++)
1042 if (enic->rq[i].in_use)
1043 vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
1044 for (i = 0; i < enic->cq_count; i++)
1045 vnic_cq_clean(&enic->cq[i]);
1046 for (i = 0; i < enic->intr_count; i++)
1047 vnic_intr_clean(&enic->intr[i]);
1052 static int enic_dev_wait(struct vnic_dev *vdev,
1053 int (*start)(struct vnic_dev *, int),
1054 int (*finished)(struct vnic_dev *, int *),
1061 err = start(vdev, arg);
1065 /* Wait for func to complete...2 seconds max */
1066 for (i = 0; i < 2000; i++) {
1067 err = finished(vdev, &done);
1077 static int enic_dev_open(struct enic *enic)
1080 int flags = CMD_OPENF_IG_DESCCACHE;
1082 err = enic_dev_wait(enic->vdev, vnic_dev_open,
1083 vnic_dev_open_done, flags);
1085 dev_err(enic_get_dev(enic),
1086 "vNIC device open failed, err %d\n", err);
1091 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
1093 dma_addr_t rss_key_buf_pa;
1094 union vnic_rss_key *rss_key_buf_va = NULL;
1098 RTE_ASSERT(user_key != NULL);
1099 snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
1100 rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1101 &rss_key_buf_pa, name);
1102 if (!rss_key_buf_va)
1105 for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1106 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1108 err = enic_set_rss_key(enic,
1110 sizeof(union vnic_rss_key));
1112 /* Save for later queries */
1114 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1115 sizeof(union vnic_rss_key));
1117 enic_free_consistent(enic, sizeof(union vnic_rss_key),
1118 rss_key_buf_va, rss_key_buf_pa);
1123 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1125 dma_addr_t rss_cpu_buf_pa;
1126 union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1130 snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
1131 rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1132 &rss_cpu_buf_pa, name);
1133 if (!rss_cpu_buf_va)
1136 rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1138 err = enic_set_rss_cpu(enic,
1140 sizeof(union vnic_rss_cpu));
1142 enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1143 rss_cpu_buf_va, rss_cpu_buf_pa);
1145 /* Save for later queries */
1147 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1151 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
1152 u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
1154 const u8 tso_ipid_split_en = 0;
1157 err = enic_set_nic_cfg(enic,
1158 rss_default_cpu, rss_hash_type,
1159 rss_hash_bits, rss_base_cpu,
1160 rss_enable, tso_ipid_split_en,
1161 enic->ig_vlan_strip_en);
1166 /* Initialize RSS with defaults, called from dev_configure */
1167 int enic_init_rss_nic_cfg(struct enic *enic)
1169 static uint8_t default_rss_key[] = {
1170 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1171 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1172 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1173 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1175 struct rte_eth_rss_conf rss_conf;
1176 union vnic_rss_cpu rss_cpu;
1179 rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1181 * If setting key for the first time, and the user gives us none, then
1182 * push the default key to NIC.
1184 if (rss_conf.rss_key == NULL) {
1185 rss_conf.rss_key = default_rss_key;
1186 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1188 ret = enic_set_rss_conf(enic, &rss_conf);
1190 dev_err(enic, "Failed to configure RSS\n");
1193 if (enic->rss_enable) {
1194 /* If enabling RSS, use the default reta */
1195 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1196 rss_cpu.cpu[i / 4].b[i % 4] =
1197 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1199 ret = enic_set_rss_reta(enic, &rss_cpu);
1201 dev_err(enic, "Failed to set RSS indirection table\n");
1206 int enic_setup_finish(struct enic *enic)
1208 enic_init_soft_stats(enic);
1211 vnic_dev_packet_filter(enic->vdev,
1224 static int enic_rss_conf_valid(struct enic *enic,
1225 struct rte_eth_rss_conf *rss_conf)
1227 /* RSS is disabled per VIC settings. Ignore rss_conf. */
1228 if (enic->flow_type_rss_offloads == 0)
1230 if (rss_conf->rss_key != NULL &&
1231 rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1232 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1233 rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1236 if (rss_conf->rss_hf != 0 &&
1237 (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1238 dev_err(enic, "Given rss_hf contains none of the supported"
1245 /* Set hash type and key according to rss_conf */
1246 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1248 struct rte_eth_dev *eth_dev;
1254 RTE_ASSERT(rss_conf != NULL);
1255 ret = enic_rss_conf_valid(enic, rss_conf);
1257 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1261 eth_dev = enic->rte_dev;
1263 rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1264 if (enic->rq_count > 1 &&
1265 (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1268 if (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1269 ETH_RSS_NONFRAG_IPV4_OTHER))
1270 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1271 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1272 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1273 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
1274 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4;
1275 if (enic->udp_rss_weak) {
1277 * 'TCP' is not a typo. The "weak" version of
1278 * UDP RSS requires both the TCP and UDP bits
1279 * be set. It does enable TCP RSS as well.
1281 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1284 if (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_IPV6_EX |
1285 ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER))
1286 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1287 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX))
1288 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1289 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_UDP_EX)) {
1290 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6;
1291 if (enic->udp_rss_weak)
1292 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1299 /* Set the hash key if provided */
1300 if (rss_enable && rss_conf->rss_key) {
1301 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1303 dev_err(enic, "Failed to set RSS key\n");
1308 ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1309 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1312 enic->rss_hf = rss_hf;
1313 enic->rss_hash_type = rss_hash_type;
1314 enic->rss_enable = rss_enable;
1316 dev_err(enic, "Failed to update RSS configurations."
1317 " hash=0x%x\n", rss_hash_type);
1322 int enic_set_vlan_strip(struct enic *enic)
1325 * Unfortunately, VLAN strip on/off and RSS on/off are configured
1326 * together. So, re-do niccfg, preserving the current RSS settings.
1328 return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1329 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1333 void enic_add_packet_filter(struct enic *enic)
1335 /* Args -> directed, multicast, broadcast, promisc, allmulti */
1336 vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1337 enic->promisc, enic->allmulti);
1340 int enic_get_link_status(struct enic *enic)
1342 return vnic_dev_link_status(enic->vdev);
1345 static void enic_dev_deinit(struct enic *enic)
1347 struct rte_eth_dev *eth_dev = enic->rte_dev;
1349 /* stop link status checking */
1350 vnic_dev_notify_unset(enic->vdev);
1352 rte_free(eth_dev->data->mac_addrs);
1354 rte_free(enic->intr);
1360 int enic_set_vnic_res(struct enic *enic)
1362 struct rte_eth_dev *eth_dev = enic->rte_dev;
1364 unsigned int required_rq, required_wq, required_cq, required_intr;
1366 /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1367 required_rq = eth_dev->data->nb_rx_queues * 2;
1368 required_wq = eth_dev->data->nb_tx_queues;
1369 required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1370 required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1371 if (eth_dev->data->dev_conf.intr_conf.rxq) {
1372 required_intr += eth_dev->data->nb_rx_queues;
1375 if (enic->conf_rq_count < required_rq) {
1376 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1377 eth_dev->data->nb_rx_queues,
1378 required_rq, enic->conf_rq_count);
1381 if (enic->conf_wq_count < required_wq) {
1382 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1383 eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1387 if (enic->conf_cq_count < required_cq) {
1388 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1389 required_cq, enic->conf_cq_count);
1392 if (enic->conf_intr_count < required_intr) {
1393 dev_err(dev, "Not enough Interrupts to support Rx queue"
1394 " interrupts. Required:%u, Configured:%u\n",
1395 required_intr, enic->conf_intr_count);
1400 enic->rq_count = eth_dev->data->nb_rx_queues;
1401 enic->wq_count = eth_dev->data->nb_tx_queues;
1402 enic->cq_count = enic->rq_count + enic->wq_count;
1403 enic->intr_count = required_intr;
1409 /* Initialize the completion queue for an RQ */
1411 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1413 struct vnic_rq *sop_rq, *data_rq;
1414 unsigned int cq_idx;
1417 sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1418 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1421 vnic_cq_clean(&enic->cq[cq_idx]);
1422 vnic_cq_init(&enic->cq[cq_idx],
1423 0 /* flow_control_enable */,
1424 1 /* color_enable */,
1427 1 /* cq_tail_color */,
1428 0 /* interrupt_enable */,
1429 1 /* cq_entry_enable */,
1430 0 /* cq_message_enable */,
1431 0 /* interrupt offset */,
1432 0 /* cq_message_addr */);
1435 vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1436 enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1437 sop_rq->ring.desc_count - 1, 1, 0);
1438 if (data_rq->in_use) {
1439 vnic_rq_init_start(data_rq,
1441 enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1442 data_rq->ring.desc_count - 1, 1, 0);
1445 rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1449 if (data_rq->in_use) {
1450 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1452 enic_rxmbuf_queue_release(enic, sop_rq);
1460 /* The Cisco NIC can send and receive packets up to a max packet size
1461 * determined by the NIC type and firmware. There is also an MTU
1462 * configured into the NIC via the CIMC/UCSM management interface
1463 * which can be overridden by this function (up to the max packet size).
1464 * Depending on the network setup, doing so may cause packet drops
1465 * and unexpected behavior.
1467 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1469 unsigned int rq_idx;
1472 uint16_t old_mtu; /* previous setting */
1473 uint16_t config_mtu; /* Value configured into NIC via CIMC/UCSM */
1474 struct rte_eth_dev *eth_dev = enic->rte_dev;
1476 old_mtu = eth_dev->data->mtu;
1477 config_mtu = enic->config.mtu;
1479 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1480 return -E_RTE_SECONDARY;
1482 if (new_mtu > enic->max_mtu) {
1484 "MTU not updated: requested (%u) greater than max (%u)\n",
1485 new_mtu, enic->max_mtu);
1488 if (new_mtu < ENIC_MIN_MTU) {
1490 "MTU not updated: requested (%u) less than min (%u)\n",
1491 new_mtu, ENIC_MIN_MTU);
1494 if (new_mtu > config_mtu)
1496 "MTU (%u) is greater than value configured in NIC (%u)\n",
1497 new_mtu, config_mtu);
1499 /* Update the MTU and maximum packet length */
1500 eth_dev->data->mtu = new_mtu;
1501 eth_dev->data->dev_conf.rxmode.max_rx_pkt_len =
1502 enic_mtu_to_max_rx_pktlen(new_mtu);
1505 * If the device has not started (enic_enable), nothing to do.
1506 * Later, enic_enable() will set up RQs reflecting the new maximum
1509 if (!eth_dev->data->dev_started)
1513 * The device has started, re-do RQs on the fly. In the process, we
1514 * pick up the new maximum packet length.
1516 * Some applications rely on the ability to change MTU without stopping
1517 * the device. So keep this behavior for now.
1519 rte_spinlock_lock(&enic->mtu_lock);
1521 /* Stop traffic on all RQs */
1522 for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1523 rq = &enic->rq[rq_idx];
1524 if (rq->is_sop && rq->in_use) {
1525 rc = enic_stop_rq(enic,
1526 enic_sop_rq_idx_to_rte_idx(rq_idx));
1528 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1534 /* replace Rx function with a no-op to avoid getting stale pkts */
1535 eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1538 /* Allow time for threads to exit the real Rx function. */
1541 /* now it is safe to reconfigure the RQs */
1544 /* free and reallocate RQs with the new MTU */
1545 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1546 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1551 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1552 rq->tot_nb_desc, rq->rx_free_thresh);
1555 "Fatal MTU alloc error- No traffic will pass\n");
1559 rc = enic_reinit_rq(enic, rq_idx);
1562 "Fatal MTU RQ reinit- No traffic will pass\n");
1567 /* put back the real receive function */
1569 pick_rx_handler(enic);
1572 /* restart Rx traffic */
1573 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1574 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1575 if (rq->is_sop && rq->in_use)
1576 enic_start_rq(enic, rq_idx);
1580 dev_info(enic, "MTU changed from %u to %u\n", old_mtu, new_mtu);
1581 rte_spinlock_unlock(&enic->mtu_lock);
1585 static int enic_dev_init(struct enic *enic)
1588 struct rte_eth_dev *eth_dev = enic->rte_dev;
1590 vnic_dev_intr_coal_timer_info_default(enic->vdev);
1592 /* Get vNIC configuration
1594 err = enic_get_vnic_config(enic);
1596 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1600 /* Get available resource counts */
1601 enic_get_res_counts(enic);
1602 if (enic->conf_rq_count == 1) {
1603 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1604 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1605 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1608 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1609 enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1610 enic->conf_cq_count, 8);
1611 enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1612 enic->conf_intr_count, 8);
1613 enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1614 enic->conf_rq_count, 8);
1615 enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1616 enic->conf_wq_count, 8);
1617 if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1618 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1621 if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1622 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1625 if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1626 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1629 if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1630 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1634 /* Get the supported filters */
1635 enic_fdir_info(enic);
1637 eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr", ETH_ALEN
1638 * ENIC_MAX_MAC_ADDR, 0);
1639 if (!eth_dev->data->mac_addrs) {
1640 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1643 ether_addr_copy((struct ether_addr *) enic->mac_addr,
1644 eth_dev->data->mac_addrs);
1646 vnic_dev_set_reset_flag(enic->vdev, 0);
1648 LIST_INIT(&enic->flows);
1649 rte_spinlock_init(&enic->flows_lock);
1650 enic->max_flow_counter = -1;
1652 /* set up link status checking */
1653 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1655 enic->overlay_offload = false;
1656 if (enic->disable_overlay && enic->vxlan) {
1658 * Explicitly disable overlay offload as the setting is
1659 * sticky, and resetting vNIC does not disable it.
1661 if (vnic_dev_overlay_offload_ctrl(enic->vdev,
1662 OVERLAY_FEATURE_VXLAN,
1663 OVERLAY_OFFLOAD_DISABLE)) {
1664 dev_err(enic, "failed to disable overlay offload\n");
1666 dev_info(enic, "Overlay offload is disabled\n");
1669 if (!enic->disable_overlay && enic->vxlan &&
1670 /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1671 vnic_dev_overlay_offload_ctrl(enic->vdev,
1672 OVERLAY_FEATURE_VXLAN,
1673 OVERLAY_OFFLOAD_ENABLE) == 0) {
1674 enic->tx_offload_capa |=
1675 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1676 DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
1677 DEV_TX_OFFLOAD_VXLAN_TNL_TSO;
1679 * Do not add PKT_TX_OUTER_{IPV4,IPV6} as they are not
1680 * 'offload' flags (i.e. not part of PKT_TX_OFFLOAD_MASK).
1682 enic->tx_offload_mask |=
1683 PKT_TX_OUTER_IP_CKSUM |
1685 enic->overlay_offload = true;
1686 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT;
1687 dev_info(enic, "Overlay offload is enabled\n");
1689 * Reset the vxlan port to the default, as the NIC firmware
1690 * does not reset it automatically and keeps the old setting.
1692 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1693 OVERLAY_CFG_VXLAN_PORT_UPDATE,
1694 ENIC_DEFAULT_VXLAN_PORT)) {
1695 dev_err(enic, "failed to update vxlan port\n");
1704 int enic_probe(struct enic *enic)
1706 struct rte_pci_device *pdev = enic->pdev;
1709 dev_debug(enic, " Initializing ENIC PMD\n");
1711 /* if this is a secondary process the hardware is already initialized */
1712 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1715 enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1716 enic->bar0.len = pdev->mem_resource[0].len;
1718 /* Register vNIC device */
1719 enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1721 dev_err(enic, "vNIC registration failed, aborting\n");
1725 LIST_INIT(&enic->memzone_list);
1726 rte_spinlock_init(&enic->memzone_list_lock);
1728 vnic_register_cbacks(enic->vdev,
1729 enic_alloc_consistent,
1730 enic_free_consistent);
1733 * Allocate the consistent memory for stats and counters upfront so
1734 * both primary and secondary processes can dump stats.
1736 err = vnic_dev_alloc_stats_mem(enic->vdev);
1738 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1739 goto err_out_unregister;
1741 err = vnic_dev_alloc_counter_mem(enic->vdev);
1743 dev_err(enic, "Failed to allocate counter memory, aborting\n");
1744 goto err_out_unregister;
1747 /* Issue device open to get device in known state */
1748 err = enic_dev_open(enic);
1750 dev_err(enic, "vNIC dev open failed, aborting\n");
1751 goto err_out_unregister;
1754 /* Set ingress vlan rewrite mode before vnic initialization */
1755 dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n",
1756 enic->ig_vlan_rewrite_mode);
1757 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1758 enic->ig_vlan_rewrite_mode);
1761 "Failed to set ingress vlan rewrite mode, aborting.\n");
1762 goto err_out_dev_close;
1765 /* Issue device init to initialize the vnic-to-switch link.
1766 * We'll start with carrier off and wait for link UP
1767 * notification later to turn on carrier. We don't need
1768 * to wait here for the vnic-to-switch link initialization
1769 * to complete; link UP notification is the indication that
1770 * the process is complete.
1773 err = vnic_dev_init(enic->vdev, 0);
1775 dev_err(enic, "vNIC dev init failed, aborting\n");
1776 goto err_out_dev_close;
1779 err = enic_dev_init(enic);
1781 dev_err(enic, "Device initialization failed, aborting\n");
1782 goto err_out_dev_close;
1788 vnic_dev_close(enic->vdev);
1790 vnic_dev_unregister(enic->vdev);
1795 void enic_remove(struct enic *enic)
1797 enic_dev_deinit(enic);
1798 vnic_dev_close(enic->vdev);
1799 vnic_dev_unregister(enic->vdev);