1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
14 #include <rte_bus_pci.h>
15 #include <rte_memzone.h>
16 #include <rte_malloc.h>
18 #include <rte_string_fns.h>
19 #include <rte_ethdev_driver.h>
21 #include "enic_compat.h"
23 #include "wq_enet_desc.h"
24 #include "rq_enet_desc.h"
25 #include "cq_enet_desc.h"
26 #include "vnic_enet.h"
31 #include "vnic_intr.h"
34 static inline int enic_is_sriov_vf(struct enic *enic)
36 return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
39 static int is_zero_addr(uint8_t *addr)
41 return !(addr[0] | addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
44 static int is_mcast_addr(uint8_t *addr)
49 static int is_eth_addr_valid(uint8_t *addr)
51 return !is_mcast_addr(addr) && !is_zero_addr(addr);
55 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
59 if (!rq || !rq->mbuf_ring) {
60 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
64 for (i = 0; i < rq->ring.desc_count; i++) {
65 if (rq->mbuf_ring[i]) {
66 rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
67 rq->mbuf_ring[i] = NULL;
72 static void enic_free_wq_buf(struct rte_mbuf **buf)
74 struct rte_mbuf *mbuf = *buf;
76 rte_pktmbuf_free_seg(mbuf);
80 static void enic_log_q_error(struct enic *enic)
85 for (i = 0; i < enic->wq_count; i++) {
86 error_status = vnic_wq_error_status(&enic->wq[i]);
88 dev_err(enic, "WQ[%d] error_status %d\n", i,
92 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
93 if (!enic->rq[i].in_use)
95 error_status = vnic_rq_error_status(&enic->rq[i]);
97 dev_err(enic, "RQ[%d] error_status %d\n", i,
102 static void enic_clear_soft_stats(struct enic *enic)
104 struct enic_soft_stats *soft_stats = &enic->soft_stats;
105 rte_atomic64_clear(&soft_stats->rx_nombuf);
106 rte_atomic64_clear(&soft_stats->rx_packet_errors);
107 rte_atomic64_clear(&soft_stats->tx_oversized);
110 static void enic_init_soft_stats(struct enic *enic)
112 struct enic_soft_stats *soft_stats = &enic->soft_stats;
113 rte_atomic64_init(&soft_stats->rx_nombuf);
114 rte_atomic64_init(&soft_stats->rx_packet_errors);
115 rte_atomic64_init(&soft_stats->tx_oversized);
116 enic_clear_soft_stats(enic);
119 void enic_dev_stats_clear(struct enic *enic)
121 if (vnic_dev_stats_clear(enic->vdev))
122 dev_err(enic, "Error in clearing stats\n");
123 enic_clear_soft_stats(enic);
126 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
128 struct vnic_stats *stats;
129 struct enic_soft_stats *soft_stats = &enic->soft_stats;
130 int64_t rx_truncated;
131 uint64_t rx_packet_errors;
132 int ret = vnic_dev_stats_dump(enic->vdev, &stats);
135 dev_err(enic, "Error in getting stats\n");
139 /* The number of truncated packets can only be calculated by
140 * subtracting a hardware counter from error packets received by
141 * the driver. Note: this causes transient inaccuracies in the
142 * ipackets count. Also, the length of truncated packets are
143 * counted in ibytes even though truncated packets are dropped
144 * which can make ibytes be slightly higher than it should be.
146 rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
147 rx_truncated = rx_packet_errors - stats->rx.rx_errors;
149 r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
150 r_stats->opackets = stats->tx.tx_frames_ok;
152 r_stats->ibytes = stats->rx.rx_bytes_ok;
153 r_stats->obytes = stats->tx.tx_bytes_ok;
155 r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
156 r_stats->oerrors = stats->tx.tx_errors
157 + rte_atomic64_read(&soft_stats->tx_oversized);
159 r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
161 r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
165 int enic_del_mac_address(struct enic *enic, int mac_index)
167 struct rte_eth_dev *eth_dev = enic->rte_dev;
168 uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
170 return vnic_dev_del_addr(enic->vdev, mac_addr);
173 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
177 if (!is_eth_addr_valid(mac_addr)) {
178 dev_err(enic, "invalid mac address\n");
182 err = vnic_dev_add_addr(enic->vdev, mac_addr);
184 dev_err(enic, "add mac addr failed\n");
189 enic_free_rq_buf(struct rte_mbuf **mbuf)
194 rte_pktmbuf_free(*mbuf);
198 void enic_init_vnic_resources(struct enic *enic)
200 unsigned int error_interrupt_enable = 1;
201 unsigned int error_interrupt_offset = 0;
202 unsigned int rxq_interrupt_enable = 0;
203 unsigned int rxq_interrupt_offset = ENICPMD_RXQ_INTR_OFFSET;
204 unsigned int index = 0;
206 struct vnic_rq *data_rq;
208 if (enic->rte_dev->data->dev_conf.intr_conf.rxq)
209 rxq_interrupt_enable = 1;
211 for (index = 0; index < enic->rq_count; index++) {
212 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
214 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
216 error_interrupt_enable,
217 error_interrupt_offset);
219 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
221 vnic_rq_init(data_rq,
223 error_interrupt_enable,
224 error_interrupt_offset);
226 vnic_cq_init(&enic->cq[cq_idx],
227 0 /* flow_control_enable */,
228 1 /* color_enable */,
231 1 /* cq_tail_color */,
232 rxq_interrupt_enable,
233 1 /* cq_entry_enable */,
234 0 /* cq_message_enable */,
235 rxq_interrupt_offset,
236 0 /* cq_message_addr */);
237 if (rxq_interrupt_enable)
238 rxq_interrupt_offset++;
241 for (index = 0; index < enic->wq_count; index++) {
242 vnic_wq_init(&enic->wq[index],
243 enic_cq_wq(enic, index),
244 error_interrupt_enable,
245 error_interrupt_offset);
246 /* Compute unsupported ol flags for enic_prep_pkts() */
247 enic->wq[index].tx_offload_notsup_mask =
248 PKT_TX_OFFLOAD_MASK ^ enic->tx_offload_mask;
250 cq_idx = enic_cq_wq(enic, index);
251 vnic_cq_init(&enic->cq[cq_idx],
252 0 /* flow_control_enable */,
253 1 /* color_enable */,
256 1 /* cq_tail_color */,
257 0 /* interrupt_enable */,
258 0 /* cq_entry_enable */,
259 1 /* cq_message_enable */,
260 0 /* interrupt offset */,
261 (u64)enic->wq[index].cqmsg_rz->iova);
264 for (index = 0; index < enic->intr_count; index++) {
265 vnic_intr_init(&enic->intr[index],
266 enic->config.intr_timer_usec,
267 enic->config.intr_timer_type,
268 /*mask_on_assertion*/1);
274 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
277 struct rq_enet_desc *rqd = rq->ring.descs;
280 uint32_t max_rx_pkt_len;
286 dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
287 rq->ring.desc_count);
290 * If *not* using scatter and the mbuf size is greater than the
291 * requested max packet size (max_rx_pkt_len), then reduce the
292 * posted buffer size to max_rx_pkt_len. HW still receives packets
293 * larger than max_rx_pkt_len, but they will be truncated, which we
294 * drop in the rx handler. Not ideal, but better than returning
295 * large packets when the user is not expecting them.
297 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
298 rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
299 if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
300 rq_buf_len = max_rx_pkt_len;
301 for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
302 mb = rte_mbuf_raw_alloc(rq->mp);
304 dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
305 (unsigned)rq->index);
309 mb->data_off = RTE_PKTMBUF_HEADROOM;
310 dma_addr = (dma_addr_t)(mb->buf_iova
311 + RTE_PKTMBUF_HEADROOM);
312 rq_enet_desc_enc(rqd, dma_addr,
313 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
314 : RQ_ENET_TYPE_NOT_SOP),
316 rq->mbuf_ring[i] = mb;
319 * Do not post the buffers to the NIC until we enable the RQ via
322 rq->need_initial_post = true;
323 /* Initialize fetch index while RQ is disabled */
324 iowrite32(0, &rq->ctrl->fetch_index);
329 * Post the Rx buffers for the first time. enic_alloc_rx_queue_mbufs() has
330 * allocated the buffers and filled the RQ descriptor ring. Just need to push
331 * the post index to the NIC.
334 enic_initial_post_rx(struct enic *enic, struct vnic_rq *rq)
336 if (!rq->in_use || !rq->need_initial_post)
339 /* make sure all prior writes are complete before doing the PIO write */
342 /* Post all but the last buffer to VIC. */
343 rq->posted_index = rq->ring.desc_count - 1;
347 dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
348 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
349 iowrite32(rq->posted_index, &rq->ctrl->posted_index);
351 rq->need_initial_post = false;
355 enic_alloc_consistent(void *priv, size_t size,
356 dma_addr_t *dma_handle, u8 *name)
359 const struct rte_memzone *rz;
361 struct enic *enic = (struct enic *)priv;
362 struct enic_memzone_entry *mze;
364 rz = rte_memzone_reserve_aligned((const char *)name, size,
365 SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
367 pr_err("%s : Failed to allocate memory requested for %s\n",
373 *dma_handle = (dma_addr_t)rz->iova;
375 mze = rte_malloc("enic memzone entry",
376 sizeof(struct enic_memzone_entry), 0);
379 pr_err("%s : Failed to allocate memory for memzone list\n",
381 rte_memzone_free(rz);
387 rte_spinlock_lock(&enic->memzone_list_lock);
388 LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
389 rte_spinlock_unlock(&enic->memzone_list_lock);
395 enic_free_consistent(void *priv,
396 __rte_unused size_t size,
398 dma_addr_t dma_handle)
400 struct enic_memzone_entry *mze;
401 struct enic *enic = (struct enic *)priv;
403 rte_spinlock_lock(&enic->memzone_list_lock);
404 LIST_FOREACH(mze, &enic->memzone_list, entries) {
405 if (mze->rz->addr == vaddr &&
406 mze->rz->iova == dma_handle)
410 rte_spinlock_unlock(&enic->memzone_list_lock);
412 "Tried to free memory, but couldn't find it in the memzone list\n");
415 LIST_REMOVE(mze, entries);
416 rte_spinlock_unlock(&enic->memzone_list_lock);
417 rte_memzone_free(mze->rz);
421 int enic_link_update(struct enic *enic)
423 struct rte_eth_dev *eth_dev = enic->rte_dev;
424 struct rte_eth_link link;
426 memset(&link, 0, sizeof(link));
427 link.link_status = enic_get_link_status(enic);
428 link.link_duplex = ETH_LINK_FULL_DUPLEX;
429 link.link_speed = vnic_dev_port_speed(enic->vdev);
431 return rte_eth_linkstatus_set(eth_dev, &link);
435 enic_intr_handler(void *arg)
437 struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
438 struct enic *enic = pmd_priv(dev);
440 vnic_intr_return_all_credits(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
442 enic_link_update(enic);
443 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
444 enic_log_q_error(enic);
447 static int enic_rxq_intr_init(struct enic *enic)
449 struct rte_intr_handle *intr_handle;
450 uint32_t rxq_intr_count, i;
453 intr_handle = enic->rte_dev->intr_handle;
454 if (!enic->rte_dev->data->dev_conf.intr_conf.rxq)
457 * Rx queue interrupts only work when we have MSI-X interrupts,
458 * one per queue. Sharing one interrupt is technically
459 * possible with VIC, but it is not worth the complications it brings.
461 if (!rte_intr_cap_multiple(intr_handle)) {
462 dev_err(enic, "Rx queue interrupts require MSI-X interrupts"
463 " (vfio-pci driver)\n");
466 rxq_intr_count = enic->intr_count - ENICPMD_RXQ_INTR_OFFSET;
467 err = rte_intr_efd_enable(intr_handle, rxq_intr_count);
469 dev_err(enic, "Failed to enable event fds for Rx queue"
473 intr_handle->intr_vec = rte_zmalloc("enic_intr_vec",
474 rxq_intr_count * sizeof(int), 0);
475 if (intr_handle->intr_vec == NULL) {
476 dev_err(enic, "Failed to allocate intr_vec\n");
479 for (i = 0; i < rxq_intr_count; i++)
480 intr_handle->intr_vec[i] = i + ENICPMD_RXQ_INTR_OFFSET;
484 static void enic_rxq_intr_deinit(struct enic *enic)
486 struct rte_intr_handle *intr_handle;
488 intr_handle = enic->rte_dev->intr_handle;
489 rte_intr_efd_disable(intr_handle);
490 if (intr_handle->intr_vec != NULL) {
491 rte_free(intr_handle->intr_vec);
492 intr_handle->intr_vec = NULL;
496 static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
498 struct wq_enet_desc *desc;
503 * Fill WQ descriptor fields that never change. Every descriptor is
504 * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
505 * descriptors (i.e. request one completion update every 32 packets).
507 wq = &enic->wq[queue_idx];
508 desc = (struct wq_enet_desc *)wq->ring.descs;
509 for (i = 0; i < wq->ring.desc_count; i++, desc++) {
510 desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
511 if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
512 desc->header_length_flags |=
513 (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
517 static void pick_rx_handler(struct enic *enic)
519 struct rte_eth_dev *eth_dev;
521 /* Use the non-scatter, simplified RX handler if possible. */
522 eth_dev = enic->rte_dev;
523 if (enic->rq_count > 0 && enic->rq[0].data_queue_enable == 0) {
524 PMD_INIT_LOG(DEBUG, " use the non-scatter Rx handler");
525 eth_dev->rx_pkt_burst = &enic_noscatter_recv_pkts;
527 PMD_INIT_LOG(DEBUG, " use the normal Rx handler");
528 eth_dev->rx_pkt_burst = &enic_recv_pkts;
532 int enic_enable(struct enic *enic)
536 struct rte_eth_dev *eth_dev = enic->rte_dev;
538 eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
539 eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
541 /* vnic notification of link status has already been turned on in
542 * enic_dev_init() which is called during probe time. Here we are
543 * just turning on interrupt vector 0 if needed.
545 if (eth_dev->data->dev_conf.intr_conf.lsc)
546 vnic_dev_notify_set(enic->vdev, 0);
548 err = enic_rxq_intr_init(enic);
551 if (enic_clsf_init(enic))
552 dev_warning(enic, "Init of hash table for clsf failed."\
553 "Flow director feature will not work\n");
555 for (index = 0; index < enic->rq_count; index++) {
556 err = enic_alloc_rx_queue_mbufs(enic,
557 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
559 dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
562 err = enic_alloc_rx_queue_mbufs(enic,
563 &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
565 /* release the allocated mbufs for the sop rq*/
566 enic_rxmbuf_queue_release(enic,
567 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
569 dev_err(enic, "Failed to alloc data RX queue mbufs\n");
575 * Use the simple TX handler if possible. All offloads must be
578 if (eth_dev->data->dev_conf.txmode.offloads == 0) {
579 PMD_INIT_LOG(DEBUG, " use the simple tx handler");
580 eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
581 for (index = 0; index < enic->wq_count; index++)
582 enic_prep_wq_for_simple_tx(enic, index);
584 PMD_INIT_LOG(DEBUG, " use the default tx handler");
585 eth_dev->tx_pkt_burst = &enic_xmit_pkts;
588 pick_rx_handler(enic);
590 for (index = 0; index < enic->wq_count; index++)
591 enic_start_wq(enic, index);
592 for (index = 0; index < enic->rq_count; index++)
593 enic_start_rq(enic, index);
595 vnic_dev_add_addr(enic->vdev, enic->mac_addr);
597 vnic_dev_enable_wait(enic->vdev);
599 /* Register and enable error interrupt */
600 rte_intr_callback_register(&(enic->pdev->intr_handle),
601 enic_intr_handler, (void *)enic->rte_dev);
603 rte_intr_enable(&(enic->pdev->intr_handle));
604 /* Unmask LSC interrupt */
605 vnic_intr_unmask(&enic->intr[ENICPMD_LSC_INTR_OFFSET]);
610 int enic_alloc_intr_resources(struct enic *enic)
615 dev_info(enic, "vNIC resources used: "\
616 "wq %d rq %d cq %d intr %d\n",
617 enic->wq_count, enic_vnic_rq_count(enic),
618 enic->cq_count, enic->intr_count);
620 for (i = 0; i < enic->intr_count; i++) {
621 err = vnic_intr_alloc(enic->vdev, &enic->intr[i], i);
623 enic_free_vnic_resources(enic);
630 void enic_free_rq(void *rxq)
632 struct vnic_rq *rq_sop, *rq_data;
638 rq_sop = (struct vnic_rq *)rxq;
639 enic = vnic_dev_priv(rq_sop->vdev);
640 rq_data = &enic->rq[rq_sop->data_queue_idx];
642 if (rq_sop->free_mbufs) {
643 struct rte_mbuf **mb;
646 mb = rq_sop->free_mbufs;
647 for (i = ENIC_RX_BURST_MAX - rq_sop->num_free_mbufs;
648 i < ENIC_RX_BURST_MAX; i++)
649 rte_pktmbuf_free(mb[i]);
650 rte_free(rq_sop->free_mbufs);
651 rq_sop->free_mbufs = NULL;
652 rq_sop->num_free_mbufs = 0;
655 enic_rxmbuf_queue_release(enic, rq_sop);
657 enic_rxmbuf_queue_release(enic, rq_data);
659 rte_free(rq_sop->mbuf_ring);
661 rte_free(rq_data->mbuf_ring);
663 rq_sop->mbuf_ring = NULL;
664 rq_data->mbuf_ring = NULL;
666 vnic_rq_free(rq_sop);
668 vnic_rq_free(rq_data);
670 vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
676 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
678 struct rte_eth_dev *eth_dev = enic->rte_dev;
679 vnic_wq_enable(&enic->wq[queue_idx]);
680 eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
683 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
685 struct rte_eth_dev *eth_dev = enic->rte_dev;
688 ret = vnic_wq_disable(&enic->wq[queue_idx]);
692 eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
696 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
698 struct vnic_rq *rq_sop;
699 struct vnic_rq *rq_data;
700 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
701 rq_data = &enic->rq[rq_sop->data_queue_idx];
702 struct rte_eth_dev *eth_dev = enic->rte_dev;
704 if (rq_data->in_use) {
705 vnic_rq_enable(rq_data);
706 enic_initial_post_rx(enic, rq_data);
709 vnic_rq_enable(rq_sop);
710 enic_initial_post_rx(enic, rq_sop);
711 eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
714 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
716 int ret1 = 0, ret2 = 0;
717 struct rte_eth_dev *eth_dev = enic->rte_dev;
718 struct vnic_rq *rq_sop;
719 struct vnic_rq *rq_data;
720 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
721 rq_data = &enic->rq[rq_sop->data_queue_idx];
723 ret2 = vnic_rq_disable(rq_sop);
726 ret1 = vnic_rq_disable(rq_data);
733 eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
737 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
738 unsigned int socket_id, struct rte_mempool *mp,
739 uint16_t nb_desc, uint16_t free_thresh)
742 uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
743 uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
744 struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
745 struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
746 unsigned int mbuf_size, mbufs_per_pkt;
747 unsigned int nb_sop_desc, nb_data_desc;
748 uint16_t min_sop, max_sop, min_data, max_data;
749 uint32_t max_rx_pkt_len;
752 rq_sop->data_queue_idx = data_queue_idx;
754 rq_data->data_queue_idx = 0;
755 rq_sop->socket_id = socket_id;
757 rq_data->socket_id = socket_id;
760 rq_sop->rx_free_thresh = free_thresh;
761 rq_data->rx_free_thresh = free_thresh;
762 dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
765 mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
766 RTE_PKTMBUF_HEADROOM);
767 /* max_rx_pkt_len includes the ethernet header and CRC. */
768 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
770 if (enic->rte_dev->data->dev_conf.rxmode.offloads &
771 DEV_RX_OFFLOAD_SCATTER) {
772 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
773 /* ceil((max pkt len)/mbuf_size) */
774 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
776 dev_info(enic, "Scatter rx mode disabled\n");
778 if (max_rx_pkt_len > mbuf_size) {
779 dev_warning(enic, "The maximum Rx packet size (%u) is"
780 " larger than the mbuf size (%u), and"
781 " scatter is disabled. Larger packets will"
783 max_rx_pkt_len, mbuf_size);
787 if (mbufs_per_pkt > 1) {
788 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
789 rq_sop->data_queue_enable = 1;
792 * HW does not directly support rxmode.max_rx_pkt_len. HW always
793 * receives packet sizes up to the "max" MTU.
794 * If not using scatter, we can achieve the effect of dropping
795 * larger packets by reducing the size of posted buffers.
796 * See enic_alloc_rx_queue_mbufs().
799 enic_mtu_to_max_rx_pktlen(enic->max_mtu)) {
800 dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
801 " when scatter rx mode is in use.\n");
804 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
806 rq_sop->data_queue_enable = 0;
810 /* number of descriptors have to be a multiple of 32 */
811 nb_sop_desc = (nb_desc / mbufs_per_pkt) & ENIC_ALIGN_DESCS_MASK;
812 nb_data_desc = (nb_desc - nb_sop_desc) & ENIC_ALIGN_DESCS_MASK;
814 rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
815 rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
817 if (mbufs_per_pkt > 1) {
818 min_sop = ENIC_RX_BURST_MAX;
819 max_sop = ((enic->config.rq_desc_count /
820 (mbufs_per_pkt - 1)) & ENIC_ALIGN_DESCS_MASK);
821 min_data = min_sop * (mbufs_per_pkt - 1);
822 max_data = enic->config.rq_desc_count;
824 min_sop = ENIC_RX_BURST_MAX;
825 max_sop = enic->config.rq_desc_count;
830 if (nb_desc < (min_sop + min_data)) {
832 "Number of rx descs too low, adjusting to minimum\n");
833 nb_sop_desc = min_sop;
834 nb_data_desc = min_data;
835 } else if (nb_desc > (max_sop + max_data)) {
837 "Number of rx_descs too high, adjusting to maximum\n");
838 nb_sop_desc = max_sop;
839 nb_data_desc = max_data;
841 if (mbufs_per_pkt > 1) {
842 dev_info(enic, "For max packet size %u and mbuf size %u valid"
843 " rx descriptor range is %u to %u\n",
844 max_rx_pkt_len, mbuf_size, min_sop + min_data,
847 dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
848 nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
850 /* Allocate sop queue resources */
851 rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
852 nb_sop_desc, sizeof(struct rq_enet_desc));
854 dev_err(enic, "error in allocation of sop rq\n");
857 nb_sop_desc = rq_sop->ring.desc_count;
859 if (rq_data->in_use) {
860 /* Allocate data queue resources */
861 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
863 sizeof(struct rq_enet_desc));
865 dev_err(enic, "error in allocation of data rq\n");
866 goto err_free_rq_sop;
868 nb_data_desc = rq_data->ring.desc_count;
870 rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
871 socket_id, nb_sop_desc + nb_data_desc,
872 sizeof(struct cq_enet_rq_desc));
874 dev_err(enic, "error in allocation of cq for rq\n");
875 goto err_free_rq_data;
878 /* Allocate the mbuf rings */
879 rq_sop->mbuf_ring = (struct rte_mbuf **)
880 rte_zmalloc_socket("rq->mbuf_ring",
881 sizeof(struct rte_mbuf *) * nb_sop_desc,
882 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
883 if (rq_sop->mbuf_ring == NULL)
886 if (rq_data->in_use) {
887 rq_data->mbuf_ring = (struct rte_mbuf **)
888 rte_zmalloc_socket("rq->mbuf_ring",
889 sizeof(struct rte_mbuf *) * nb_data_desc,
890 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
891 if (rq_data->mbuf_ring == NULL)
892 goto err_free_sop_mbuf;
895 rq_sop->free_mbufs = (struct rte_mbuf **)
896 rte_zmalloc_socket("rq->free_mbufs",
897 sizeof(struct rte_mbuf *) *
899 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
900 if (rq_sop->free_mbufs == NULL)
901 goto err_free_data_mbuf;
902 rq_sop->num_free_mbufs = 0;
904 rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
909 rte_free(rq_data->mbuf_ring);
911 rte_free(rq_sop->mbuf_ring);
913 /* cleanup on error */
914 vnic_cq_free(&enic->cq[queue_idx]);
917 vnic_rq_free(rq_data);
919 vnic_rq_free(rq_sop);
924 void enic_free_wq(void *txq)
932 wq = (struct vnic_wq *)txq;
933 enic = vnic_dev_priv(wq->vdev);
934 rte_memzone_free(wq->cqmsg_rz);
936 vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
939 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
940 unsigned int socket_id, uint16_t nb_desc)
943 struct vnic_wq *wq = &enic->wq[queue_idx];
944 unsigned int cq_index = enic_cq_wq(enic, queue_idx);
948 wq->socket_id = socket_id;
950 * rte_eth_tx_queue_setup() checks min, max, and alignment. So just
951 * print an info message for diagnostics.
953 dev_info(enic, "TX Queues - effective number of descs:%d\n", nb_desc);
955 /* Allocate queue resources */
956 err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
958 sizeof(struct wq_enet_desc));
960 dev_err(enic, "error in allocation of wq\n");
964 err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
966 sizeof(struct cq_enet_wq_desc));
969 dev_err(enic, "error in allocation of cq for wq\n");
972 /* setup up CQ message */
973 snprintf((char *)name, sizeof(name),
974 "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
977 wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
978 sizeof(uint32_t), SOCKET_ID_ANY,
979 RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
986 int enic_disable(struct enic *enic)
991 for (i = 0; i < enic->intr_count; i++) {
992 vnic_intr_mask(&enic->intr[i]);
993 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */
995 enic_rxq_intr_deinit(enic);
996 rte_intr_disable(&enic->pdev->intr_handle);
997 rte_intr_callback_unregister(&enic->pdev->intr_handle,
999 (void *)enic->rte_dev);
1001 vnic_dev_disable(enic->vdev);
1003 enic_clsf_destroy(enic);
1005 if (!enic_is_sriov_vf(enic))
1006 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
1008 for (i = 0; i < enic->wq_count; i++) {
1009 err = vnic_wq_disable(&enic->wq[i]);
1013 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
1014 if (enic->rq[i].in_use) {
1015 err = vnic_rq_disable(&enic->rq[i]);
1021 /* If we were using interrupts, set the interrupt vector to -1
1022 * to disable interrupts. We are not disabling link notifcations,
1023 * though, as we want the polling of link status to continue working.
1025 if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
1026 vnic_dev_notify_set(enic->vdev, -1);
1028 vnic_dev_set_reset_flag(enic->vdev, 1);
1030 for (i = 0; i < enic->wq_count; i++)
1031 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
1033 for (i = 0; i < enic_vnic_rq_count(enic); i++)
1034 if (enic->rq[i].in_use)
1035 vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
1036 for (i = 0; i < enic->cq_count; i++)
1037 vnic_cq_clean(&enic->cq[i]);
1038 for (i = 0; i < enic->intr_count; i++)
1039 vnic_intr_clean(&enic->intr[i]);
1044 static int enic_dev_wait(struct vnic_dev *vdev,
1045 int (*start)(struct vnic_dev *, int),
1046 int (*finished)(struct vnic_dev *, int *),
1053 err = start(vdev, arg);
1057 /* Wait for func to complete...2 seconds max */
1058 for (i = 0; i < 2000; i++) {
1059 err = finished(vdev, &done);
1069 static int enic_dev_open(struct enic *enic)
1072 int flags = CMD_OPENF_IG_DESCCACHE;
1074 err = enic_dev_wait(enic->vdev, vnic_dev_open,
1075 vnic_dev_open_done, flags);
1077 dev_err(enic_get_dev(enic),
1078 "vNIC device open failed, err %d\n", err);
1083 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
1085 dma_addr_t rss_key_buf_pa;
1086 union vnic_rss_key *rss_key_buf_va = NULL;
1090 RTE_ASSERT(user_key != NULL);
1091 snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
1092 rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
1093 &rss_key_buf_pa, name);
1094 if (!rss_key_buf_va)
1097 for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
1098 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
1100 err = enic_set_rss_key(enic,
1102 sizeof(union vnic_rss_key));
1104 /* Save for later queries */
1106 rte_memcpy(&enic->rss_key, rss_key_buf_va,
1107 sizeof(union vnic_rss_key));
1109 enic_free_consistent(enic, sizeof(union vnic_rss_key),
1110 rss_key_buf_va, rss_key_buf_pa);
1115 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
1117 dma_addr_t rss_cpu_buf_pa;
1118 union vnic_rss_cpu *rss_cpu_buf_va = NULL;
1122 snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
1123 rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
1124 &rss_cpu_buf_pa, name);
1125 if (!rss_cpu_buf_va)
1128 rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
1130 err = enic_set_rss_cpu(enic,
1132 sizeof(union vnic_rss_cpu));
1134 enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
1135 rss_cpu_buf_va, rss_cpu_buf_pa);
1137 /* Save for later queries */
1139 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
1143 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
1144 u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
1146 const u8 tso_ipid_split_en = 0;
1149 err = enic_set_nic_cfg(enic,
1150 rss_default_cpu, rss_hash_type,
1151 rss_hash_bits, rss_base_cpu,
1152 rss_enable, tso_ipid_split_en,
1153 enic->ig_vlan_strip_en);
1158 /* Initialize RSS with defaults, called from dev_configure */
1159 int enic_init_rss_nic_cfg(struct enic *enic)
1161 static uint8_t default_rss_key[] = {
1162 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1163 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1164 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1165 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1167 struct rte_eth_rss_conf rss_conf;
1168 union vnic_rss_cpu rss_cpu;
1171 rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1173 * If setting key for the first time, and the user gives us none, then
1174 * push the default key to NIC.
1176 if (rss_conf.rss_key == NULL) {
1177 rss_conf.rss_key = default_rss_key;
1178 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1180 ret = enic_set_rss_conf(enic, &rss_conf);
1182 dev_err(enic, "Failed to configure RSS\n");
1185 if (enic->rss_enable) {
1186 /* If enabling RSS, use the default reta */
1187 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1188 rss_cpu.cpu[i / 4].b[i % 4] =
1189 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1191 ret = enic_set_rss_reta(enic, &rss_cpu);
1193 dev_err(enic, "Failed to set RSS indirection table\n");
1198 int enic_setup_finish(struct enic *enic)
1200 enic_init_soft_stats(enic);
1203 vnic_dev_packet_filter(enic->vdev,
1216 static int enic_rss_conf_valid(struct enic *enic,
1217 struct rte_eth_rss_conf *rss_conf)
1219 /* RSS is disabled per VIC settings. Ignore rss_conf. */
1220 if (enic->flow_type_rss_offloads == 0)
1222 if (rss_conf->rss_key != NULL &&
1223 rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1224 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1225 rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1228 if (rss_conf->rss_hf != 0 &&
1229 (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1230 dev_err(enic, "Given rss_hf contains none of the supported"
1237 /* Set hash type and key according to rss_conf */
1238 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1240 struct rte_eth_dev *eth_dev;
1246 RTE_ASSERT(rss_conf != NULL);
1247 ret = enic_rss_conf_valid(enic, rss_conf);
1249 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1253 eth_dev = enic->rte_dev;
1255 rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1256 if (enic->rq_count > 1 &&
1257 (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1260 if (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1261 ETH_RSS_NONFRAG_IPV4_OTHER))
1262 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1263 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1264 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1265 if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
1266 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV4;
1267 if (enic->udp_rss_weak) {
1269 * 'TCP' is not a typo. The "weak" version of
1270 * UDP RSS requires both the TCP and UDP bits
1271 * be set. It does enable TCP RSS as well.
1273 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1276 if (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_IPV6_EX |
1277 ETH_RSS_FRAG_IPV6 | ETH_RSS_NONFRAG_IPV6_OTHER))
1278 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1279 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX))
1280 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1281 if (rss_hf & (ETH_RSS_NONFRAG_IPV6_UDP | ETH_RSS_IPV6_UDP_EX)) {
1282 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_UDP_IPV6;
1283 if (enic->udp_rss_weak)
1284 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1291 /* Set the hash key if provided */
1292 if (rss_enable && rss_conf->rss_key) {
1293 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1295 dev_err(enic, "Failed to set RSS key\n");
1300 ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1301 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1304 enic->rss_hf = rss_hf;
1305 enic->rss_hash_type = rss_hash_type;
1306 enic->rss_enable = rss_enable;
1308 dev_err(enic, "Failed to update RSS configurations."
1309 " hash=0x%x\n", rss_hash_type);
1314 int enic_set_vlan_strip(struct enic *enic)
1317 * Unfortunately, VLAN strip on/off and RSS on/off are configured
1318 * together. So, re-do niccfg, preserving the current RSS settings.
1320 return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1321 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1325 void enic_add_packet_filter(struct enic *enic)
1327 /* Args -> directed, multicast, broadcast, promisc, allmulti */
1328 vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1329 enic->promisc, enic->allmulti);
1332 int enic_get_link_status(struct enic *enic)
1334 return vnic_dev_link_status(enic->vdev);
1337 static void enic_dev_deinit(struct enic *enic)
1339 struct rte_eth_dev *eth_dev = enic->rte_dev;
1341 /* stop link status checking */
1342 vnic_dev_notify_unset(enic->vdev);
1344 rte_free(eth_dev->data->mac_addrs);
1346 rte_free(enic->intr);
1352 int enic_set_vnic_res(struct enic *enic)
1354 struct rte_eth_dev *eth_dev = enic->rte_dev;
1356 unsigned int required_rq, required_wq, required_cq, required_intr;
1358 /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1359 required_rq = eth_dev->data->nb_rx_queues * 2;
1360 required_wq = eth_dev->data->nb_tx_queues;
1361 required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1362 required_intr = 1; /* 1 for LSC even if intr_conf.lsc is 0 */
1363 if (eth_dev->data->dev_conf.intr_conf.rxq) {
1364 required_intr += eth_dev->data->nb_rx_queues;
1367 if (enic->conf_rq_count < required_rq) {
1368 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1369 eth_dev->data->nb_rx_queues,
1370 required_rq, enic->conf_rq_count);
1373 if (enic->conf_wq_count < required_wq) {
1374 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1375 eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1379 if (enic->conf_cq_count < required_cq) {
1380 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1381 required_cq, enic->conf_cq_count);
1384 if (enic->conf_intr_count < required_intr) {
1385 dev_err(dev, "Not enough Interrupts to support Rx queue"
1386 " interrupts. Required:%u, Configured:%u\n",
1387 required_intr, enic->conf_intr_count);
1392 enic->rq_count = eth_dev->data->nb_rx_queues;
1393 enic->wq_count = eth_dev->data->nb_tx_queues;
1394 enic->cq_count = enic->rq_count + enic->wq_count;
1395 enic->intr_count = required_intr;
1401 /* Initialize the completion queue for an RQ */
1403 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1405 struct vnic_rq *sop_rq, *data_rq;
1406 unsigned int cq_idx;
1409 sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1410 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1413 vnic_cq_clean(&enic->cq[cq_idx]);
1414 vnic_cq_init(&enic->cq[cq_idx],
1415 0 /* flow_control_enable */,
1416 1 /* color_enable */,
1419 1 /* cq_tail_color */,
1420 0 /* interrupt_enable */,
1421 1 /* cq_entry_enable */,
1422 0 /* cq_message_enable */,
1423 0 /* interrupt offset */,
1424 0 /* cq_message_addr */);
1427 vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1428 enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1429 sop_rq->ring.desc_count - 1, 1, 0);
1430 if (data_rq->in_use) {
1431 vnic_rq_init_start(data_rq,
1433 enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1434 data_rq->ring.desc_count - 1, 1, 0);
1437 rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1441 if (data_rq->in_use) {
1442 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1444 enic_rxmbuf_queue_release(enic, sop_rq);
1452 /* The Cisco NIC can send and receive packets up to a max packet size
1453 * determined by the NIC type and firmware. There is also an MTU
1454 * configured into the NIC via the CIMC/UCSM management interface
1455 * which can be overridden by this function (up to the max packet size).
1456 * Depending on the network setup, doing so may cause packet drops
1457 * and unexpected behavior.
1459 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1461 unsigned int rq_idx;
1464 uint16_t old_mtu; /* previous setting */
1465 uint16_t config_mtu; /* Value configured into NIC via CIMC/UCSM */
1466 struct rte_eth_dev *eth_dev = enic->rte_dev;
1468 old_mtu = eth_dev->data->mtu;
1469 config_mtu = enic->config.mtu;
1471 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1472 return -E_RTE_SECONDARY;
1474 if (new_mtu > enic->max_mtu) {
1476 "MTU not updated: requested (%u) greater than max (%u)\n",
1477 new_mtu, enic->max_mtu);
1480 if (new_mtu < ENIC_MIN_MTU) {
1482 "MTU not updated: requested (%u) less than min (%u)\n",
1483 new_mtu, ENIC_MIN_MTU);
1486 if (new_mtu > config_mtu)
1488 "MTU (%u) is greater than value configured in NIC (%u)\n",
1489 new_mtu, config_mtu);
1491 /* Update the MTU and maximum packet length */
1492 eth_dev->data->mtu = new_mtu;
1493 eth_dev->data->dev_conf.rxmode.max_rx_pkt_len =
1494 enic_mtu_to_max_rx_pktlen(new_mtu);
1497 * If the device has not started (enic_enable), nothing to do.
1498 * Later, enic_enable() will set up RQs reflecting the new maximum
1501 if (!eth_dev->data->dev_started)
1505 * The device has started, re-do RQs on the fly. In the process, we
1506 * pick up the new maximum packet length.
1508 * Some applications rely on the ability to change MTU without stopping
1509 * the device. So keep this behavior for now.
1511 rte_spinlock_lock(&enic->mtu_lock);
1513 /* Stop traffic on all RQs */
1514 for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1515 rq = &enic->rq[rq_idx];
1516 if (rq->is_sop && rq->in_use) {
1517 rc = enic_stop_rq(enic,
1518 enic_sop_rq_idx_to_rte_idx(rq_idx));
1520 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1526 /* replace Rx function with a no-op to avoid getting stale pkts */
1527 eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1530 /* Allow time for threads to exit the real Rx function. */
1533 /* now it is safe to reconfigure the RQs */
1536 /* free and reallocate RQs with the new MTU */
1537 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1538 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1543 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1544 rq->tot_nb_desc, rq->rx_free_thresh);
1547 "Fatal MTU alloc error- No traffic will pass\n");
1551 rc = enic_reinit_rq(enic, rq_idx);
1554 "Fatal MTU RQ reinit- No traffic will pass\n");
1559 /* put back the real receive function */
1561 pick_rx_handler(enic);
1564 /* restart Rx traffic */
1565 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1566 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1567 if (rq->is_sop && rq->in_use)
1568 enic_start_rq(enic, rq_idx);
1572 dev_info(enic, "MTU changed from %u to %u\n", old_mtu, new_mtu);
1573 rte_spinlock_unlock(&enic->mtu_lock);
1577 static int enic_dev_init(struct enic *enic)
1580 struct rte_eth_dev *eth_dev = enic->rte_dev;
1582 vnic_dev_intr_coal_timer_info_default(enic->vdev);
1584 /* Get vNIC configuration
1586 err = enic_get_vnic_config(enic);
1588 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1592 /* Get available resource counts */
1593 enic_get_res_counts(enic);
1594 if (enic->conf_rq_count == 1) {
1595 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1596 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1597 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1600 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1601 enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1602 enic->conf_cq_count, 8);
1603 enic->intr = rte_zmalloc("enic_vnic_intr", sizeof(struct vnic_intr) *
1604 enic->conf_intr_count, 8);
1605 enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1606 enic->conf_rq_count, 8);
1607 enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1608 enic->conf_wq_count, 8);
1609 if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1610 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1613 if (enic->conf_intr_count > 0 && enic->intr == NULL) {
1614 dev_err(enic, "failed to allocate vnic_intr, aborting.\n");
1617 if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1618 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1621 if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1622 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1626 /* Get the supported filters */
1627 enic_fdir_info(enic);
1629 eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr", ETH_ALEN
1630 * ENIC_MAX_MAC_ADDR, 0);
1631 if (!eth_dev->data->mac_addrs) {
1632 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1635 ether_addr_copy((struct ether_addr *) enic->mac_addr,
1636 eth_dev->data->mac_addrs);
1638 vnic_dev_set_reset_flag(enic->vdev, 0);
1640 LIST_INIT(&enic->flows);
1641 rte_spinlock_init(&enic->flows_lock);
1643 /* set up link status checking */
1644 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1646 enic->overlay_offload = false;
1647 if (!enic->disable_overlay && enic->vxlan &&
1648 /* 'VXLAN feature' enables VXLAN, NVGRE, and GENEVE. */
1649 vnic_dev_overlay_offload_ctrl(enic->vdev,
1650 OVERLAY_FEATURE_VXLAN,
1651 OVERLAY_OFFLOAD_ENABLE) == 0) {
1652 enic->tx_offload_capa |=
1653 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
1654 DEV_TX_OFFLOAD_GENEVE_TNL_TSO |
1655 DEV_TX_OFFLOAD_VXLAN_TNL_TSO;
1657 * Do not add PKT_TX_OUTER_{IPV4,IPV6} as they are not
1658 * 'offload' flags (i.e. not part of PKT_TX_OFFLOAD_MASK).
1660 enic->tx_offload_mask |=
1661 PKT_TX_OUTER_IP_CKSUM |
1663 enic->overlay_offload = true;
1664 enic->vxlan_port = ENIC_DEFAULT_VXLAN_PORT;
1665 dev_info(enic, "Overlay offload is enabled\n");
1667 * Reset the vxlan port to the default, as the NIC firmware
1668 * does not reset it automatically and keeps the old setting.
1670 if (vnic_dev_overlay_offload_cfg(enic->vdev,
1671 OVERLAY_CFG_VXLAN_PORT_UPDATE,
1672 ENIC_DEFAULT_VXLAN_PORT)) {
1673 dev_err(enic, "failed to update vxlan port\n");
1682 int enic_probe(struct enic *enic)
1684 struct rte_pci_device *pdev = enic->pdev;
1687 dev_debug(enic, " Initializing ENIC PMD\n");
1689 /* if this is a secondary process the hardware is already initialized */
1690 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1693 enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1694 enic->bar0.len = pdev->mem_resource[0].len;
1696 /* Register vNIC device */
1697 enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1699 dev_err(enic, "vNIC registration failed, aborting\n");
1703 LIST_INIT(&enic->memzone_list);
1704 rte_spinlock_init(&enic->memzone_list_lock);
1706 vnic_register_cbacks(enic->vdev,
1707 enic_alloc_consistent,
1708 enic_free_consistent);
1711 * Allocate the consistent memory for stats upfront so both primary and
1712 * secondary processes can dump stats.
1714 err = vnic_dev_alloc_stats_mem(enic->vdev);
1716 dev_err(enic, "Failed to allocate cmd memory, aborting\n");
1717 goto err_out_unregister;
1719 /* Issue device open to get device in known state */
1720 err = enic_dev_open(enic);
1722 dev_err(enic, "vNIC dev open failed, aborting\n");
1723 goto err_out_unregister;
1726 /* Set ingress vlan rewrite mode before vnic initialization */
1727 dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n",
1728 enic->ig_vlan_rewrite_mode);
1729 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1730 enic->ig_vlan_rewrite_mode);
1733 "Failed to set ingress vlan rewrite mode, aborting.\n");
1734 goto err_out_dev_close;
1737 /* Issue device init to initialize the vnic-to-switch link.
1738 * We'll start with carrier off and wait for link UP
1739 * notification later to turn on carrier. We don't need
1740 * to wait here for the vnic-to-switch link initialization
1741 * to complete; link UP notification is the indication that
1742 * the process is complete.
1745 err = vnic_dev_init(enic->vdev, 0);
1747 dev_err(enic, "vNIC dev init failed, aborting\n");
1748 goto err_out_dev_close;
1751 err = enic_dev_init(enic);
1753 dev_err(enic, "Device initialization failed, aborting\n");
1754 goto err_out_dev_close;
1760 vnic_dev_close(enic->vdev);
1762 vnic_dev_unregister(enic->vdev);
1767 void enic_remove(struct enic *enic)
1769 enic_dev_deinit(enic);
1770 vnic_dev_close(enic->vdev);
1771 vnic_dev_unregister(enic->vdev);