1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
14 #include <rte_bus_pci.h>
15 #include <rte_memzone.h>
16 #include <rte_malloc.h>
18 #include <rte_string_fns.h>
19 #include <rte_ethdev_driver.h>
21 #include "enic_compat.h"
23 #include "wq_enet_desc.h"
24 #include "rq_enet_desc.h"
25 #include "cq_enet_desc.h"
26 #include "vnic_enet.h"
31 #include "vnic_intr.h"
34 static inline int enic_is_sriov_vf(struct enic *enic)
36 return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
39 static int is_zero_addr(uint8_t *addr)
41 return !(addr[0] | addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
44 static int is_mcast_addr(uint8_t *addr)
49 static int is_eth_addr_valid(uint8_t *addr)
51 return !is_mcast_addr(addr) && !is_zero_addr(addr);
55 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
59 if (!rq || !rq->mbuf_ring) {
60 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
64 for (i = 0; i < rq->ring.desc_count; i++) {
65 if (rq->mbuf_ring[i]) {
66 rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
67 rq->mbuf_ring[i] = NULL;
72 static void enic_free_wq_buf(struct vnic_wq_buf *buf)
74 struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->mb;
76 rte_pktmbuf_free_seg(mbuf);
80 static void enic_log_q_error(struct enic *enic)
85 for (i = 0; i < enic->wq_count; i++) {
86 error_status = vnic_wq_error_status(&enic->wq[i]);
88 dev_err(enic, "WQ[%d] error_status %d\n", i,
92 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
93 if (!enic->rq[i].in_use)
95 error_status = vnic_rq_error_status(&enic->rq[i]);
97 dev_err(enic, "RQ[%d] error_status %d\n", i,
102 static void enic_clear_soft_stats(struct enic *enic)
104 struct enic_soft_stats *soft_stats = &enic->soft_stats;
105 rte_atomic64_clear(&soft_stats->rx_nombuf);
106 rte_atomic64_clear(&soft_stats->rx_packet_errors);
107 rte_atomic64_clear(&soft_stats->tx_oversized);
110 static void enic_init_soft_stats(struct enic *enic)
112 struct enic_soft_stats *soft_stats = &enic->soft_stats;
113 rte_atomic64_init(&soft_stats->rx_nombuf);
114 rte_atomic64_init(&soft_stats->rx_packet_errors);
115 rte_atomic64_init(&soft_stats->tx_oversized);
116 enic_clear_soft_stats(enic);
119 void enic_dev_stats_clear(struct enic *enic)
121 if (vnic_dev_stats_clear(enic->vdev))
122 dev_err(enic, "Error in clearing stats\n");
123 enic_clear_soft_stats(enic);
126 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
128 struct vnic_stats *stats;
129 struct enic_soft_stats *soft_stats = &enic->soft_stats;
130 int64_t rx_truncated;
131 uint64_t rx_packet_errors;
132 int ret = vnic_dev_stats_dump(enic->vdev, &stats);
135 dev_err(enic, "Error in getting stats\n");
139 /* The number of truncated packets can only be calculated by
140 * subtracting a hardware counter from error packets received by
141 * the driver. Note: this causes transient inaccuracies in the
142 * ipackets count. Also, the length of truncated packets are
143 * counted in ibytes even though truncated packets are dropped
144 * which can make ibytes be slightly higher than it should be.
146 rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
147 rx_truncated = rx_packet_errors - stats->rx.rx_errors;
149 r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
150 r_stats->opackets = stats->tx.tx_frames_ok;
152 r_stats->ibytes = stats->rx.rx_bytes_ok;
153 r_stats->obytes = stats->tx.tx_bytes_ok;
155 r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
156 r_stats->oerrors = stats->tx.tx_errors
157 + rte_atomic64_read(&soft_stats->tx_oversized);
159 r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
161 r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
165 void enic_del_mac_address(struct enic *enic, int mac_index)
167 struct rte_eth_dev *eth_dev = enic->rte_dev;
168 uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
170 if (vnic_dev_del_addr(enic->vdev, mac_addr))
171 dev_err(enic, "del mac addr failed\n");
174 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
178 if (!is_eth_addr_valid(mac_addr)) {
179 dev_err(enic, "invalid mac address\n");
183 err = vnic_dev_add_addr(enic->vdev, mac_addr);
185 dev_err(enic, "add mac addr failed\n");
190 enic_free_rq_buf(struct rte_mbuf **mbuf)
195 rte_pktmbuf_free(*mbuf);
199 void enic_init_vnic_resources(struct enic *enic)
201 unsigned int error_interrupt_enable = 1;
202 unsigned int error_interrupt_offset = 0;
203 unsigned int index = 0;
205 struct vnic_rq *data_rq;
207 for (index = 0; index < enic->rq_count; index++) {
208 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
210 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
212 error_interrupt_enable,
213 error_interrupt_offset);
215 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
217 vnic_rq_init(data_rq,
219 error_interrupt_enable,
220 error_interrupt_offset);
222 vnic_cq_init(&enic->cq[cq_idx],
223 0 /* flow_control_enable */,
224 1 /* color_enable */,
227 1 /* cq_tail_color */,
228 0 /* interrupt_enable */,
229 1 /* cq_entry_enable */,
230 0 /* cq_message_enable */,
231 0 /* interrupt offset */,
232 0 /* cq_message_addr */);
235 for (index = 0; index < enic->wq_count; index++) {
236 vnic_wq_init(&enic->wq[index],
237 enic_cq_wq(enic, index),
238 error_interrupt_enable,
239 error_interrupt_offset);
241 cq_idx = enic_cq_wq(enic, index);
242 vnic_cq_init(&enic->cq[cq_idx],
243 0 /* flow_control_enable */,
244 1 /* color_enable */,
247 1 /* cq_tail_color */,
248 0 /* interrupt_enable */,
249 0 /* cq_entry_enable */,
250 1 /* cq_message_enable */,
251 0 /* interrupt offset */,
252 (u64)enic->wq[index].cqmsg_rz->iova);
255 vnic_intr_init(&enic->intr,
256 enic->config.intr_timer_usec,
257 enic->config.intr_timer_type,
258 /*mask_on_assertion*/1);
263 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
266 struct rq_enet_desc *rqd = rq->ring.descs;
269 uint32_t max_rx_pkt_len;
275 dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
276 rq->ring.desc_count);
279 * If *not* using scatter and the mbuf size is smaller than the
280 * requested max packet size (max_rx_pkt_len), then reduce the
281 * posted buffer size to max_rx_pkt_len. HW still receives packets
282 * larger than max_rx_pkt_len, but they will be truncated, which we
283 * drop in the rx handler. Not ideal, but better than returning
284 * large packets when the user is not expecting them.
286 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
287 rq_buf_len = rte_pktmbuf_data_room_size(rq->mp) - RTE_PKTMBUF_HEADROOM;
288 if (max_rx_pkt_len < rq_buf_len && !rq->data_queue_enable)
289 rq_buf_len = max_rx_pkt_len;
290 for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
291 mb = rte_mbuf_raw_alloc(rq->mp);
293 dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
294 (unsigned)rq->index);
298 mb->data_off = RTE_PKTMBUF_HEADROOM;
299 dma_addr = (dma_addr_t)(mb->buf_iova
300 + RTE_PKTMBUF_HEADROOM);
301 rq_enet_desc_enc(rqd, dma_addr,
302 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
303 : RQ_ENET_TYPE_NOT_SOP),
305 rq->mbuf_ring[i] = mb;
308 /* make sure all prior writes are complete before doing the PIO write */
311 /* Post all but the last buffer to VIC. */
312 rq->posted_index = rq->ring.desc_count - 1;
316 dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
317 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
318 iowrite32(rq->posted_index, &rq->ctrl->posted_index);
319 iowrite32(0, &rq->ctrl->fetch_index);
327 enic_alloc_consistent(void *priv, size_t size,
328 dma_addr_t *dma_handle, u8 *name)
331 const struct rte_memzone *rz;
333 struct enic *enic = (struct enic *)priv;
334 struct enic_memzone_entry *mze;
336 rz = rte_memzone_reserve_aligned((const char *)name,
337 size, SOCKET_ID_ANY, 0, ENIC_ALIGN);
339 pr_err("%s : Failed to allocate memory requested for %s\n",
345 *dma_handle = (dma_addr_t)rz->iova;
347 mze = rte_malloc("enic memzone entry",
348 sizeof(struct enic_memzone_entry), 0);
351 pr_err("%s : Failed to allocate memory for memzone list\n",
353 rte_memzone_free(rz);
359 rte_spinlock_lock(&enic->memzone_list_lock);
360 LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
361 rte_spinlock_unlock(&enic->memzone_list_lock);
367 enic_free_consistent(void *priv,
368 __rte_unused size_t size,
370 dma_addr_t dma_handle)
372 struct enic_memzone_entry *mze;
373 struct enic *enic = (struct enic *)priv;
375 rte_spinlock_lock(&enic->memzone_list_lock);
376 LIST_FOREACH(mze, &enic->memzone_list, entries) {
377 if (mze->rz->addr == vaddr &&
378 mze->rz->iova == dma_handle)
382 rte_spinlock_unlock(&enic->memzone_list_lock);
384 "Tried to free memory, but couldn't find it in the memzone list\n");
387 LIST_REMOVE(mze, entries);
388 rte_spinlock_unlock(&enic->memzone_list_lock);
389 rte_memzone_free(mze->rz);
393 int enic_link_update(struct enic *enic)
395 struct rte_eth_dev *eth_dev = enic->rte_dev;
399 link_status = enic_get_link_status(enic);
400 ret = (link_status == enic->link_status);
401 enic->link_status = link_status;
402 eth_dev->data->dev_link.link_status = link_status;
403 eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
404 eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
409 enic_intr_handler(void *arg)
411 struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
412 struct enic *enic = pmd_priv(dev);
414 vnic_intr_return_all_credits(&enic->intr);
416 enic_link_update(enic);
417 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
418 enic_log_q_error(enic);
421 int enic_enable(struct enic *enic)
425 struct rte_eth_dev *eth_dev = enic->rte_dev;
427 eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
428 eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
430 /* vnic notification of link status has already been turned on in
431 * enic_dev_init() which is called during probe time. Here we are
432 * just turning on interrupt vector 0 if needed.
434 if (eth_dev->data->dev_conf.intr_conf.lsc)
435 vnic_dev_notify_set(enic->vdev, 0);
437 if (enic_clsf_init(enic))
438 dev_warning(enic, "Init of hash table for clsf failed."\
439 "Flow director feature will not work\n");
441 for (index = 0; index < enic->rq_count; index++) {
442 err = enic_alloc_rx_queue_mbufs(enic,
443 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
445 dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
448 err = enic_alloc_rx_queue_mbufs(enic,
449 &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
451 /* release the allocated mbufs for the sop rq*/
452 enic_rxmbuf_queue_release(enic,
453 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
455 dev_err(enic, "Failed to alloc data RX queue mbufs\n");
460 for (index = 0; index < enic->wq_count; index++)
461 enic_start_wq(enic, index);
462 for (index = 0; index < enic->rq_count; index++)
463 enic_start_rq(enic, index);
465 vnic_dev_add_addr(enic->vdev, enic->mac_addr);
467 vnic_dev_enable_wait(enic->vdev);
469 /* Register and enable error interrupt */
470 rte_intr_callback_register(&(enic->pdev->intr_handle),
471 enic_intr_handler, (void *)enic->rte_dev);
473 rte_intr_enable(&(enic->pdev->intr_handle));
474 vnic_intr_unmask(&enic->intr);
479 int enic_alloc_intr_resources(struct enic *enic)
483 dev_info(enic, "vNIC resources used: "\
484 "wq %d rq %d cq %d intr %d\n",
485 enic->wq_count, enic_vnic_rq_count(enic),
486 enic->cq_count, enic->intr_count);
488 err = vnic_intr_alloc(enic->vdev, &enic->intr, 0);
490 enic_free_vnic_resources(enic);
495 void enic_free_rq(void *rxq)
497 struct vnic_rq *rq_sop, *rq_data;
503 rq_sop = (struct vnic_rq *)rxq;
504 enic = vnic_dev_priv(rq_sop->vdev);
505 rq_data = &enic->rq[rq_sop->data_queue_idx];
507 enic_rxmbuf_queue_release(enic, rq_sop);
509 enic_rxmbuf_queue_release(enic, rq_data);
511 rte_free(rq_sop->mbuf_ring);
513 rte_free(rq_data->mbuf_ring);
515 rq_sop->mbuf_ring = NULL;
516 rq_data->mbuf_ring = NULL;
518 vnic_rq_free(rq_sop);
520 vnic_rq_free(rq_data);
522 vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
528 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
530 struct rte_eth_dev *eth_dev = enic->rte_dev;
531 vnic_wq_enable(&enic->wq[queue_idx]);
532 eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
535 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
537 struct rte_eth_dev *eth_dev = enic->rte_dev;
540 ret = vnic_wq_disable(&enic->wq[queue_idx]);
544 eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
548 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
550 struct vnic_rq *rq_sop;
551 struct vnic_rq *rq_data;
552 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
553 rq_data = &enic->rq[rq_sop->data_queue_idx];
554 struct rte_eth_dev *eth_dev = enic->rte_dev;
557 vnic_rq_enable(rq_data);
559 vnic_rq_enable(rq_sop);
560 eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
563 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
565 int ret1 = 0, ret2 = 0;
566 struct rte_eth_dev *eth_dev = enic->rte_dev;
567 struct vnic_rq *rq_sop;
568 struct vnic_rq *rq_data;
569 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
570 rq_data = &enic->rq[rq_sop->data_queue_idx];
572 ret2 = vnic_rq_disable(rq_sop);
575 ret1 = vnic_rq_disable(rq_data);
582 eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
586 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
587 unsigned int socket_id, struct rte_mempool *mp,
588 uint16_t nb_desc, uint16_t free_thresh)
591 uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
592 uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
593 struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
594 struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
595 unsigned int mbuf_size, mbufs_per_pkt;
596 unsigned int nb_sop_desc, nb_data_desc;
597 uint16_t min_sop, max_sop, min_data, max_data;
598 uint32_t max_rx_pkt_len;
601 rq_sop->data_queue_idx = data_queue_idx;
603 rq_data->data_queue_idx = 0;
604 rq_sop->socket_id = socket_id;
606 rq_data->socket_id = socket_id;
609 rq_sop->rx_free_thresh = free_thresh;
610 rq_data->rx_free_thresh = free_thresh;
611 dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
614 mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
615 RTE_PKTMBUF_HEADROOM);
616 /* max_rx_pkt_len includes the ethernet header and CRC. */
617 max_rx_pkt_len = enic->rte_dev->data->dev_conf.rxmode.max_rx_pkt_len;
619 if (enic->rte_dev->data->dev_conf.rxmode.offloads &
620 DEV_RX_OFFLOAD_SCATTER) {
621 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
622 /* ceil((max pkt len)/mbuf_size) */
623 mbufs_per_pkt = (max_rx_pkt_len + mbuf_size - 1) / mbuf_size;
625 dev_info(enic, "Scatter rx mode disabled\n");
627 if (max_rx_pkt_len > mbuf_size) {
628 dev_warning(enic, "The maximum Rx packet size (%u) is"
629 " larger than the mbuf size (%u), and"
630 " scatter is disabled. Larger packets will"
632 max_rx_pkt_len, mbuf_size);
636 if (mbufs_per_pkt > 1) {
637 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
638 rq_sop->data_queue_enable = 1;
641 * HW does not directly support rxmode.max_rx_pkt_len. HW always
642 * receives packet sizes up to the "max" MTU.
643 * If not using scatter, we can achieve the effect of dropping
644 * larger packets by reducing the size of posted buffers.
645 * See enic_alloc_rx_queue_mbufs().
648 enic_mtu_to_max_rx_pktlen(enic->rte_dev->data->mtu)) {
649 dev_warning(enic, "rxmode.max_rx_pkt_len is ignored"
650 " when scatter rx mode is in use.\n");
653 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
655 rq_sop->data_queue_enable = 0;
659 /* number of descriptors have to be a multiple of 32 */
660 nb_sop_desc = (nb_desc / mbufs_per_pkt) & ~0x1F;
661 nb_data_desc = (nb_desc - nb_sop_desc) & ~0x1F;
663 rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
664 rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
666 if (mbufs_per_pkt > 1) {
668 max_sop = ((enic->config.rq_desc_count /
669 (mbufs_per_pkt - 1)) & ~0x1F);
670 min_data = min_sop * (mbufs_per_pkt - 1);
671 max_data = enic->config.rq_desc_count;
674 max_sop = enic->config.rq_desc_count;
679 if (nb_desc < (min_sop + min_data)) {
681 "Number of rx descs too low, adjusting to minimum\n");
682 nb_sop_desc = min_sop;
683 nb_data_desc = min_data;
684 } else if (nb_desc > (max_sop + max_data)) {
686 "Number of rx_descs too high, adjusting to maximum\n");
687 nb_sop_desc = max_sop;
688 nb_data_desc = max_data;
690 if (mbufs_per_pkt > 1) {
691 dev_info(enic, "For max packet size %u and mbuf size %u valid"
692 " rx descriptor range is %u to %u\n",
693 max_rx_pkt_len, mbuf_size, min_sop + min_data,
696 dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
697 nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
699 /* Allocate sop queue resources */
700 rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
701 nb_sop_desc, sizeof(struct rq_enet_desc));
703 dev_err(enic, "error in allocation of sop rq\n");
706 nb_sop_desc = rq_sop->ring.desc_count;
708 if (rq_data->in_use) {
709 /* Allocate data queue resources */
710 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
712 sizeof(struct rq_enet_desc));
714 dev_err(enic, "error in allocation of data rq\n");
715 goto err_free_rq_sop;
717 nb_data_desc = rq_data->ring.desc_count;
719 rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
720 socket_id, nb_sop_desc + nb_data_desc,
721 sizeof(struct cq_enet_rq_desc));
723 dev_err(enic, "error in allocation of cq for rq\n");
724 goto err_free_rq_data;
727 /* Allocate the mbuf rings */
728 rq_sop->mbuf_ring = (struct rte_mbuf **)
729 rte_zmalloc_socket("rq->mbuf_ring",
730 sizeof(struct rte_mbuf *) * nb_sop_desc,
731 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
732 if (rq_sop->mbuf_ring == NULL)
735 if (rq_data->in_use) {
736 rq_data->mbuf_ring = (struct rte_mbuf **)
737 rte_zmalloc_socket("rq->mbuf_ring",
738 sizeof(struct rte_mbuf *) * nb_data_desc,
739 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
740 if (rq_data->mbuf_ring == NULL)
741 goto err_free_sop_mbuf;
744 rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
749 rte_free(rq_sop->mbuf_ring);
751 /* cleanup on error */
752 vnic_cq_free(&enic->cq[queue_idx]);
755 vnic_rq_free(rq_data);
757 vnic_rq_free(rq_sop);
762 void enic_free_wq(void *txq)
770 wq = (struct vnic_wq *)txq;
771 enic = vnic_dev_priv(wq->vdev);
772 rte_memzone_free(wq->cqmsg_rz);
774 vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
777 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
778 unsigned int socket_id, uint16_t nb_desc)
781 struct vnic_wq *wq = &enic->wq[queue_idx];
782 unsigned int cq_index = enic_cq_wq(enic, queue_idx);
786 wq->socket_id = socket_id;
788 if (nb_desc > enic->config.wq_desc_count) {
790 "WQ %d - number of tx desc in cmd line (%d)"\
791 "is greater than that in the UCSM/CIMC adapter"\
792 "policy. Applying the value in the adapter "\
794 queue_idx, nb_desc, enic->config.wq_desc_count);
795 } else if (nb_desc != enic->config.wq_desc_count) {
796 enic->config.wq_desc_count = nb_desc;
798 "TX Queues - effective number of descs:%d\n",
803 /* Allocate queue resources */
804 err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
805 enic->config.wq_desc_count,
806 sizeof(struct wq_enet_desc));
808 dev_err(enic, "error in allocation of wq\n");
812 err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
813 socket_id, enic->config.wq_desc_count,
814 sizeof(struct cq_enet_wq_desc));
817 dev_err(enic, "error in allocation of cq for wq\n");
820 /* setup up CQ message */
821 snprintf((char *)name, sizeof(name),
822 "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
825 wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
835 int enic_disable(struct enic *enic)
840 vnic_intr_mask(&enic->intr);
841 (void)vnic_intr_masked(&enic->intr); /* flush write */
842 rte_intr_disable(&enic->pdev->intr_handle);
843 rte_intr_callback_unregister(&enic->pdev->intr_handle,
845 (void *)enic->rte_dev);
847 vnic_dev_disable(enic->vdev);
849 enic_clsf_destroy(enic);
851 if (!enic_is_sriov_vf(enic))
852 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
854 for (i = 0; i < enic->wq_count; i++) {
855 err = vnic_wq_disable(&enic->wq[i]);
859 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
860 if (enic->rq[i].in_use) {
861 err = vnic_rq_disable(&enic->rq[i]);
867 /* If we were using interrupts, set the interrupt vector to -1
868 * to disable interrupts. We are not disabling link notifcations,
869 * though, as we want the polling of link status to continue working.
871 if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
872 vnic_dev_notify_set(enic->vdev, -1);
874 vnic_dev_set_reset_flag(enic->vdev, 1);
876 for (i = 0; i < enic->wq_count; i++)
877 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
879 for (i = 0; i < enic_vnic_rq_count(enic); i++)
880 if (enic->rq[i].in_use)
881 vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
882 for (i = 0; i < enic->cq_count; i++)
883 vnic_cq_clean(&enic->cq[i]);
884 vnic_intr_clean(&enic->intr);
889 static int enic_dev_wait(struct vnic_dev *vdev,
890 int (*start)(struct vnic_dev *, int),
891 int (*finished)(struct vnic_dev *, int *),
898 err = start(vdev, arg);
902 /* Wait for func to complete...2 seconds max */
903 for (i = 0; i < 2000; i++) {
904 err = finished(vdev, &done);
914 static int enic_dev_open(struct enic *enic)
918 err = enic_dev_wait(enic->vdev, vnic_dev_open,
919 vnic_dev_open_done, 0);
921 dev_err(enic_get_dev(enic),
922 "vNIC device open failed, err %d\n", err);
927 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
929 dma_addr_t rss_key_buf_pa;
930 union vnic_rss_key *rss_key_buf_va = NULL;
934 RTE_ASSERT(user_key != NULL);
935 snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
936 rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
937 &rss_key_buf_pa, name);
941 for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
942 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
944 err = enic_set_rss_key(enic,
946 sizeof(union vnic_rss_key));
948 /* Save for later queries */
950 rte_memcpy(&enic->rss_key, rss_key_buf_va,
951 sizeof(union vnic_rss_key));
953 enic_free_consistent(enic, sizeof(union vnic_rss_key),
954 rss_key_buf_va, rss_key_buf_pa);
959 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
961 dma_addr_t rss_cpu_buf_pa;
962 union vnic_rss_cpu *rss_cpu_buf_va = NULL;
966 snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
967 rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
968 &rss_cpu_buf_pa, name);
972 rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
974 err = enic_set_rss_cpu(enic,
976 sizeof(union vnic_rss_cpu));
978 enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
979 rss_cpu_buf_va, rss_cpu_buf_pa);
981 /* Save for later queries */
983 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
987 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
988 u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
990 const u8 tso_ipid_split_en = 0;
993 err = enic_set_nic_cfg(enic,
994 rss_default_cpu, rss_hash_type,
995 rss_hash_bits, rss_base_cpu,
996 rss_enable, tso_ipid_split_en,
997 enic->ig_vlan_strip_en);
1002 /* Initialize RSS with defaults, called from dev_configure */
1003 int enic_init_rss_nic_cfg(struct enic *enic)
1005 static uint8_t default_rss_key[] = {
1006 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
1007 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
1008 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
1009 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
1011 struct rte_eth_rss_conf rss_conf;
1012 union vnic_rss_cpu rss_cpu;
1015 rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
1017 * If setting key for the first time, and the user gives us none, then
1018 * push the default key to NIC.
1020 if (rss_conf.rss_key == NULL) {
1021 rss_conf.rss_key = default_rss_key;
1022 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
1024 ret = enic_set_rss_conf(enic, &rss_conf);
1026 dev_err(enic, "Failed to configure RSS\n");
1029 if (enic->rss_enable) {
1030 /* If enabling RSS, use the default reta */
1031 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
1032 rss_cpu.cpu[i / 4].b[i % 4] =
1033 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1035 ret = enic_set_rss_reta(enic, &rss_cpu);
1037 dev_err(enic, "Failed to set RSS indirection table\n");
1042 int enic_setup_finish(struct enic *enic)
1044 enic_init_soft_stats(enic);
1047 vnic_dev_packet_filter(enic->vdev,
1060 static int enic_rss_conf_valid(struct enic *enic,
1061 struct rte_eth_rss_conf *rss_conf)
1063 /* RSS is disabled per VIC settings. Ignore rss_conf. */
1064 if (enic->flow_type_rss_offloads == 0)
1066 if (rss_conf->rss_key != NULL &&
1067 rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1068 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1069 rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1072 if (rss_conf->rss_hf != 0 &&
1073 (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1074 dev_err(enic, "Given rss_hf contains none of the supported"
1081 /* Set hash type and key according to rss_conf */
1082 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1084 struct rte_eth_dev *eth_dev;
1090 RTE_ASSERT(rss_conf != NULL);
1091 ret = enic_rss_conf_valid(enic, rss_conf);
1093 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1097 eth_dev = enic->rte_dev;
1099 rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1100 if (enic->rq_count > 1 &&
1101 (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1104 if (rss_hf & ETH_RSS_IPV4)
1105 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1106 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1107 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1108 if (rss_hf & ETH_RSS_IPV6)
1109 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1110 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1111 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1112 if (rss_hf & ETH_RSS_IPV6_EX)
1113 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6_EX;
1114 if (rss_hf & ETH_RSS_IPV6_TCP_EX)
1115 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6_EX;
1121 /* Set the hash key if provided */
1122 if (rss_enable && rss_conf->rss_key) {
1123 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1125 dev_err(enic, "Failed to set RSS key\n");
1130 ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1131 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1134 enic->rss_hf = rss_hf;
1135 enic->rss_hash_type = rss_hash_type;
1136 enic->rss_enable = rss_enable;
1141 int enic_set_vlan_strip(struct enic *enic)
1144 * Unfortunately, VLAN strip on/off and RSS on/off are configured
1145 * together. So, re-do niccfg, preserving the current RSS settings.
1147 return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1148 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1152 void enic_add_packet_filter(struct enic *enic)
1154 /* Args -> directed, multicast, broadcast, promisc, allmulti */
1155 vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1156 enic->promisc, enic->allmulti);
1159 int enic_get_link_status(struct enic *enic)
1161 return vnic_dev_link_status(enic->vdev);
1164 static void enic_dev_deinit(struct enic *enic)
1166 struct rte_eth_dev *eth_dev = enic->rte_dev;
1168 /* stop link status checking */
1169 vnic_dev_notify_unset(enic->vdev);
1171 rte_free(eth_dev->data->mac_addrs);
1178 int enic_set_vnic_res(struct enic *enic)
1180 struct rte_eth_dev *eth_dev = enic->rte_dev;
1182 unsigned int required_rq, required_wq, required_cq;
1184 /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1185 required_rq = eth_dev->data->nb_rx_queues * 2;
1186 required_wq = eth_dev->data->nb_tx_queues;
1187 required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1189 if (enic->conf_rq_count < required_rq) {
1190 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1191 eth_dev->data->nb_rx_queues,
1192 required_rq, enic->conf_rq_count);
1195 if (enic->conf_wq_count < required_wq) {
1196 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1197 eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1201 if (enic->conf_cq_count < required_cq) {
1202 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1203 required_cq, enic->conf_cq_count);
1208 enic->rq_count = eth_dev->data->nb_rx_queues;
1209 enic->wq_count = eth_dev->data->nb_tx_queues;
1210 enic->cq_count = enic->rq_count + enic->wq_count;
1216 /* Initialize the completion queue for an RQ */
1218 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1220 struct vnic_rq *sop_rq, *data_rq;
1221 unsigned int cq_idx;
1224 sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1225 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1228 vnic_cq_clean(&enic->cq[cq_idx]);
1229 vnic_cq_init(&enic->cq[cq_idx],
1230 0 /* flow_control_enable */,
1231 1 /* color_enable */,
1234 1 /* cq_tail_color */,
1235 0 /* interrupt_enable */,
1236 1 /* cq_entry_enable */,
1237 0 /* cq_message_enable */,
1238 0 /* interrupt offset */,
1239 0 /* cq_message_addr */);
1242 vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1243 enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1244 sop_rq->ring.desc_count - 1, 1, 0);
1245 if (data_rq->in_use) {
1246 vnic_rq_init_start(data_rq,
1248 enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1249 data_rq->ring.desc_count - 1, 1, 0);
1252 rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1256 if (data_rq->in_use) {
1257 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1259 enic_rxmbuf_queue_release(enic, sop_rq);
1267 /* The Cisco NIC can send and receive packets up to a max packet size
1268 * determined by the NIC type and firmware. There is also an MTU
1269 * configured into the NIC via the CIMC/UCSM management interface
1270 * which can be overridden by this function (up to the max packet size).
1271 * Depending on the network setup, doing so may cause packet drops
1272 * and unexpected behavior.
1274 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1276 unsigned int rq_idx;
1279 uint16_t old_mtu; /* previous setting */
1280 uint16_t config_mtu; /* Value configured into NIC via CIMC/UCSM */
1281 struct rte_eth_dev *eth_dev = enic->rte_dev;
1283 old_mtu = eth_dev->data->mtu;
1284 config_mtu = enic->config.mtu;
1286 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1287 return -E_RTE_SECONDARY;
1289 if (new_mtu > enic->max_mtu) {
1291 "MTU not updated: requested (%u) greater than max (%u)\n",
1292 new_mtu, enic->max_mtu);
1295 if (new_mtu < ENIC_MIN_MTU) {
1297 "MTU not updated: requested (%u) less than min (%u)\n",
1298 new_mtu, ENIC_MIN_MTU);
1301 if (new_mtu > config_mtu)
1303 "MTU (%u) is greater than value configured in NIC (%u)\n",
1304 new_mtu, config_mtu);
1306 /* The easy case is when scatter is disabled. However if the MTU
1307 * becomes greater than the mbuf data size, packet drops will ensue.
1309 if (!(enic->rte_dev->data->dev_conf.rxmode.offloads &
1310 DEV_RX_OFFLOAD_SCATTER)) {
1311 eth_dev->data->mtu = new_mtu;
1315 /* Rx scatter is enabled so reconfigure RQ's on the fly. The point is to
1316 * change Rx scatter mode if necessary for better performance. I.e. if
1317 * MTU was greater than the mbuf size and now it's less, scatter Rx
1318 * doesn't have to be used and vice versa.
1320 rte_spinlock_lock(&enic->mtu_lock);
1322 /* Stop traffic on all RQs */
1323 for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1324 rq = &enic->rq[rq_idx];
1325 if (rq->is_sop && rq->in_use) {
1326 rc = enic_stop_rq(enic,
1327 enic_sop_rq_idx_to_rte_idx(rq_idx));
1329 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1335 /* replace Rx function with a no-op to avoid getting stale pkts */
1336 eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1339 /* Allow time for threads to exit the real Rx function. */
1342 /* now it is safe to reconfigure the RQs */
1344 /* update the mtu */
1345 eth_dev->data->mtu = new_mtu;
1347 /* free and reallocate RQs with the new MTU */
1348 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1349 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1352 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1353 rq->tot_nb_desc, rq->rx_free_thresh);
1356 "Fatal MTU alloc error- No traffic will pass\n");
1360 rc = enic_reinit_rq(enic, rq_idx);
1363 "Fatal MTU RQ reinit- No traffic will pass\n");
1368 /* put back the real receive function */
1370 eth_dev->rx_pkt_burst = enic_recv_pkts;
1373 /* restart Rx traffic */
1374 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1375 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1376 if (rq->is_sop && rq->in_use)
1377 enic_start_rq(enic, rq_idx);
1381 dev_info(enic, "MTU changed from %u to %u\n", old_mtu, new_mtu);
1382 rte_spinlock_unlock(&enic->mtu_lock);
1386 static int enic_dev_init(struct enic *enic)
1389 struct rte_eth_dev *eth_dev = enic->rte_dev;
1391 vnic_dev_intr_coal_timer_info_default(enic->vdev);
1393 /* Get vNIC configuration
1395 err = enic_get_vnic_config(enic);
1397 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1401 /* Get available resource counts */
1402 enic_get_res_counts(enic);
1403 if (enic->conf_rq_count == 1) {
1404 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1405 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1406 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1409 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1410 enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1411 enic->conf_cq_count, 8);
1412 enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1413 enic->conf_rq_count, 8);
1414 enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1415 enic->conf_wq_count, 8);
1416 if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1417 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1420 if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1421 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1424 if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1425 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1429 /* Get the supported filters */
1430 enic_fdir_info(enic);
1432 eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr", ETH_ALEN
1433 * ENIC_MAX_MAC_ADDR, 0);
1434 if (!eth_dev->data->mac_addrs) {
1435 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1438 ether_addr_copy((struct ether_addr *) enic->mac_addr,
1439 eth_dev->data->mac_addrs);
1441 vnic_dev_set_reset_flag(enic->vdev, 0);
1443 LIST_INIT(&enic->flows);
1444 rte_spinlock_init(&enic->flows_lock);
1446 /* set up link status checking */
1447 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1453 int enic_probe(struct enic *enic)
1455 struct rte_pci_device *pdev = enic->pdev;
1458 dev_debug(enic, " Initializing ENIC PMD\n");
1460 /* if this is a secondary process the hardware is already initialized */
1461 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1464 enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1465 enic->bar0.len = pdev->mem_resource[0].len;
1467 /* Register vNIC device */
1468 enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1470 dev_err(enic, "vNIC registration failed, aborting\n");
1474 LIST_INIT(&enic->memzone_list);
1475 rte_spinlock_init(&enic->memzone_list_lock);
1477 vnic_register_cbacks(enic->vdev,
1478 enic_alloc_consistent,
1479 enic_free_consistent);
1481 /* Issue device open to get device in known state */
1482 err = enic_dev_open(enic);
1484 dev_err(enic, "vNIC dev open failed, aborting\n");
1485 goto err_out_unregister;
1488 /* Set ingress vlan rewrite mode before vnic initialization */
1489 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1490 IG_VLAN_REWRITE_MODE_PASS_THRU);
1493 "Failed to set ingress vlan rewrite mode, aborting.\n");
1494 goto err_out_dev_close;
1497 /* Issue device init to initialize the vnic-to-switch link.
1498 * We'll start with carrier off and wait for link UP
1499 * notification later to turn on carrier. We don't need
1500 * to wait here for the vnic-to-switch link initialization
1501 * to complete; link UP notification is the indication that
1502 * the process is complete.
1505 err = vnic_dev_init(enic->vdev, 0);
1507 dev_err(enic, "vNIC dev init failed, aborting\n");
1508 goto err_out_dev_close;
1511 err = enic_dev_init(enic);
1513 dev_err(enic, "Device initialization failed, aborting\n");
1514 goto err_out_dev_close;
1520 vnic_dev_close(enic->vdev);
1522 vnic_dev_unregister(enic->vdev);
1527 void enic_remove(struct enic *enic)
1529 enic_dev_deinit(enic);
1530 vnic_dev_close(enic->vdev);
1531 vnic_dev_unregister(enic->vdev);