1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2008-2017 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
14 #include <rte_bus_pci.h>
15 #include <rte_memzone.h>
16 #include <rte_malloc.h>
18 #include <rte_string_fns.h>
19 #include <rte_ethdev_driver.h>
21 #include "enic_compat.h"
23 #include "wq_enet_desc.h"
24 #include "rq_enet_desc.h"
25 #include "cq_enet_desc.h"
26 #include "vnic_enet.h"
31 #include "vnic_intr.h"
34 static inline int enic_is_sriov_vf(struct enic *enic)
36 return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
39 static int is_zero_addr(uint8_t *addr)
41 return !(addr[0] | addr[1] | addr[2] | addr[3] | addr[4] | addr[5]);
44 static int is_mcast_addr(uint8_t *addr)
49 static int is_eth_addr_valid(uint8_t *addr)
51 return !is_mcast_addr(addr) && !is_zero_addr(addr);
55 enic_rxmbuf_queue_release(__rte_unused struct enic *enic, struct vnic_rq *rq)
59 if (!rq || !rq->mbuf_ring) {
60 dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
64 for (i = 0; i < rq->ring.desc_count; i++) {
65 if (rq->mbuf_ring[i]) {
66 rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
67 rq->mbuf_ring[i] = NULL;
72 static void enic_free_wq_buf(struct vnic_wq_buf *buf)
74 struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->mb;
76 rte_pktmbuf_free_seg(mbuf);
80 static void enic_log_q_error(struct enic *enic)
85 for (i = 0; i < enic->wq_count; i++) {
86 error_status = vnic_wq_error_status(&enic->wq[i]);
88 dev_err(enic, "WQ[%d] error_status %d\n", i,
92 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
93 if (!enic->rq[i].in_use)
95 error_status = vnic_rq_error_status(&enic->rq[i]);
97 dev_err(enic, "RQ[%d] error_status %d\n", i,
102 static void enic_clear_soft_stats(struct enic *enic)
104 struct enic_soft_stats *soft_stats = &enic->soft_stats;
105 rte_atomic64_clear(&soft_stats->rx_nombuf);
106 rte_atomic64_clear(&soft_stats->rx_packet_errors);
107 rte_atomic64_clear(&soft_stats->tx_oversized);
110 static void enic_init_soft_stats(struct enic *enic)
112 struct enic_soft_stats *soft_stats = &enic->soft_stats;
113 rte_atomic64_init(&soft_stats->rx_nombuf);
114 rte_atomic64_init(&soft_stats->rx_packet_errors);
115 rte_atomic64_init(&soft_stats->tx_oversized);
116 enic_clear_soft_stats(enic);
119 void enic_dev_stats_clear(struct enic *enic)
121 if (vnic_dev_stats_clear(enic->vdev))
122 dev_err(enic, "Error in clearing stats\n");
123 enic_clear_soft_stats(enic);
126 int enic_dev_stats_get(struct enic *enic, struct rte_eth_stats *r_stats)
128 struct vnic_stats *stats;
129 struct enic_soft_stats *soft_stats = &enic->soft_stats;
130 int64_t rx_truncated;
131 uint64_t rx_packet_errors;
132 int ret = vnic_dev_stats_dump(enic->vdev, &stats);
135 dev_err(enic, "Error in getting stats\n");
139 /* The number of truncated packets can only be calculated by
140 * subtracting a hardware counter from error packets received by
141 * the driver. Note: this causes transient inaccuracies in the
142 * ipackets count. Also, the length of truncated packets are
143 * counted in ibytes even though truncated packets are dropped
144 * which can make ibytes be slightly higher than it should be.
146 rx_packet_errors = rte_atomic64_read(&soft_stats->rx_packet_errors);
147 rx_truncated = rx_packet_errors - stats->rx.rx_errors;
149 r_stats->ipackets = stats->rx.rx_frames_ok - rx_truncated;
150 r_stats->opackets = stats->tx.tx_frames_ok;
152 r_stats->ibytes = stats->rx.rx_bytes_ok;
153 r_stats->obytes = stats->tx.tx_bytes_ok;
155 r_stats->ierrors = stats->rx.rx_errors + stats->rx.rx_drop;
156 r_stats->oerrors = stats->tx.tx_errors
157 + rte_atomic64_read(&soft_stats->tx_oversized);
159 r_stats->imissed = stats->rx.rx_no_bufs + rx_truncated;
161 r_stats->rx_nombuf = rte_atomic64_read(&soft_stats->rx_nombuf);
165 void enic_del_mac_address(struct enic *enic, int mac_index)
167 struct rte_eth_dev *eth_dev = enic->rte_dev;
168 uint8_t *mac_addr = eth_dev->data->mac_addrs[mac_index].addr_bytes;
170 if (vnic_dev_del_addr(enic->vdev, mac_addr))
171 dev_err(enic, "del mac addr failed\n");
174 int enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
178 if (!is_eth_addr_valid(mac_addr)) {
179 dev_err(enic, "invalid mac address\n");
183 err = vnic_dev_add_addr(enic->vdev, mac_addr);
185 dev_err(enic, "add mac addr failed\n");
190 enic_free_rq_buf(struct rte_mbuf **mbuf)
195 rte_pktmbuf_free(*mbuf);
199 void enic_init_vnic_resources(struct enic *enic)
201 unsigned int error_interrupt_enable = 1;
202 unsigned int error_interrupt_offset = 0;
203 unsigned int index = 0;
205 struct vnic_rq *data_rq;
207 for (index = 0; index < enic->rq_count; index++) {
208 cq_idx = enic_cq_rq(enic, enic_rte_rq_idx_to_sop_idx(index));
210 vnic_rq_init(&enic->rq[enic_rte_rq_idx_to_sop_idx(index)],
212 error_interrupt_enable,
213 error_interrupt_offset);
215 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(index)];
217 vnic_rq_init(data_rq,
219 error_interrupt_enable,
220 error_interrupt_offset);
222 vnic_cq_init(&enic->cq[cq_idx],
223 0 /* flow_control_enable */,
224 1 /* color_enable */,
227 1 /* cq_tail_color */,
228 0 /* interrupt_enable */,
229 1 /* cq_entry_enable */,
230 0 /* cq_message_enable */,
231 0 /* interrupt offset */,
232 0 /* cq_message_addr */);
235 for (index = 0; index < enic->wq_count; index++) {
236 vnic_wq_init(&enic->wq[index],
237 enic_cq_wq(enic, index),
238 error_interrupt_enable,
239 error_interrupt_offset);
241 cq_idx = enic_cq_wq(enic, index);
242 vnic_cq_init(&enic->cq[cq_idx],
243 0 /* flow_control_enable */,
244 1 /* color_enable */,
247 1 /* cq_tail_color */,
248 0 /* interrupt_enable */,
249 0 /* cq_entry_enable */,
250 1 /* cq_message_enable */,
251 0 /* interrupt offset */,
252 (u64)enic->wq[index].cqmsg_rz->iova);
255 vnic_intr_init(&enic->intr,
256 enic->config.intr_timer_usec,
257 enic->config.intr_timer_type,
258 /*mask_on_assertion*/1);
263 enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
266 struct rq_enet_desc *rqd = rq->ring.descs;
273 dev_debug(enic, "queue %u, allocating %u rx queue mbufs\n", rq->index,
274 rq->ring.desc_count);
276 for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
277 mb = rte_mbuf_raw_alloc(rq->mp);
279 dev_err(enic, "RX mbuf alloc failed queue_id=%u\n",
280 (unsigned)rq->index);
284 mb->data_off = RTE_PKTMBUF_HEADROOM;
285 dma_addr = (dma_addr_t)(mb->buf_iova
286 + RTE_PKTMBUF_HEADROOM);
287 rq_enet_desc_enc(rqd, dma_addr,
288 (rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
289 : RQ_ENET_TYPE_NOT_SOP),
290 mb->buf_len - RTE_PKTMBUF_HEADROOM);
291 rq->mbuf_ring[i] = mb;
294 /* make sure all prior writes are complete before doing the PIO write */
297 /* Post all but the last buffer to VIC. */
298 rq->posted_index = rq->ring.desc_count - 1;
302 dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
303 enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
304 iowrite32(rq->posted_index, &rq->ctrl->posted_index);
305 iowrite32(0, &rq->ctrl->fetch_index);
313 enic_alloc_consistent(void *priv, size_t size,
314 dma_addr_t *dma_handle, u8 *name)
317 const struct rte_memzone *rz;
319 struct enic *enic = (struct enic *)priv;
320 struct enic_memzone_entry *mze;
322 rz = rte_memzone_reserve_aligned((const char *)name,
323 size, SOCKET_ID_ANY, 0, ENIC_ALIGN);
325 pr_err("%s : Failed to allocate memory requested for %s\n",
331 *dma_handle = (dma_addr_t)rz->iova;
333 mze = rte_malloc("enic memzone entry",
334 sizeof(struct enic_memzone_entry), 0);
337 pr_err("%s : Failed to allocate memory for memzone list\n",
339 rte_memzone_free(rz);
345 rte_spinlock_lock(&enic->memzone_list_lock);
346 LIST_INSERT_HEAD(&enic->memzone_list, mze, entries);
347 rte_spinlock_unlock(&enic->memzone_list_lock);
353 enic_free_consistent(void *priv,
354 __rte_unused size_t size,
356 dma_addr_t dma_handle)
358 struct enic_memzone_entry *mze;
359 struct enic *enic = (struct enic *)priv;
361 rte_spinlock_lock(&enic->memzone_list_lock);
362 LIST_FOREACH(mze, &enic->memzone_list, entries) {
363 if (mze->rz->addr == vaddr &&
364 mze->rz->iova == dma_handle)
368 rte_spinlock_unlock(&enic->memzone_list_lock);
370 "Tried to free memory, but couldn't find it in the memzone list\n");
373 LIST_REMOVE(mze, entries);
374 rte_spinlock_unlock(&enic->memzone_list_lock);
375 rte_memzone_free(mze->rz);
379 int enic_link_update(struct enic *enic)
381 struct rte_eth_dev *eth_dev = enic->rte_dev;
385 link_status = enic_get_link_status(enic);
386 ret = (link_status == enic->link_status);
387 enic->link_status = link_status;
388 eth_dev->data->dev_link.link_status = link_status;
389 eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
390 eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
395 enic_intr_handler(void *arg)
397 struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
398 struct enic *enic = pmd_priv(dev);
400 vnic_intr_return_all_credits(&enic->intr);
402 enic_link_update(enic);
403 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
404 enic_log_q_error(enic);
407 int enic_enable(struct enic *enic)
411 struct rte_eth_dev *eth_dev = enic->rte_dev;
413 eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
414 eth_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
416 /* vnic notification of link status has already been turned on in
417 * enic_dev_init() which is called during probe time. Here we are
418 * just turning on interrupt vector 0 if needed.
420 if (eth_dev->data->dev_conf.intr_conf.lsc)
421 vnic_dev_notify_set(enic->vdev, 0);
423 if (enic_clsf_init(enic))
424 dev_warning(enic, "Init of hash table for clsf failed."\
425 "Flow director feature will not work\n");
427 for (index = 0; index < enic->rq_count; index++) {
428 err = enic_alloc_rx_queue_mbufs(enic,
429 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
431 dev_err(enic, "Failed to alloc sop RX queue mbufs\n");
434 err = enic_alloc_rx_queue_mbufs(enic,
435 &enic->rq[enic_rte_rq_idx_to_data_idx(index)]);
437 /* release the allocated mbufs for the sop rq*/
438 enic_rxmbuf_queue_release(enic,
439 &enic->rq[enic_rte_rq_idx_to_sop_idx(index)]);
441 dev_err(enic, "Failed to alloc data RX queue mbufs\n");
446 for (index = 0; index < enic->wq_count; index++)
447 enic_start_wq(enic, index);
448 for (index = 0; index < enic->rq_count; index++)
449 enic_start_rq(enic, index);
451 vnic_dev_add_addr(enic->vdev, enic->mac_addr);
453 vnic_dev_enable_wait(enic->vdev);
455 /* Register and enable error interrupt */
456 rte_intr_callback_register(&(enic->pdev->intr_handle),
457 enic_intr_handler, (void *)enic->rte_dev);
459 rte_intr_enable(&(enic->pdev->intr_handle));
460 vnic_intr_unmask(&enic->intr);
465 int enic_alloc_intr_resources(struct enic *enic)
469 dev_info(enic, "vNIC resources used: "\
470 "wq %d rq %d cq %d intr %d\n",
471 enic->wq_count, enic_vnic_rq_count(enic),
472 enic->cq_count, enic->intr_count);
474 err = vnic_intr_alloc(enic->vdev, &enic->intr, 0);
476 enic_free_vnic_resources(enic);
481 void enic_free_rq(void *rxq)
483 struct vnic_rq *rq_sop, *rq_data;
489 rq_sop = (struct vnic_rq *)rxq;
490 enic = vnic_dev_priv(rq_sop->vdev);
491 rq_data = &enic->rq[rq_sop->data_queue_idx];
493 enic_rxmbuf_queue_release(enic, rq_sop);
495 enic_rxmbuf_queue_release(enic, rq_data);
497 rte_free(rq_sop->mbuf_ring);
499 rte_free(rq_data->mbuf_ring);
501 rq_sop->mbuf_ring = NULL;
502 rq_data->mbuf_ring = NULL;
504 vnic_rq_free(rq_sop);
506 vnic_rq_free(rq_data);
508 vnic_cq_free(&enic->cq[enic_sop_rq_idx_to_cq_idx(rq_sop->index)]);
514 void enic_start_wq(struct enic *enic, uint16_t queue_idx)
516 struct rte_eth_dev *eth_dev = enic->rte_dev;
517 vnic_wq_enable(&enic->wq[queue_idx]);
518 eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
521 int enic_stop_wq(struct enic *enic, uint16_t queue_idx)
523 struct rte_eth_dev *eth_dev = enic->rte_dev;
526 ret = vnic_wq_disable(&enic->wq[queue_idx]);
530 eth_dev->data->tx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
534 void enic_start_rq(struct enic *enic, uint16_t queue_idx)
536 struct vnic_rq *rq_sop;
537 struct vnic_rq *rq_data;
538 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
539 rq_data = &enic->rq[rq_sop->data_queue_idx];
540 struct rte_eth_dev *eth_dev = enic->rte_dev;
543 vnic_rq_enable(rq_data);
545 vnic_rq_enable(rq_sop);
546 eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STARTED;
549 int enic_stop_rq(struct enic *enic, uint16_t queue_idx)
551 int ret1 = 0, ret2 = 0;
552 struct rte_eth_dev *eth_dev = enic->rte_dev;
553 struct vnic_rq *rq_sop;
554 struct vnic_rq *rq_data;
555 rq_sop = &enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
556 rq_data = &enic->rq[rq_sop->data_queue_idx];
558 ret2 = vnic_rq_disable(rq_sop);
561 ret1 = vnic_rq_disable(rq_data);
568 eth_dev->data->rx_queue_state[queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
572 int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
573 unsigned int socket_id, struct rte_mempool *mp,
574 uint16_t nb_desc, uint16_t free_thresh)
577 uint16_t sop_queue_idx = enic_rte_rq_idx_to_sop_idx(queue_idx);
578 uint16_t data_queue_idx = enic_rte_rq_idx_to_data_idx(queue_idx);
579 struct vnic_rq *rq_sop = &enic->rq[sop_queue_idx];
580 struct vnic_rq *rq_data = &enic->rq[data_queue_idx];
581 unsigned int mbuf_size, mbufs_per_pkt;
582 unsigned int nb_sop_desc, nb_data_desc;
583 uint16_t min_sop, max_sop, min_data, max_data;
584 uint16_t mtu = enic->rte_dev->data->mtu;
587 rq_sop->data_queue_idx = data_queue_idx;
589 rq_data->data_queue_idx = 0;
590 rq_sop->socket_id = socket_id;
592 rq_data->socket_id = socket_id;
595 rq_sop->rx_free_thresh = free_thresh;
596 rq_data->rx_free_thresh = free_thresh;
597 dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
600 mbuf_size = (uint16_t)(rte_pktmbuf_data_room_size(mp) -
601 RTE_PKTMBUF_HEADROOM);
603 if (enic->rte_dev->data->dev_conf.rxmode.offloads &
604 DEV_RX_OFFLOAD_SCATTER) {
605 dev_info(enic, "Rq %u Scatter rx mode enabled\n", queue_idx);
606 /* ceil((mtu + ETHER_HDR_LEN + 4)/mbuf_size) */
607 mbufs_per_pkt = ((mtu + ETHER_HDR_LEN + 4) +
608 (mbuf_size - 1)) / mbuf_size;
610 dev_info(enic, "Scatter rx mode disabled\n");
614 if (mbufs_per_pkt > 1) {
615 dev_info(enic, "Rq %u Scatter rx mode in use\n", queue_idx);
616 rq_sop->data_queue_enable = 1;
619 dev_info(enic, "Rq %u Scatter rx mode not being used\n",
621 rq_sop->data_queue_enable = 0;
625 /* number of descriptors have to be a multiple of 32 */
626 nb_sop_desc = (nb_desc / mbufs_per_pkt) & ~0x1F;
627 nb_data_desc = (nb_desc - nb_sop_desc) & ~0x1F;
629 rq_sop->max_mbufs_per_pkt = mbufs_per_pkt;
630 rq_data->max_mbufs_per_pkt = mbufs_per_pkt;
632 if (mbufs_per_pkt > 1) {
634 max_sop = ((enic->config.rq_desc_count /
635 (mbufs_per_pkt - 1)) & ~0x1F);
636 min_data = min_sop * (mbufs_per_pkt - 1);
637 max_data = enic->config.rq_desc_count;
640 max_sop = enic->config.rq_desc_count;
645 if (nb_desc < (min_sop + min_data)) {
647 "Number of rx descs too low, adjusting to minimum\n");
648 nb_sop_desc = min_sop;
649 nb_data_desc = min_data;
650 } else if (nb_desc > (max_sop + max_data)) {
652 "Number of rx_descs too high, adjusting to maximum\n");
653 nb_sop_desc = max_sop;
654 nb_data_desc = max_data;
656 if (mbufs_per_pkt > 1) {
657 dev_info(enic, "For mtu %d and mbuf size %d valid rx descriptor range is %d to %d\n",
658 mtu, mbuf_size, min_sop + min_data,
661 dev_info(enic, "Using %d rx descriptors (sop %d, data %d)\n",
662 nb_sop_desc + nb_data_desc, nb_sop_desc, nb_data_desc);
664 /* Allocate sop queue resources */
665 rc = vnic_rq_alloc(enic->vdev, rq_sop, sop_queue_idx,
666 nb_sop_desc, sizeof(struct rq_enet_desc));
668 dev_err(enic, "error in allocation of sop rq\n");
671 nb_sop_desc = rq_sop->ring.desc_count;
673 if (rq_data->in_use) {
674 /* Allocate data queue resources */
675 rc = vnic_rq_alloc(enic->vdev, rq_data, data_queue_idx,
677 sizeof(struct rq_enet_desc));
679 dev_err(enic, "error in allocation of data rq\n");
680 goto err_free_rq_sop;
682 nb_data_desc = rq_data->ring.desc_count;
684 rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
685 socket_id, nb_sop_desc + nb_data_desc,
686 sizeof(struct cq_enet_rq_desc));
688 dev_err(enic, "error in allocation of cq for rq\n");
689 goto err_free_rq_data;
692 /* Allocate the mbuf rings */
693 rq_sop->mbuf_ring = (struct rte_mbuf **)
694 rte_zmalloc_socket("rq->mbuf_ring",
695 sizeof(struct rte_mbuf *) * nb_sop_desc,
696 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
697 if (rq_sop->mbuf_ring == NULL)
700 if (rq_data->in_use) {
701 rq_data->mbuf_ring = (struct rte_mbuf **)
702 rte_zmalloc_socket("rq->mbuf_ring",
703 sizeof(struct rte_mbuf *) * nb_data_desc,
704 RTE_CACHE_LINE_SIZE, rq_sop->socket_id);
705 if (rq_data->mbuf_ring == NULL)
706 goto err_free_sop_mbuf;
709 rq_sop->tot_nb_desc = nb_desc; /* squirl away for MTU update function */
714 rte_free(rq_sop->mbuf_ring);
716 /* cleanup on error */
717 vnic_cq_free(&enic->cq[queue_idx]);
720 vnic_rq_free(rq_data);
722 vnic_rq_free(rq_sop);
727 void enic_free_wq(void *txq)
735 wq = (struct vnic_wq *)txq;
736 enic = vnic_dev_priv(wq->vdev);
737 rte_memzone_free(wq->cqmsg_rz);
739 vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
742 int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
743 unsigned int socket_id, uint16_t nb_desc)
746 struct vnic_wq *wq = &enic->wq[queue_idx];
747 unsigned int cq_index = enic_cq_wq(enic, queue_idx);
751 wq->socket_id = socket_id;
753 if (nb_desc > enic->config.wq_desc_count) {
755 "WQ %d - number of tx desc in cmd line (%d)"\
756 "is greater than that in the UCSM/CIMC adapter"\
757 "policy. Applying the value in the adapter "\
759 queue_idx, nb_desc, enic->config.wq_desc_count);
760 } else if (nb_desc != enic->config.wq_desc_count) {
761 enic->config.wq_desc_count = nb_desc;
763 "TX Queues - effective number of descs:%d\n",
768 /* Allocate queue resources */
769 err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx,
770 enic->config.wq_desc_count,
771 sizeof(struct wq_enet_desc));
773 dev_err(enic, "error in allocation of wq\n");
777 err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index,
778 socket_id, enic->config.wq_desc_count,
779 sizeof(struct cq_enet_wq_desc));
782 dev_err(enic, "error in allocation of cq for wq\n");
785 /* setup up CQ message */
786 snprintf((char *)name, sizeof(name),
787 "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
790 wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
800 int enic_disable(struct enic *enic)
805 vnic_intr_mask(&enic->intr);
806 (void)vnic_intr_masked(&enic->intr); /* flush write */
807 rte_intr_disable(&enic->pdev->intr_handle);
808 rte_intr_callback_unregister(&enic->pdev->intr_handle,
810 (void *)enic->rte_dev);
812 vnic_dev_disable(enic->vdev);
814 enic_clsf_destroy(enic);
816 if (!enic_is_sriov_vf(enic))
817 vnic_dev_del_addr(enic->vdev, enic->mac_addr);
819 for (i = 0; i < enic->wq_count; i++) {
820 err = vnic_wq_disable(&enic->wq[i]);
824 for (i = 0; i < enic_vnic_rq_count(enic); i++) {
825 if (enic->rq[i].in_use) {
826 err = vnic_rq_disable(&enic->rq[i]);
832 /* If we were using interrupts, set the interrupt vector to -1
833 * to disable interrupts. We are not disabling link notifcations,
834 * though, as we want the polling of link status to continue working.
836 if (enic->rte_dev->data->dev_conf.intr_conf.lsc)
837 vnic_dev_notify_set(enic->vdev, -1);
839 vnic_dev_set_reset_flag(enic->vdev, 1);
841 for (i = 0; i < enic->wq_count; i++)
842 vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
844 for (i = 0; i < enic_vnic_rq_count(enic); i++)
845 if (enic->rq[i].in_use)
846 vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
847 for (i = 0; i < enic->cq_count; i++)
848 vnic_cq_clean(&enic->cq[i]);
849 vnic_intr_clean(&enic->intr);
854 static int enic_dev_wait(struct vnic_dev *vdev,
855 int (*start)(struct vnic_dev *, int),
856 int (*finished)(struct vnic_dev *, int *),
863 err = start(vdev, arg);
867 /* Wait for func to complete...2 seconds max */
868 for (i = 0; i < 2000; i++) {
869 err = finished(vdev, &done);
879 static int enic_dev_open(struct enic *enic)
883 err = enic_dev_wait(enic->vdev, vnic_dev_open,
884 vnic_dev_open_done, 0);
886 dev_err(enic_get_dev(enic),
887 "vNIC device open failed, err %d\n", err);
892 static int enic_set_rsskey(struct enic *enic, uint8_t *user_key)
894 dma_addr_t rss_key_buf_pa;
895 union vnic_rss_key *rss_key_buf_va = NULL;
899 RTE_ASSERT(user_key != NULL);
900 snprintf((char *)name, NAME_MAX, "rss_key-%s", enic->bdf_name);
901 rss_key_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_key),
902 &rss_key_buf_pa, name);
906 for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++)
907 rss_key_buf_va->key[i / 10].b[i % 10] = user_key[i];
909 err = enic_set_rss_key(enic,
911 sizeof(union vnic_rss_key));
913 /* Save for later queries */
915 rte_memcpy(&enic->rss_key, rss_key_buf_va,
916 sizeof(union vnic_rss_key));
918 enic_free_consistent(enic, sizeof(union vnic_rss_key),
919 rss_key_buf_va, rss_key_buf_pa);
924 int enic_set_rss_reta(struct enic *enic, union vnic_rss_cpu *rss_cpu)
926 dma_addr_t rss_cpu_buf_pa;
927 union vnic_rss_cpu *rss_cpu_buf_va = NULL;
931 snprintf((char *)name, NAME_MAX, "rss_cpu-%s", enic->bdf_name);
932 rss_cpu_buf_va = enic_alloc_consistent(enic, sizeof(union vnic_rss_cpu),
933 &rss_cpu_buf_pa, name);
937 rte_memcpy(rss_cpu_buf_va, rss_cpu, sizeof(union vnic_rss_cpu));
939 err = enic_set_rss_cpu(enic,
941 sizeof(union vnic_rss_cpu));
943 enic_free_consistent(enic, sizeof(union vnic_rss_cpu),
944 rss_cpu_buf_va, rss_cpu_buf_pa);
946 /* Save for later queries */
948 rte_memcpy(&enic->rss_cpu, rss_cpu, sizeof(union vnic_rss_cpu));
952 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
953 u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
955 const u8 tso_ipid_split_en = 0;
958 err = enic_set_nic_cfg(enic,
959 rss_default_cpu, rss_hash_type,
960 rss_hash_bits, rss_base_cpu,
961 rss_enable, tso_ipid_split_en,
962 enic->ig_vlan_strip_en);
967 /* Initialize RSS with defaults, called from dev_configure */
968 int enic_init_rss_nic_cfg(struct enic *enic)
970 static uint8_t default_rss_key[] = {
971 85, 67, 83, 97, 119, 101, 115, 111, 109, 101,
972 80, 65, 76, 79, 117, 110, 105, 113, 117, 101,
973 76, 73, 78, 85, 88, 114, 111, 99, 107, 115,
974 69, 78, 73, 67, 105, 115, 99, 111, 111, 108,
976 struct rte_eth_rss_conf rss_conf;
977 union vnic_rss_cpu rss_cpu;
980 rss_conf = enic->rte_dev->data->dev_conf.rx_adv_conf.rss_conf;
982 * If setting key for the first time, and the user gives us none, then
983 * push the default key to NIC.
985 if (rss_conf.rss_key == NULL) {
986 rss_conf.rss_key = default_rss_key;
987 rss_conf.rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
989 ret = enic_set_rss_conf(enic, &rss_conf);
991 dev_err(enic, "Failed to configure RSS\n");
994 if (enic->rss_enable) {
995 /* If enabling RSS, use the default reta */
996 for (i = 0; i < ENIC_RSS_RETA_SIZE; i++) {
997 rss_cpu.cpu[i / 4].b[i % 4] =
998 enic_rte_rq_idx_to_sop_idx(i % enic->rq_count);
1000 ret = enic_set_rss_reta(enic, &rss_cpu);
1002 dev_err(enic, "Failed to set RSS indirection table\n");
1007 int enic_setup_finish(struct enic *enic)
1009 enic_init_soft_stats(enic);
1012 vnic_dev_packet_filter(enic->vdev,
1025 static int enic_rss_conf_valid(struct enic *enic,
1026 struct rte_eth_rss_conf *rss_conf)
1028 /* RSS is disabled per VIC settings. Ignore rss_conf. */
1029 if (enic->flow_type_rss_offloads == 0)
1031 if (rss_conf->rss_key != NULL &&
1032 rss_conf->rss_key_len != ENIC_RSS_HASH_KEY_SIZE) {
1033 dev_err(enic, "Given rss_key is %d bytes, it must be %d\n",
1034 rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
1037 if (rss_conf->rss_hf != 0 &&
1038 (rss_conf->rss_hf & enic->flow_type_rss_offloads) == 0) {
1039 dev_err(enic, "Given rss_hf contains none of the supported"
1046 /* Set hash type and key according to rss_conf */
1047 int enic_set_rss_conf(struct enic *enic, struct rte_eth_rss_conf *rss_conf)
1049 struct rte_eth_dev *eth_dev;
1055 RTE_ASSERT(rss_conf != NULL);
1056 ret = enic_rss_conf_valid(enic, rss_conf);
1058 dev_err(enic, "RSS configuration (rss_conf) is invalid\n");
1062 eth_dev = enic->rte_dev;
1064 rss_hf = rss_conf->rss_hf & enic->flow_type_rss_offloads;
1065 if (enic->rq_count > 1 &&
1066 (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) &&
1069 if (rss_hf & ETH_RSS_IPV4)
1070 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV4;
1071 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1072 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV4;
1073 if (rss_hf & ETH_RSS_IPV6)
1074 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6;
1075 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1076 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
1077 if (rss_hf & ETH_RSS_IPV6_EX)
1078 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_IPV6_EX;
1079 if (rss_hf & ETH_RSS_IPV6_TCP_EX)
1080 rss_hash_type |= NIC_CFG_RSS_HASH_TYPE_TCP_IPV6_EX;
1086 /* Set the hash key if provided */
1087 if (rss_enable && rss_conf->rss_key) {
1088 ret = enic_set_rsskey(enic, rss_conf->rss_key);
1090 dev_err(enic, "Failed to set RSS key\n");
1095 ret = enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, rss_hash_type,
1096 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1099 enic->rss_hf = rss_hf;
1100 enic->rss_hash_type = rss_hash_type;
1101 enic->rss_enable = rss_enable;
1106 int enic_set_vlan_strip(struct enic *enic)
1109 * Unfortunately, VLAN strip on/off and RSS on/off are configured
1110 * together. So, re-do niccfg, preserving the current RSS settings.
1112 return enic_set_niccfg(enic, ENIC_RSS_DEFAULT_CPU, enic->rss_hash_type,
1113 ENIC_RSS_HASH_BITS, ENIC_RSS_BASE_CPU,
1117 void enic_add_packet_filter(struct enic *enic)
1119 /* Args -> directed, multicast, broadcast, promisc, allmulti */
1120 vnic_dev_packet_filter(enic->vdev, 1, 1, 1,
1121 enic->promisc, enic->allmulti);
1124 int enic_get_link_status(struct enic *enic)
1126 return vnic_dev_link_status(enic->vdev);
1129 static void enic_dev_deinit(struct enic *enic)
1131 struct rte_eth_dev *eth_dev = enic->rte_dev;
1133 /* stop link status checking */
1134 vnic_dev_notify_unset(enic->vdev);
1136 rte_free(eth_dev->data->mac_addrs);
1143 int enic_set_vnic_res(struct enic *enic)
1145 struct rte_eth_dev *eth_dev = enic->rte_dev;
1147 unsigned int required_rq, required_wq, required_cq;
1149 /* Always use two vNIC RQs per eth_dev RQ, regardless of Rx scatter. */
1150 required_rq = eth_dev->data->nb_rx_queues * 2;
1151 required_wq = eth_dev->data->nb_tx_queues;
1152 required_cq = eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues;
1154 if (enic->conf_rq_count < required_rq) {
1155 dev_err(dev, "Not enough Receive queues. Requested:%u which uses %d RQs on VIC, Configured:%u\n",
1156 eth_dev->data->nb_rx_queues,
1157 required_rq, enic->conf_rq_count);
1160 if (enic->conf_wq_count < required_wq) {
1161 dev_err(dev, "Not enough Transmit queues. Requested:%u, Configured:%u\n",
1162 eth_dev->data->nb_tx_queues, enic->conf_wq_count);
1166 if (enic->conf_cq_count < required_cq) {
1167 dev_err(dev, "Not enough Completion queues. Required:%u, Configured:%u\n",
1168 required_cq, enic->conf_cq_count);
1173 enic->rq_count = eth_dev->data->nb_rx_queues;
1174 enic->wq_count = eth_dev->data->nb_tx_queues;
1175 enic->cq_count = enic->rq_count + enic->wq_count;
1181 /* Initialize the completion queue for an RQ */
1183 enic_reinit_rq(struct enic *enic, unsigned int rq_idx)
1185 struct vnic_rq *sop_rq, *data_rq;
1186 unsigned int cq_idx;
1189 sop_rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1190 data_rq = &enic->rq[enic_rte_rq_idx_to_data_idx(rq_idx)];
1193 vnic_cq_clean(&enic->cq[cq_idx]);
1194 vnic_cq_init(&enic->cq[cq_idx],
1195 0 /* flow_control_enable */,
1196 1 /* color_enable */,
1199 1 /* cq_tail_color */,
1200 0 /* interrupt_enable */,
1201 1 /* cq_entry_enable */,
1202 0 /* cq_message_enable */,
1203 0 /* interrupt offset */,
1204 0 /* cq_message_addr */);
1207 vnic_rq_init_start(sop_rq, enic_cq_rq(enic,
1208 enic_rte_rq_idx_to_sop_idx(rq_idx)), 0,
1209 sop_rq->ring.desc_count - 1, 1, 0);
1210 if (data_rq->in_use) {
1211 vnic_rq_init_start(data_rq,
1213 enic_rte_rq_idx_to_data_idx(rq_idx)), 0,
1214 data_rq->ring.desc_count - 1, 1, 0);
1217 rc = enic_alloc_rx_queue_mbufs(enic, sop_rq);
1221 if (data_rq->in_use) {
1222 rc = enic_alloc_rx_queue_mbufs(enic, data_rq);
1224 enic_rxmbuf_queue_release(enic, sop_rq);
1232 /* The Cisco NIC can send and receive packets up to a max packet size
1233 * determined by the NIC type and firmware. There is also an MTU
1234 * configured into the NIC via the CIMC/UCSM management interface
1235 * which can be overridden by this function (up to the max packet size).
1236 * Depending on the network setup, doing so may cause packet drops
1237 * and unexpected behavior.
1239 int enic_set_mtu(struct enic *enic, uint16_t new_mtu)
1241 unsigned int rq_idx;
1244 uint16_t old_mtu; /* previous setting */
1245 uint16_t config_mtu; /* Value configured into NIC via CIMC/UCSM */
1246 struct rte_eth_dev *eth_dev = enic->rte_dev;
1248 old_mtu = eth_dev->data->mtu;
1249 config_mtu = enic->config.mtu;
1251 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1252 return -E_RTE_SECONDARY;
1254 if (new_mtu > enic->max_mtu) {
1256 "MTU not updated: requested (%u) greater than max (%u)\n",
1257 new_mtu, enic->max_mtu);
1260 if (new_mtu < ENIC_MIN_MTU) {
1262 "MTU not updated: requested (%u) less than min (%u)\n",
1263 new_mtu, ENIC_MIN_MTU);
1266 if (new_mtu > config_mtu)
1268 "MTU (%u) is greater than value configured in NIC (%u)\n",
1269 new_mtu, config_mtu);
1271 /* The easy case is when scatter is disabled. However if the MTU
1272 * becomes greater than the mbuf data size, packet drops will ensue.
1274 if (!(enic->rte_dev->data->dev_conf.rxmode.offloads &
1275 DEV_RX_OFFLOAD_SCATTER)) {
1276 eth_dev->data->mtu = new_mtu;
1280 /* Rx scatter is enabled so reconfigure RQ's on the fly. The point is to
1281 * change Rx scatter mode if necessary for better performance. I.e. if
1282 * MTU was greater than the mbuf size and now it's less, scatter Rx
1283 * doesn't have to be used and vice versa.
1285 rte_spinlock_lock(&enic->mtu_lock);
1287 /* Stop traffic on all RQs */
1288 for (rq_idx = 0; rq_idx < enic->rq_count * 2; rq_idx++) {
1289 rq = &enic->rq[rq_idx];
1290 if (rq->is_sop && rq->in_use) {
1291 rc = enic_stop_rq(enic,
1292 enic_sop_rq_idx_to_rte_idx(rq_idx));
1294 dev_err(enic, "Failed to stop Rq %u\n", rq_idx);
1300 /* replace Rx function with a no-op to avoid getting stale pkts */
1301 eth_dev->rx_pkt_burst = enic_dummy_recv_pkts;
1304 /* Allow time for threads to exit the real Rx function. */
1307 /* now it is safe to reconfigure the RQs */
1309 /* update the mtu */
1310 eth_dev->data->mtu = new_mtu;
1312 /* free and reallocate RQs with the new MTU */
1313 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1314 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1317 rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp,
1318 rq->tot_nb_desc, rq->rx_free_thresh);
1321 "Fatal MTU alloc error- No traffic will pass\n");
1325 rc = enic_reinit_rq(enic, rq_idx);
1328 "Fatal MTU RQ reinit- No traffic will pass\n");
1333 /* put back the real receive function */
1335 eth_dev->rx_pkt_burst = enic_recv_pkts;
1338 /* restart Rx traffic */
1339 for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) {
1340 rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)];
1341 if (rq->is_sop && rq->in_use)
1342 enic_start_rq(enic, rq_idx);
1346 dev_info(enic, "MTU changed from %u to %u\n", old_mtu, new_mtu);
1347 rte_spinlock_unlock(&enic->mtu_lock);
1351 static int enic_dev_init(struct enic *enic)
1354 struct rte_eth_dev *eth_dev = enic->rte_dev;
1356 vnic_dev_intr_coal_timer_info_default(enic->vdev);
1358 /* Get vNIC configuration
1360 err = enic_get_vnic_config(enic);
1362 dev_err(dev, "Get vNIC configuration failed, aborting\n");
1366 /* Get available resource counts */
1367 enic_get_res_counts(enic);
1368 if (enic->conf_rq_count == 1) {
1369 dev_err(enic, "Running with only 1 RQ configured in the vNIC is not supported.\n");
1370 dev_err(enic, "Please configure 2 RQs in the vNIC for each Rx queue used by DPDK.\n");
1371 dev_err(enic, "See the ENIC PMD guide for more information.\n");
1374 /* Queue counts may be zeros. rte_zmalloc returns NULL in that case. */
1375 enic->cq = rte_zmalloc("enic_vnic_cq", sizeof(struct vnic_cq) *
1376 enic->conf_cq_count, 8);
1377 enic->rq = rte_zmalloc("enic_vnic_rq", sizeof(struct vnic_rq) *
1378 enic->conf_rq_count, 8);
1379 enic->wq = rte_zmalloc("enic_vnic_wq", sizeof(struct vnic_wq) *
1380 enic->conf_wq_count, 8);
1381 if (enic->conf_cq_count > 0 && enic->cq == NULL) {
1382 dev_err(enic, "failed to allocate vnic_cq, aborting.\n");
1385 if (enic->conf_rq_count > 0 && enic->rq == NULL) {
1386 dev_err(enic, "failed to allocate vnic_rq, aborting.\n");
1389 if (enic->conf_wq_count > 0 && enic->wq == NULL) {
1390 dev_err(enic, "failed to allocate vnic_wq, aborting.\n");
1394 /* Get the supported filters */
1395 enic_fdir_info(enic);
1397 eth_dev->data->mac_addrs = rte_zmalloc("enic_mac_addr", ETH_ALEN
1398 * ENIC_MAX_MAC_ADDR, 0);
1399 if (!eth_dev->data->mac_addrs) {
1400 dev_err(enic, "mac addr storage alloc failed, aborting.\n");
1403 ether_addr_copy((struct ether_addr *) enic->mac_addr,
1404 eth_dev->data->mac_addrs);
1406 vnic_dev_set_reset_flag(enic->vdev, 0);
1408 LIST_INIT(&enic->flows);
1409 rte_spinlock_init(&enic->flows_lock);
1411 /* set up link status checking */
1412 vnic_dev_notify_set(enic->vdev, -1); /* No Intr for notify */
1418 int enic_probe(struct enic *enic)
1420 struct rte_pci_device *pdev = enic->pdev;
1423 dev_debug(enic, " Initializing ENIC PMD\n");
1425 /* if this is a secondary process the hardware is already initialized */
1426 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1429 enic->bar0.vaddr = (void *)pdev->mem_resource[0].addr;
1430 enic->bar0.len = pdev->mem_resource[0].len;
1432 /* Register vNIC device */
1433 enic->vdev = vnic_dev_register(NULL, enic, enic->pdev, &enic->bar0, 1);
1435 dev_err(enic, "vNIC registration failed, aborting\n");
1439 LIST_INIT(&enic->memzone_list);
1440 rte_spinlock_init(&enic->memzone_list_lock);
1442 vnic_register_cbacks(enic->vdev,
1443 enic_alloc_consistent,
1444 enic_free_consistent);
1446 /* Issue device open to get device in known state */
1447 err = enic_dev_open(enic);
1449 dev_err(enic, "vNIC dev open failed, aborting\n");
1450 goto err_out_unregister;
1453 /* Set ingress vlan rewrite mode before vnic initialization */
1454 err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
1455 IG_VLAN_REWRITE_MODE_PASS_THRU);
1458 "Failed to set ingress vlan rewrite mode, aborting.\n");
1459 goto err_out_dev_close;
1462 /* Issue device init to initialize the vnic-to-switch link.
1463 * We'll start with carrier off and wait for link UP
1464 * notification later to turn on carrier. We don't need
1465 * to wait here for the vnic-to-switch link initialization
1466 * to complete; link UP notification is the indication that
1467 * the process is complete.
1470 err = vnic_dev_init(enic->vdev, 0);
1472 dev_err(enic, "vNIC dev init failed, aborting\n");
1473 goto err_out_dev_close;
1476 err = enic_dev_init(enic);
1478 dev_err(enic, "Device initialization failed, aborting\n");
1479 goto err_out_dev_close;
1485 vnic_dev_close(enic->vdev);
1487 vnic_dev_unregister(enic->vdev);
1492 void enic_remove(struct enic *enic)
1494 enic_dev_deinit(enic);
1495 vnic_dev_close(enic->vdev);
1496 vnic_dev_unregister(enic->vdev);