1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019 Intel Corporation.
9 #include <rte_common.h>
10 #include <rte_lcore.h>
11 #include <rte_cycles.h>
16 #include <rte_bus_pci.h>
17 #include <rte_memzone.h>
18 #include <rte_memcpy.h>
19 #include <rte_rawdev.h>
20 #include <rte_rawdev_pmd.h>
22 #include "ntb_hw_intel.h"
23 #include "rte_pmd_ntb.h"
28 static const struct rte_pci_id pci_id_ntb_map[] = {
29 { RTE_PCI_DEVICE(NTB_INTEL_VENDOR_ID, NTB_INTEL_DEV_ID_B2B_SKX) },
30 { .vendor_id = 0, /* sentinel */ },
33 /* Align with enum ntb_xstats_idx */
34 static struct rte_rawdev_xstats_name ntb_xstats_names[] = {
42 #define NTB_XSTATS_NUM RTE_DIM(ntb_xstats_names)
45 ntb_link_cleanup(struct rte_rawdev *dev)
47 struct ntb_hw *hw = dev->dev_private;
50 if (hw->ntb_ops->spad_write == NULL ||
51 hw->ntb_ops->mw_set_trans == NULL) {
52 NTB_LOG(ERR, "Not supported to clean up link.");
56 /* Clean spad registers. */
57 for (i = 0; i < hw->spad_cnt; i++) {
58 status = (*hw->ntb_ops->spad_write)(dev, i, 0, 0);
60 NTB_LOG(ERR, "Failed to clean local spad.");
63 /* Clear mw so that peer cannot access local memory.*/
64 for (i = 0; i < hw->used_mw_num; i++) {
65 status = (*hw->ntb_ops->mw_set_trans)(dev, i, 0, 0);
67 NTB_LOG(ERR, "Failed to clean mw.");
72 ntb_handshake_work(const struct rte_rawdev *dev)
74 struct ntb_hw *hw = dev->dev_private;
78 if (hw->ntb_ops->spad_write == NULL ||
79 hw->ntb_ops->mw_set_trans == NULL) {
80 NTB_LOG(ERR, "Scratchpad/MW setting is not supported.");
84 /* Tell peer the mw info of local side. */
85 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_NUM_MWS, 1, hw->mw_cnt);
88 for (i = 0; i < hw->mw_cnt; i++) {
89 NTB_LOG(INFO, "Local %u mw size: 0x%"PRIx64"", i,
91 val = hw->mw_size[i] >> 32;
92 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_SZ_H + 2 * i,
97 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_SZ_L + 2 * i,
103 /* Tell peer about the queue info and map memory to the peer. */
104 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_Q_SZ, 1, hw->queue_size);
107 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_NUM_QPS, 1,
111 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_USED_MWS, 1,
115 for (i = 0; i < hw->used_mw_num; i++) {
116 val = (uint64_t)(size_t)(hw->mz[i]->addr) >> 32;
117 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_BA_H + 2 * i,
121 val = (uint64_t)(size_t)(hw->mz[i]->addr);
122 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_BA_L + 2 * i,
128 for (i = 0; i < hw->used_mw_num; i++) {
129 ret = (*hw->ntb_ops->mw_set_trans)(dev, i, hw->mz[i]->iova,
135 /* Ring doorbell 0 to tell peer the device is ready. */
136 ret = (*hw->ntb_ops->peer_db_set)(dev, 0);
144 ntb_dev_intr_handler(void *param)
146 struct rte_rawdev *dev = (struct rte_rawdev *)param;
147 struct ntb_hw *hw = dev->dev_private;
148 uint32_t val_h, val_l;
149 uint64_t peer_mw_size;
150 uint64_t db_bits = 0;
154 if (hw->ntb_ops->db_read == NULL ||
155 hw->ntb_ops->db_clear == NULL ||
156 hw->ntb_ops->peer_db_set == NULL) {
157 NTB_LOG(ERR, "Doorbell is not supported.");
161 db_bits = (*hw->ntb_ops->db_read)(dev);
163 NTB_LOG(ERR, "No doorbells");
165 /* Doorbell 0 is for peer device ready. */
167 NTB_LOG(INFO, "DB0: Peer device is up.");
168 /* Clear received doorbell. */
169 (*hw->ntb_ops->db_clear)(dev, 1);
172 * Peer dev is already up. All mw settings are already done.
178 if (hw->ntb_ops->spad_read == NULL) {
179 NTB_LOG(ERR, "Scratchpad read is not supported.");
183 /* Check if mw setting on the peer is the same as local. */
184 peer_mw_cnt = (*hw->ntb_ops->spad_read)(dev, SPAD_NUM_MWS, 0);
185 if (peer_mw_cnt != hw->mw_cnt) {
186 NTB_LOG(ERR, "Both mw cnt must be the same.");
190 for (i = 0; i < hw->mw_cnt; i++) {
191 val_h = (*hw->ntb_ops->spad_read)
192 (dev, SPAD_MW0_SZ_H + 2 * i, 0);
193 val_l = (*hw->ntb_ops->spad_read)
194 (dev, SPAD_MW0_SZ_L + 2 * i, 0);
195 peer_mw_size = ((uint64_t)val_h << 32) | val_l;
196 NTB_LOG(DEBUG, "Peer %u mw size: 0x%"PRIx64"", i,
198 if (peer_mw_size != hw->mw_size[i]) {
199 NTB_LOG(ERR, "Mw config must be the same.");
207 * Handshake with peer. Spad_write & mw_set_trans only works
208 * when both devices are up. So write spad again when db is
209 * received. And set db again for the later device who may miss
212 if (ntb_handshake_work(dev) < 0) {
213 NTB_LOG(ERR, "Handshake work failed.");
217 /* To get the link info. */
218 if (hw->ntb_ops->get_link_status == NULL) {
219 NTB_LOG(ERR, "Not supported to get link status.");
222 (*hw->ntb_ops->get_link_status)(dev);
223 NTB_LOG(INFO, "Link is up. Link speed: %u. Link width: %u",
224 hw->link_speed, hw->link_width);
228 if (db_bits & (1 << 1)) {
229 NTB_LOG(INFO, "DB1: Peer device is down.");
230 /* Clear received doorbell. */
231 (*hw->ntb_ops->db_clear)(dev, 2);
233 /* Peer device will be down, So clean local side too. */
234 ntb_link_cleanup(dev);
237 /* Response peer's dev_stop request. */
238 (*hw->ntb_ops->peer_db_set)(dev, 2);
242 if (db_bits & (1 << 2)) {
243 NTB_LOG(INFO, "DB2: Peer device agrees dev to be down.");
244 /* Clear received doorbell. */
245 (*hw->ntb_ops->db_clear)(dev, (1 << 2));
252 ntb_queue_conf_get(struct rte_rawdev *dev,
254 rte_rawdev_obj_t queue_conf)
256 struct ntb_queue_conf *q_conf = queue_conf;
257 struct ntb_hw *hw = dev->dev_private;
259 q_conf->tx_free_thresh = hw->tx_queues[queue_id]->tx_free_thresh;
260 q_conf->nb_desc = hw->rx_queues[queue_id]->nb_rx_desc;
261 q_conf->rx_mp = hw->rx_queues[queue_id]->mpool;
265 ntb_rxq_release_mbufs(struct ntb_rx_queue *q)
269 if (!q || !q->sw_ring) {
270 NTB_LOG(ERR, "Pointer to rxq or sw_ring is NULL");
274 for (i = 0; i < q->nb_rx_desc; i++) {
275 if (q->sw_ring[i].mbuf) {
276 rte_pktmbuf_free_seg(q->sw_ring[i].mbuf);
277 q->sw_ring[i].mbuf = NULL;
283 ntb_rxq_release(struct ntb_rx_queue *rxq)
286 NTB_LOG(ERR, "Pointer to rxq is NULL");
290 ntb_rxq_release_mbufs(rxq);
292 rte_free(rxq->sw_ring);
297 ntb_rxq_setup(struct rte_rawdev *dev,
299 rte_rawdev_obj_t queue_conf)
301 struct ntb_queue_conf *rxq_conf = queue_conf;
302 struct ntb_hw *hw = dev->dev_private;
303 struct ntb_rx_queue *rxq;
305 /* Allocate the rx queue data structure */
306 rxq = rte_zmalloc_socket("ntb rx queue",
307 sizeof(struct ntb_rx_queue),
311 NTB_LOG(ERR, "Failed to allocate memory for "
312 "rx queue data structure.");
316 if (rxq_conf->rx_mp == NULL) {
317 NTB_LOG(ERR, "Invalid null mempool pointer.");
320 rxq->nb_rx_desc = rxq_conf->nb_desc;
321 rxq->mpool = rxq_conf->rx_mp;
322 rxq->port_id = dev->dev_id;
323 rxq->queue_id = qp_id;
326 /* Allocate the software ring. */
328 rte_zmalloc_socket("ntb rx sw ring",
329 sizeof(struct ntb_rx_entry) *
334 ntb_rxq_release(rxq);
336 NTB_LOG(ERR, "Failed to allocate memory for SW ring");
340 hw->rx_queues[qp_id] = rxq;
346 ntb_txq_release_mbufs(struct ntb_tx_queue *q)
350 if (!q || !q->sw_ring) {
351 NTB_LOG(ERR, "Pointer to txq or sw_ring is NULL");
355 for (i = 0; i < q->nb_tx_desc; i++) {
356 if (q->sw_ring[i].mbuf) {
357 rte_pktmbuf_free_seg(q->sw_ring[i].mbuf);
358 q->sw_ring[i].mbuf = NULL;
364 ntb_txq_release(struct ntb_tx_queue *txq)
367 NTB_LOG(ERR, "Pointer to txq is NULL");
371 ntb_txq_release_mbufs(txq);
373 rte_free(txq->sw_ring);
378 ntb_txq_setup(struct rte_rawdev *dev,
380 rte_rawdev_obj_t queue_conf)
382 struct ntb_queue_conf *txq_conf = queue_conf;
383 struct ntb_hw *hw = dev->dev_private;
384 struct ntb_tx_queue *txq;
387 /* Allocate the TX queue data structure. */
388 txq = rte_zmalloc_socket("ntb tx queue",
389 sizeof(struct ntb_tx_queue),
393 NTB_LOG(ERR, "Failed to allocate memory for "
394 "tx queue structure");
398 txq->nb_tx_desc = txq_conf->nb_desc;
399 txq->port_id = dev->dev_id;
400 txq->queue_id = qp_id;
403 /* Allocate software ring */
405 rte_zmalloc_socket("ntb tx sw ring",
406 sizeof(struct ntb_tx_entry) *
411 ntb_txq_release(txq);
413 NTB_LOG(ERR, "Failed to allocate memory for SW TX ring");
417 prev = txq->nb_tx_desc - 1;
418 for (i = 0; i < txq->nb_tx_desc; i++) {
419 txq->sw_ring[i].mbuf = NULL;
420 txq->sw_ring[i].last_id = i;
421 txq->sw_ring[prev].next_id = i;
425 txq->tx_free_thresh = txq_conf->tx_free_thresh ?
426 txq_conf->tx_free_thresh :
427 NTB_DFLT_TX_FREE_THRESH;
428 if (txq->tx_free_thresh >= txq->nb_tx_desc - 3) {
429 NTB_LOG(ERR, "tx_free_thresh must be less than nb_desc - 3. "
430 "(tx_free_thresh=%u qp_id=%u)", txq->tx_free_thresh,
435 hw->tx_queues[qp_id] = txq;
442 ntb_queue_setup(struct rte_rawdev *dev,
444 rte_rawdev_obj_t queue_conf)
446 struct ntb_hw *hw = dev->dev_private;
449 if (queue_id >= hw->queue_pairs)
452 ret = ntb_txq_setup(dev, queue_id, queue_conf);
456 ret = ntb_rxq_setup(dev, queue_id, queue_conf);
462 ntb_queue_release(struct rte_rawdev *dev, uint16_t queue_id)
464 struct ntb_hw *hw = dev->dev_private;
466 if (queue_id >= hw->queue_pairs)
469 ntb_txq_release(hw->tx_queues[queue_id]);
470 hw->tx_queues[queue_id] = NULL;
471 ntb_rxq_release(hw->rx_queues[queue_id]);
472 hw->rx_queues[queue_id] = NULL;
478 ntb_queue_count(struct rte_rawdev *dev)
480 struct ntb_hw *hw = dev->dev_private;
481 return hw->queue_pairs;
485 ntb_queue_init(struct rte_rawdev *dev, uint16_t qp_id)
487 struct ntb_hw *hw = dev->dev_private;
488 struct ntb_rx_queue *rxq = hw->rx_queues[qp_id];
489 struct ntb_tx_queue *txq = hw->tx_queues[qp_id];
490 volatile struct ntb_header *local_hdr;
491 struct ntb_header *remote_hdr;
492 uint16_t q_size = hw->queue_size;
497 if (hw->ntb_ops->get_peer_mw_addr == NULL) {
498 NTB_LOG(ERR, "Getting peer mw addr is not supported.");
502 /* Put queue info into the start of shared memory. */
503 hdr_offset = hw->hdr_size_per_queue * qp_id;
504 local_hdr = (volatile struct ntb_header *)
505 ((size_t)hw->mz[0]->addr + hdr_offset);
506 bar_addr = (*hw->ntb_ops->get_peer_mw_addr)(dev, 0);
507 if (bar_addr == NULL)
509 remote_hdr = (struct ntb_header *)
510 ((size_t)bar_addr + hdr_offset);
513 rxq->rx_desc_ring = (struct ntb_desc *)
514 (&remote_hdr->desc_ring);
515 rxq->rx_used_ring = (volatile struct ntb_used *)
516 (&local_hdr->desc_ring[q_size]);
517 rxq->avail_cnt = &remote_hdr->avail_cnt;
518 rxq->used_cnt = &local_hdr->used_cnt;
520 for (i = 0; i < rxq->nb_rx_desc - 1; i++) {
521 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mpool);
522 if (unlikely(!mbuf)) {
523 NTB_LOG(ERR, "Failed to allocate mbuf for RX");
526 mbuf->port = dev->dev_id;
528 rxq->sw_ring[i].mbuf = mbuf;
530 rxq->rx_desc_ring[i].addr = rte_pktmbuf_mtod(mbuf, size_t);
531 rxq->rx_desc_ring[i].len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM;
534 *rxq->avail_cnt = rxq->nb_rx_desc - 1;
535 rxq->last_avail = rxq->nb_rx_desc - 1;
539 txq->tx_desc_ring = (volatile struct ntb_desc *)
540 (&local_hdr->desc_ring);
541 txq->tx_used_ring = (struct ntb_used *)
542 (&remote_hdr->desc_ring[q_size]);
543 txq->avail_cnt = &local_hdr->avail_cnt;
544 txq->used_cnt = &remote_hdr->used_cnt;
550 txq->nb_tx_free = txq->nb_tx_desc - 1;
552 /* Set per queue stats. */
553 for (i = 0; i < NTB_XSTATS_NUM; i++) {
554 hw->ntb_xstats[i + NTB_XSTATS_NUM * (qp_id + 1)] = 0;
555 hw->ntb_xstats_off[i + NTB_XSTATS_NUM * (qp_id + 1)] = 0;
562 ntb_enqueue_bufs(struct rte_rawdev *dev,
563 struct rte_rawdev_buf **buffers,
565 rte_rawdev_obj_t context)
567 /* Not FIFO right now. Just for testing memory write. */
568 struct ntb_hw *hw = dev->dev_private;
573 if (hw->ntb_ops->get_peer_mw_addr == NULL)
575 bar_addr = (*hw->ntb_ops->get_peer_mw_addr)(dev, 0);
576 size = (size_t)context;
578 for (i = 0; i < count; i++)
579 rte_memcpy(bar_addr, buffers[i]->buf_addr, size);
584 ntb_dequeue_bufs(struct rte_rawdev *dev,
585 struct rte_rawdev_buf **buffers,
587 rte_rawdev_obj_t context)
589 /* Not FIFO. Just for testing memory read. */
590 struct ntb_hw *hw = dev->dev_private;
594 size = (size_t)context;
596 for (i = 0; i < count; i++)
597 rte_memcpy(buffers[i]->buf_addr, hw->mz[i]->addr, size);
602 ntb_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info)
604 struct ntb_hw *hw = dev->dev_private;
605 struct ntb_dev_info *info = dev_info;
607 info->mw_cnt = hw->mw_cnt;
608 info->mw_size = hw->mw_size;
611 * Intel hardware requires that mapped memory base address should be
612 * aligned with EMBARSZ and needs continuous memzone.
614 info->mw_size_align = (uint8_t)(hw->pci_dev->id.vendor_id ==
615 NTB_INTEL_VENDOR_ID);
617 if (!hw->queue_size || !hw->queue_pairs) {
618 NTB_LOG(ERR, "No queue size and queue num assigned.");
622 hw->hdr_size_per_queue = RTE_ALIGN(sizeof(struct ntb_header) +
623 hw->queue_size * sizeof(struct ntb_desc) +
624 hw->queue_size * sizeof(struct ntb_used),
625 RTE_CACHE_LINE_SIZE);
626 info->ntb_hdr_size = hw->hdr_size_per_queue * hw->queue_pairs;
630 ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config)
632 struct ntb_dev_config *conf = config;
633 struct ntb_hw *hw = dev->dev_private;
637 hw->queue_pairs = conf->num_queues;
638 hw->queue_size = conf->queue_size;
639 hw->used_mw_num = conf->mz_num;
640 hw->mz = conf->mz_list;
641 hw->rx_queues = rte_zmalloc("ntb_rx_queues",
642 sizeof(struct ntb_rx_queue *) * hw->queue_pairs, 0);
643 hw->tx_queues = rte_zmalloc("ntb_tx_queues",
644 sizeof(struct ntb_tx_queue *) * hw->queue_pairs, 0);
645 /* First total stats, then per queue stats. */
646 xstats_num = (hw->queue_pairs + 1) * NTB_XSTATS_NUM;
647 hw->ntb_xstats = rte_zmalloc("ntb_xstats", xstats_num *
648 sizeof(uint64_t), 0);
649 hw->ntb_xstats_off = rte_zmalloc("ntb_xstats_off", xstats_num *
650 sizeof(uint64_t), 0);
652 /* Start handshake with the peer. */
653 ret = ntb_handshake_work(dev);
655 rte_free(hw->rx_queues);
656 rte_free(hw->tx_queues);
657 hw->rx_queues = NULL;
658 hw->tx_queues = NULL;
666 ntb_dev_start(struct rte_rawdev *dev)
668 struct ntb_hw *hw = dev->dev_private;
669 uint32_t peer_base_l, peer_val;
670 uint64_t peer_base_h;
674 if (!hw->link_status || !hw->peer_dev_up)
677 /* Set total stats. */
678 for (i = 0; i < NTB_XSTATS_NUM; i++) {
679 hw->ntb_xstats[i] = 0;
680 hw->ntb_xstats_off[i] = 0;
683 for (i = 0; i < hw->queue_pairs; i++) {
684 ret = ntb_queue_init(dev, i);
686 NTB_LOG(ERR, "Failed to init queue.");
691 hw->peer_mw_base = rte_zmalloc("ntb_peer_mw_base", hw->mw_cnt *
692 sizeof(uint64_t), 0);
694 if (hw->ntb_ops->spad_read == NULL) {
699 peer_val = (*hw->ntb_ops->spad_read)(dev, SPAD_Q_SZ, 0);
700 if (peer_val != hw->queue_size) {
701 NTB_LOG(ERR, "Inconsistent queue size! (local: %u peer: %u)",
702 hw->queue_size, peer_val);
707 peer_val = (*hw->ntb_ops->spad_read)(dev, SPAD_NUM_QPS, 0);
708 if (peer_val != hw->queue_pairs) {
709 NTB_LOG(ERR, "Inconsistent number of queues! (local: %u peer:"
710 " %u)", hw->queue_pairs, peer_val);
715 hw->peer_used_mws = (*hw->ntb_ops->spad_read)(dev, SPAD_USED_MWS, 0);
717 for (i = 0; i < hw->peer_used_mws; i++) {
718 peer_base_h = (*hw->ntb_ops->spad_read)(dev,
719 SPAD_MW0_BA_H + 2 * i, 0);
720 peer_base_l = (*hw->ntb_ops->spad_read)(dev,
721 SPAD_MW0_BA_L + 2 * i, 0);
722 hw->peer_mw_base[i] = (peer_base_h << 32) + peer_base_l;
730 rte_free(hw->peer_mw_base);
732 for (i = 0; i < hw->queue_pairs; i++) {
733 ntb_rxq_release_mbufs(hw->rx_queues[i]);
734 ntb_txq_release_mbufs(hw->tx_queues[i]);
741 ntb_dev_stop(struct rte_rawdev *dev)
743 struct ntb_hw *hw = dev->dev_private;
747 if (!hw->peer_dev_up)
750 ntb_link_cleanup(dev);
752 /* Notify the peer that device will be down. */
753 if (hw->ntb_ops->peer_db_set == NULL) {
754 NTB_LOG(ERR, "Peer doorbell setting is not supported.");
757 status = (*hw->ntb_ops->peer_db_set)(dev, 1);
759 NTB_LOG(ERR, "Failed to tell peer device is down.");
764 * Set time out as 1s in case that the peer is stopped accidently
765 * without any notification.
769 /* Wait for cleanup work down before db mask clear. */
770 while (hw->peer_dev_up && time_out) {
776 /* Clear doorbells mask. */
777 if (hw->ntb_ops->db_set_mask == NULL) {
778 NTB_LOG(ERR, "Doorbell mask setting is not supported.");
781 status = (*hw->ntb_ops->db_set_mask)(dev,
782 (((uint64_t)1 << hw->db_cnt) - 1));
784 NTB_LOG(ERR, "Failed to clear doorbells.");
786 for (i = 0; i < hw->queue_pairs; i++) {
787 ntb_rxq_release_mbufs(hw->rx_queues[i]);
788 ntb_txq_release_mbufs(hw->tx_queues[i]);
795 ntb_dev_close(struct rte_rawdev *dev)
797 struct ntb_hw *hw = dev->dev_private;
798 struct rte_intr_handle *intr_handle;
805 for (i = 0; i < hw->queue_pairs; i++)
806 ntb_queue_release(dev, i);
809 intr_handle = &hw->pci_dev->intr_handle;
810 /* Clean datapath event and vec mapping */
811 rte_intr_efd_disable(intr_handle);
812 if (intr_handle->intr_vec) {
813 rte_free(intr_handle->intr_vec);
814 intr_handle->intr_vec = NULL;
816 /* Disable uio intr before callback unregister */
817 rte_intr_disable(intr_handle);
819 /* Unregister callback func to eal lib */
820 rte_intr_callback_unregister(intr_handle,
821 ntb_dev_intr_handler, dev);
827 ntb_dev_reset(struct rte_rawdev *rawdev __rte_unused)
833 ntb_attr_set(struct rte_rawdev *dev, const char *attr_name,
839 if (dev == NULL || attr_name == NULL) {
840 NTB_LOG(ERR, "Invalid arguments for setting attributes");
844 hw = dev->dev_private;
846 if (!strncmp(attr_name, NTB_SPAD_USER, NTB_SPAD_USER_LEN)) {
847 if (hw->ntb_ops->spad_write == NULL)
849 index = atoi(&attr_name[NTB_SPAD_USER_LEN]);
850 (*hw->ntb_ops->spad_write)(dev, hw->spad_user_list[index],
852 NTB_LOG(DEBUG, "Set attribute (%s) Value (%" PRIu64 ")",
853 attr_name, attr_value);
857 if (!strncmp(attr_name, NTB_QUEUE_SZ_NAME, NTB_ATTR_NAME_LEN)) {
858 hw->queue_size = attr_value;
859 NTB_LOG(DEBUG, "Set attribute (%s) Value (%" PRIu64 ")",
860 attr_name, attr_value);
864 if (!strncmp(attr_name, NTB_QUEUE_NUM_NAME, NTB_ATTR_NAME_LEN)) {
865 hw->queue_pairs = attr_value;
866 NTB_LOG(DEBUG, "Set attribute (%s) Value (%" PRIu64 ")",
867 attr_name, attr_value);
871 /* Attribute not found. */
872 NTB_LOG(ERR, "Attribute not found.");
877 ntb_attr_get(struct rte_rawdev *dev, const char *attr_name,
878 uint64_t *attr_value)
883 if (dev == NULL || attr_name == NULL || attr_value == NULL) {
884 NTB_LOG(ERR, "Invalid arguments for getting attributes");
888 hw = dev->dev_private;
890 if (!strncmp(attr_name, NTB_TOPO_NAME, NTB_ATTR_NAME_LEN)) {
891 *attr_value = hw->topo;
892 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")",
893 attr_name, *attr_value);
897 if (!strncmp(attr_name, NTB_LINK_STATUS_NAME, NTB_ATTR_NAME_LEN)) {
898 /* hw->link_status only indicates hw link status. */
899 *attr_value = hw->link_status && hw->peer_dev_up;
900 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")",
901 attr_name, *attr_value);
905 if (!strncmp(attr_name, NTB_SPEED_NAME, NTB_ATTR_NAME_LEN)) {
906 *attr_value = hw->link_speed;
907 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")",
908 attr_name, *attr_value);
912 if (!strncmp(attr_name, NTB_WIDTH_NAME, NTB_ATTR_NAME_LEN)) {
913 *attr_value = hw->link_width;
914 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")",
915 attr_name, *attr_value);
919 if (!strncmp(attr_name, NTB_MW_CNT_NAME, NTB_ATTR_NAME_LEN)) {
920 *attr_value = hw->mw_cnt;
921 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")",
922 attr_name, *attr_value);
926 if (!strncmp(attr_name, NTB_DB_CNT_NAME, NTB_ATTR_NAME_LEN)) {
927 *attr_value = hw->db_cnt;
928 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")",
929 attr_name, *attr_value);
933 if (!strncmp(attr_name, NTB_SPAD_CNT_NAME, NTB_ATTR_NAME_LEN)) {
934 *attr_value = hw->spad_cnt;
935 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")",
936 attr_name, *attr_value);
940 if (!strncmp(attr_name, NTB_SPAD_USER, NTB_SPAD_USER_LEN)) {
941 if (hw->ntb_ops->spad_read == NULL)
943 index = atoi(&attr_name[NTB_SPAD_USER_LEN]);
944 *attr_value = (*hw->ntb_ops->spad_read)(dev,
945 hw->spad_user_list[index], 0);
946 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")",
947 attr_name, *attr_value);
951 /* Attribute not found. */
952 NTB_LOG(ERR, "Attribute not found.");
956 static inline uint64_t
957 ntb_stats_update(uint64_t offset, uint64_t stat)
960 return (stat - offset);
962 return (uint64_t)(((uint64_t)-1) - offset + stat + 1);
966 ntb_xstats_get(const struct rte_rawdev *dev,
967 const unsigned int ids[],
971 struct ntb_hw *hw = dev->dev_private;
972 uint32_t i, j, off, xstats_num;
974 /* Calculate total stats of all queues. */
975 for (i = 0; i < NTB_XSTATS_NUM; i++) {
976 hw->ntb_xstats[i] = 0;
977 for (j = 0; j < hw->queue_pairs; j++) {
978 off = NTB_XSTATS_NUM * (j + 1) + i;
980 ntb_stats_update(hw->ntb_xstats_off[off],
981 hw->ntb_xstats[off]);
985 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1);
986 for (i = 0; i < n && ids[i] < xstats_num; i++) {
987 if (ids[i] < NTB_XSTATS_NUM)
988 values[i] = hw->ntb_xstats[ids[i]];
991 ntb_stats_update(hw->ntb_xstats_off[ids[i]],
992 hw->ntb_xstats[ids[i]]);
999 ntb_xstats_get_names(const struct rte_rawdev *dev,
1000 struct rte_rawdev_xstats_name *xstats_names,
1003 struct ntb_hw *hw = dev->dev_private;
1004 uint32_t xstats_num, i, j, off;
1006 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1);
1007 if (xstats_names == NULL || size < xstats_num)
1010 /* Total stats names */
1011 memcpy(xstats_names, ntb_xstats_names, sizeof(ntb_xstats_names));
1013 /* Queue stats names */
1014 for (i = 0; i < hw->queue_pairs; i++) {
1015 for (j = 0; j < NTB_XSTATS_NUM; j++) {
1016 off = j + (i + 1) * NTB_XSTATS_NUM;
1017 snprintf(xstats_names[off].name,
1018 sizeof(xstats_names[0].name),
1019 "%s_q%u", ntb_xstats_names[j].name, i);
1027 ntb_xstats_get_by_name(const struct rte_rawdev *dev,
1028 const char *name, unsigned int *id)
1030 struct rte_rawdev_xstats_name *xstats_names;
1031 struct ntb_hw *hw = dev->dev_private;
1032 uint32_t xstats_num, i, j, off;
1037 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1);
1038 xstats_names = rte_zmalloc("ntb_stats_name",
1039 sizeof(struct rte_rawdev_xstats_name) *
1041 ntb_xstats_get_names(dev, xstats_names, xstats_num);
1043 /* Calculate total stats of all queues. */
1044 for (i = 0; i < NTB_XSTATS_NUM; i++) {
1045 for (j = 0; j < hw->queue_pairs; j++) {
1046 off = NTB_XSTATS_NUM * (j + 1) + i;
1047 hw->ntb_xstats[i] +=
1048 ntb_stats_update(hw->ntb_xstats_off[off],
1049 hw->ntb_xstats[off]);
1053 for (i = 0; i < xstats_num; i++) {
1054 if (!strncmp(name, xstats_names[i].name,
1055 RTE_RAW_DEV_XSTATS_NAME_SIZE)) {
1057 rte_free(xstats_names);
1058 if (i < NTB_XSTATS_NUM)
1059 return hw->ntb_xstats[i];
1061 return ntb_stats_update(hw->ntb_xstats_off[i],
1066 NTB_LOG(ERR, "Cannot find the xstats name.");
1072 ntb_xstats_reset(struct rte_rawdev *dev,
1073 const uint32_t ids[],
1076 struct ntb_hw *hw = dev->dev_private;
1077 uint32_t i, j, off, xstats_num;
1079 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1);
1080 for (i = 0; i < nb_ids && ids[i] < xstats_num; i++) {
1081 if (ids[i] < NTB_XSTATS_NUM) {
1082 for (j = 0; j < hw->queue_pairs; j++) {
1083 off = NTB_XSTATS_NUM * (j + 1) + ids[i];
1084 hw->ntb_xstats_off[off] = hw->ntb_xstats[off];
1087 hw->ntb_xstats_off[ids[i]] = hw->ntb_xstats[ids[i]];
1094 static const struct rte_rawdev_ops ntb_ops = {
1095 .dev_info_get = ntb_dev_info_get,
1096 .dev_configure = ntb_dev_configure,
1097 .dev_start = ntb_dev_start,
1098 .dev_stop = ntb_dev_stop,
1099 .dev_close = ntb_dev_close,
1100 .dev_reset = ntb_dev_reset,
1102 .queue_def_conf = ntb_queue_conf_get,
1103 .queue_setup = ntb_queue_setup,
1104 .queue_release = ntb_queue_release,
1105 .queue_count = ntb_queue_count,
1107 .enqueue_bufs = ntb_enqueue_bufs,
1108 .dequeue_bufs = ntb_dequeue_bufs,
1110 .attr_get = ntb_attr_get,
1111 .attr_set = ntb_attr_set,
1113 .xstats_get = ntb_xstats_get,
1114 .xstats_get_names = ntb_xstats_get_names,
1115 .xstats_get_by_name = ntb_xstats_get_by_name,
1116 .xstats_reset = ntb_xstats_reset,
1120 ntb_init_hw(struct rte_rawdev *dev, struct rte_pci_device *pci_dev)
1122 struct ntb_hw *hw = dev->dev_private;
1123 struct rte_intr_handle *intr_handle;
1126 hw->pci_dev = pci_dev;
1127 hw->peer_dev_up = 0;
1128 hw->link_status = NTB_LINK_DOWN;
1129 hw->link_speed = NTB_SPEED_NONE;
1130 hw->link_width = NTB_WIDTH_NONE;
1132 switch (pci_dev->id.device_id) {
1133 case NTB_INTEL_DEV_ID_B2B_SKX:
1134 hw->ntb_ops = &intel_ntb_ops;
1137 NTB_LOG(ERR, "Not supported device.");
1141 if (hw->ntb_ops->ntb_dev_init == NULL)
1143 ret = (*hw->ntb_ops->ntb_dev_init)(dev);
1145 NTB_LOG(ERR, "Unable to init ntb dev.");
1149 if (hw->ntb_ops->set_link == NULL)
1151 ret = (*hw->ntb_ops->set_link)(dev, 1);
1155 /* Init doorbell. */
1156 hw->db_valid_mask = RTE_LEN2MASK(hw->db_cnt, uint64_t);
1158 intr_handle = &pci_dev->intr_handle;
1159 /* Register callback func to eal lib */
1160 rte_intr_callback_register(intr_handle,
1161 ntb_dev_intr_handler, dev);
1163 ret = rte_intr_efd_enable(intr_handle, hw->db_cnt);
1167 /* To clarify, the interrupt for each doorbell is already mapped
1168 * by default for intel gen3. They are mapped to msix vec 1-32,
1169 * and hardware intr is mapped to 0. Map all to 0 for uio.
1171 if (!rte_intr_cap_multiple(intr_handle)) {
1172 for (i = 0; i < hw->db_cnt; i++) {
1173 if (hw->ntb_ops->vector_bind == NULL)
1175 ret = (*hw->ntb_ops->vector_bind)(dev, i, 0);
1181 if (hw->ntb_ops->db_set_mask == NULL ||
1182 hw->ntb_ops->peer_db_set == NULL) {
1183 NTB_LOG(ERR, "Doorbell is not supported.");
1187 ret = (*hw->ntb_ops->db_set_mask)(dev, hw->db_mask);
1189 NTB_LOG(ERR, "Unable to enable intr for all dbs.");
1193 /* enable uio intr after callback register */
1194 rte_intr_enable(intr_handle);
1200 ntb_create(struct rte_pci_device *pci_dev, int socket_id)
1202 char name[RTE_RAWDEV_NAME_MAX_LEN];
1203 struct rte_rawdev *rawdev = NULL;
1206 if (pci_dev == NULL) {
1207 NTB_LOG(ERR, "Invalid pci_dev.");
1211 memset(name, 0, sizeof(name));
1212 snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "NTB:%x:%02x.%x",
1213 pci_dev->addr.bus, pci_dev->addr.devid,
1214 pci_dev->addr.function);
1216 NTB_LOG(INFO, "Init %s on NUMA node %d", name, socket_id);
1218 /* Allocate device structure. */
1219 rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct ntb_hw),
1221 if (rawdev == NULL) {
1222 NTB_LOG(ERR, "Unable to allocate rawdev.");
1226 rawdev->dev_ops = &ntb_ops;
1227 rawdev->device = &pci_dev->device;
1228 rawdev->driver_name = pci_dev->driver->driver.name;
1230 ret = ntb_init_hw(rawdev, pci_dev);
1232 NTB_LOG(ERR, "Unable to init ntb hw.");
1240 rte_rawdev_pmd_release(rawdev);
1246 ntb_destroy(struct rte_pci_device *pci_dev)
1248 char name[RTE_RAWDEV_NAME_MAX_LEN];
1249 struct rte_rawdev *rawdev;
1252 if (pci_dev == NULL) {
1253 NTB_LOG(ERR, "Invalid pci_dev.");
1258 memset(name, 0, sizeof(name));
1259 snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "NTB:%x:%02x.%x",
1260 pci_dev->addr.bus, pci_dev->addr.devid,
1261 pci_dev->addr.function);
1263 NTB_LOG(INFO, "Closing %s on NUMA node %d", name, rte_socket_id());
1265 rawdev = rte_rawdev_pmd_get_named_dev(name);
1266 if (rawdev == NULL) {
1267 NTB_LOG(ERR, "Invalid device name (%s)", name);
1272 ret = rte_rawdev_pmd_release(rawdev);
1274 NTB_LOG(ERR, "Failed to destroy ntb rawdev.");
1280 ntb_probe(struct rte_pci_driver *pci_drv __rte_unused,
1281 struct rte_pci_device *pci_dev)
1283 return ntb_create(pci_dev, rte_socket_id());
1287 ntb_remove(struct rte_pci_device *pci_dev)
1289 return ntb_destroy(pci_dev);
1293 static struct rte_pci_driver rte_ntb_pmd = {
1294 .id_table = pci_id_ntb_map,
1295 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
1297 .remove = ntb_remove,
1300 RTE_PMD_REGISTER_PCI(raw_ntb, rte_ntb_pmd);
1301 RTE_PMD_REGISTER_PCI_TABLE(raw_ntb, pci_id_ntb_map);
1302 RTE_PMD_REGISTER_KMOD_DEP(raw_ntb, "* igb_uio | uio_pci_generic | vfio-pci");
1304 RTE_INIT(ntb_init_log)
1306 ntb_logtype = rte_log_register("pmd.raw.ntb");
1307 if (ntb_logtype >= 0)
1308 rte_log_set_level(ntb_logtype, RTE_LOG_INFO);