From 277310027965628ec8d24651d46f924fd53eea46 Mon Sep 17 00:00:00 2001 From: Xiaoyun Li Date: Tue, 2 Jul 2019 14:25:17 +0800 Subject: [PATCH] raw/ntb: introduce NTB raw device driver Introduce rawdev driver support for NTB (Non-transparent Bridge) which can help to connect two separate hosts with each other. Signed-off-by: Xiaoyun Li Acked-by: Jingjing Wu Reviewed-by: Xiaolong Ye --- MAINTAINERS | 6 + config/common_base | 5 + doc/guides/rawdevs/index.rst | 1 + doc/guides/rawdevs/ntb.rst | 22 ++ doc/guides/rel_notes/release_19_08.rst | 6 + drivers/raw/Makefile | 1 + drivers/raw/meson.build | 2 +- drivers/raw/ntb/Makefile | 27 ++ drivers/raw/ntb/meson.build | 7 + drivers/raw/ntb/ntb.c | 488 ++++++++++++++++++++++++ drivers/raw/ntb/ntb.h | 164 ++++++++ drivers/raw/ntb/rte_pmd_ntb_version.map | 4 + mk/rte.app.mk | 1 + 13 files changed, 733 insertions(+), 1 deletion(-) create mode 100644 doc/guides/rawdevs/ntb.rst create mode 100644 drivers/raw/ntb/Makefile create mode 100644 drivers/raw/ntb/meson.build create mode 100644 drivers/raw/ntb/ntb.c create mode 100644 drivers/raw/ntb/ntb.h create mode 100644 drivers/raw/ntb/rte_pmd_ntb_version.map diff --git a/MAINTAINERS b/MAINTAINERS index e35cc9a7a8..4aba96a374 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1111,6 +1111,12 @@ M: Vamsi Attunuru F: drivers/raw/octeontx2_dma/ F: doc/guides/rawdevs/octeontx2_dma.rst +NTB +M: Xiaoyun Li +M: Jingjing Wu +F: drivers/raw/ntb/ +F: doc/guides/rawdevs/ntb.rst + Packet processing ----------------- diff --git a/config/common_base b/config/common_base index 9ace507589..853c187cad 100644 --- a/config/common_base +++ b/config/common_base @@ -767,6 +767,11 @@ CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV=y # CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_DMA_RAWDEV=y +# +# Compile PMD for NTB raw device +# +CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV=y + # # Compile librte_ring # diff --git a/doc/guides/rawdevs/index.rst b/doc/guides/rawdevs/index.rst index 413c543239..fad951637a 100644 --- a/doc/guides/rawdevs/index.rst +++ b/doc/guides/rawdevs/index.rst @@ -15,4 +15,5 @@ application through rawdev API. dpaa2_qdma ifpga_rawdev ioat_rawdev + ntb octeontx2_dma diff --git a/doc/guides/rawdevs/ntb.rst b/doc/guides/rawdevs/ntb.rst new file mode 100644 index 0000000000..9559ea4b03 --- /dev/null +++ b/doc/guides/rawdevs/ntb.rst @@ -0,0 +1,22 @@ +.. SPDX-License-Identifier: BSD-3-Clause + Copyright(c) 2018 Intel Corporation. + +NTB Rawdev Driver +================= + +The ``ntb`` rawdev driver provides a non-transparent bridge between two +separate hosts so that they can communicate with each other. Thus, many +user cases can benefit from this, such as fault tolerance and visual +acceleration. + +Build Options +------------- + +- ``CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV`` (default ``y``) + + Toggle compilation of the ``ntb`` driver. + +Limitation +---------- + +- The FIFO hasn't been introduced and will come in 19.11 release. diff --git a/doc/guides/rel_notes/release_19_08.rst b/doc/guides/rel_notes/release_19_08.rst index 21934bf01d..8cd90e0c3a 100644 --- a/doc/guides/rel_notes/release_19_08.rst +++ b/doc/guides/rel_notes/release_19_08.rst @@ -129,6 +129,12 @@ New Features of via software, reducing cycles spent copying large blocks of data in applications. +* **Introduced NTB PMD.** + + Added a PMD for Intel NTB (Non-transparent Bridge). This PMD implemented + handshake between two separate hosts and can share local memory for peer + host to directly access. + * **Updated telemetry library for global metrics support.** Updated ``librte_telemetry`` to fetch the global metrics from the diff --git a/drivers/raw/Makefile b/drivers/raw/Makefile index 5be58057aa..879fcca615 100644 --- a/drivers/raw/Makefile +++ b/drivers/raw/Makefile @@ -11,6 +11,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_PMD_DPAA2_QDMA_RAWDEV) += dpaa2_qdma endif DIRS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) += ifpga_rawdev DIRS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV) += ioat +DIRS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += ntb DIRS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_DMA_RAWDEV) += octeontx2_dma include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/drivers/raw/meson.build b/drivers/raw/meson.build index 64db944560..c2d034862a 100644 --- a/drivers/raw/meson.build +++ b/drivers/raw/meson.build @@ -2,7 +2,7 @@ # Copyright 2018 NXP drivers = ['dpaa2_cmdif', 'dpaa2_qdma', - 'ifpga_rawdev', 'ioat', + 'ifpga_rawdev', 'ioat', 'ntb', 'octeontx2_dma', 'skeleton_rawdev'] std_deps = ['rawdev'] diff --git a/drivers/raw/ntb/Makefile b/drivers/raw/ntb/Makefile new file mode 100644 index 0000000000..ec53edd33f --- /dev/null +++ b/drivers/raw/ntb/Makefile @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2019 Intel Corporation + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +LIB = librte_pmd_ntb.a + +CFLAGS += -DALLOW_EXPERIMENTAL_API +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool +LDLIBS += -lrte_pci -lrte_bus_pci +LDLIBS += -lrte_rawdev + +EXPORT_MAP := rte_pmd_ntb_version.map + +LIBABIVER := 1 + +# +# all source are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += ntb.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/raw/ntb/meson.build b/drivers/raw/ntb/meson.build new file mode 100644 index 0000000000..0fb44452cc --- /dev/null +++ b/drivers/raw/ntb/meson.build @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2019 Intel Corporation. + +deps += ['rawdev', 'mbuf', 'mempool', + 'pci', 'bus_pci'] +sources = files('ntb.c') +allow_experimental_apis = true diff --git a/drivers/raw/ntb/ntb.c b/drivers/raw/ntb/ntb.c new file mode 100644 index 0000000000..2ae3cbea0f --- /dev/null +++ b/drivers/raw/ntb/ntb.c @@ -0,0 +1,488 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2019 Intel Corporation. + */ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ntb.h" + +int ntb_logtype; + +static const struct rte_pci_id pci_id_ntb_map[] = { + { .vendor_id = 0, /* sentinel */ }, +}; + +static void +ntb_queue_conf_get(struct rte_rawdev *dev __rte_unused, + uint16_t queue_id __rte_unused, + rte_rawdev_obj_t queue_conf __rte_unused) +{ +} + +static int +ntb_queue_setup(struct rte_rawdev *dev __rte_unused, + uint16_t queue_id __rte_unused, + rte_rawdev_obj_t queue_conf __rte_unused) +{ + return 0; +} + +static int +ntb_queue_release(struct rte_rawdev *dev __rte_unused, + uint16_t queue_id __rte_unused) +{ + return 0; +} + +static uint16_t +ntb_queue_count(struct rte_rawdev *dev) +{ + struct ntb_hw *hw = dev->dev_private; + return hw->queue_pairs; +} + +static int +ntb_enqueue_bufs(struct rte_rawdev *dev, + struct rte_rawdev_buf **buffers, + unsigned int count, + rte_rawdev_obj_t context) +{ + RTE_SET_USED(dev); + RTE_SET_USED(buffers); + RTE_SET_USED(count); + RTE_SET_USED(context); + + return 0; +} + +static int +ntb_dequeue_bufs(struct rte_rawdev *dev, + struct rte_rawdev_buf **buffers, + unsigned int count, + rte_rawdev_obj_t context) +{ + RTE_SET_USED(dev); + RTE_SET_USED(buffers); + RTE_SET_USED(count); + RTE_SET_USED(context); + + return 0; +} + +static void +ntb_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info) +{ + struct ntb_hw *hw = dev->dev_private; + struct ntb_attr *ntb_attrs = dev_info; + + strncpy(ntb_attrs[NTB_TOPO_ID].name, NTB_TOPO_NAME, NTB_ATTR_NAME_LEN); + switch (hw->topo) { + case NTB_TOPO_B2B_DSD: + strncpy(ntb_attrs[NTB_TOPO_ID].value, "B2B DSD", + NTB_ATTR_VAL_LEN); + break; + case NTB_TOPO_B2B_USD: + strncpy(ntb_attrs[NTB_TOPO_ID].value, "B2B USD", + NTB_ATTR_VAL_LEN); + break; + default: + strncpy(ntb_attrs[NTB_TOPO_ID].value, "Unsupported", + NTB_ATTR_VAL_LEN); + } + + strncpy(ntb_attrs[NTB_LINK_STATUS_ID].name, NTB_LINK_STATUS_NAME, + NTB_ATTR_NAME_LEN); + snprintf(ntb_attrs[NTB_LINK_STATUS_ID].value, NTB_ATTR_VAL_LEN, + "%d", hw->link_status); + + strncpy(ntb_attrs[NTB_SPEED_ID].name, NTB_SPEED_NAME, + NTB_ATTR_NAME_LEN); + snprintf(ntb_attrs[NTB_SPEED_ID].value, NTB_ATTR_VAL_LEN, + "%d", hw->link_speed); + + strncpy(ntb_attrs[NTB_WIDTH_ID].name, NTB_WIDTH_NAME, + NTB_ATTR_NAME_LEN); + snprintf(ntb_attrs[NTB_WIDTH_ID].value, NTB_ATTR_VAL_LEN, + "%d", hw->link_width); + + strncpy(ntb_attrs[NTB_MW_CNT_ID].name, NTB_MW_CNT_NAME, + NTB_ATTR_NAME_LEN); + snprintf(ntb_attrs[NTB_MW_CNT_ID].value, NTB_ATTR_VAL_LEN, + "%d", hw->mw_cnt); + + strncpy(ntb_attrs[NTB_DB_CNT_ID].name, NTB_DB_CNT_NAME, + NTB_ATTR_NAME_LEN); + snprintf(ntb_attrs[NTB_DB_CNT_ID].value, NTB_ATTR_VAL_LEN, + "%d", hw->db_cnt); + + strncpy(ntb_attrs[NTB_SPAD_CNT_ID].name, NTB_SPAD_CNT_NAME, + NTB_ATTR_NAME_LEN); + snprintf(ntb_attrs[NTB_SPAD_CNT_ID].value, NTB_ATTR_VAL_LEN, + "%d", hw->spad_cnt); +} + +static int +ntb_dev_configure(const struct rte_rawdev *dev __rte_unused, + rte_rawdev_obj_t config __rte_unused) +{ + return 0; +} + +static int +ntb_dev_start(struct rte_rawdev *dev) +{ + /* TODO: init queues and start queues. */ + dev->started = 1; + + return 0; +} + +static void +ntb_dev_stop(struct rte_rawdev *dev) +{ + /* TODO: stop rx/tx queues. */ + dev->started = 0; +} + +static int +ntb_dev_close(struct rte_rawdev *dev) +{ + int ret = 0; + + if (dev->started) + ntb_dev_stop(dev); + + /* TODO: free queues. */ + + return ret; +} + +static int +ntb_dev_reset(struct rte_rawdev *rawdev __rte_unused) +{ + return 0; +} + +static int +ntb_attr_set(struct rte_rawdev *dev, const char *attr_name, + uint64_t attr_value) +{ + struct ntb_hw *hw = dev->dev_private; + int index = 0; + + if (dev == NULL || attr_name == NULL) { + NTB_LOG(ERR, "Invalid arguments for setting attributes"); + return -EINVAL; + } + + if (!strncmp(attr_name, NTB_SPAD_USER, NTB_SPAD_USER_LEN)) { + if (hw->ntb_ops->spad_write == NULL) + return -ENOTSUP; + index = atoi(&attr_name[NTB_SPAD_USER_LEN]); + (*hw->ntb_ops->spad_write)(dev, hw->spad_user_list[index], + 1, attr_value); + NTB_LOG(INFO, "Set attribute (%s) Value (%" PRIu64 ")", + attr_name, attr_value); + return 0; + } + + /* Attribute not found. */ + NTB_LOG(ERR, "Attribute not found."); + return -EINVAL; +} + +static int +ntb_attr_get(struct rte_rawdev *dev, const char *attr_name, + uint64_t *attr_value) +{ + struct ntb_hw *hw = dev->dev_private; + int index = 0; + + if (dev == NULL || attr_name == NULL || attr_value == NULL) { + NTB_LOG(ERR, "Invalid arguments for getting attributes"); + return -EINVAL; + } + + if (!strncmp(attr_name, NTB_TOPO_NAME, NTB_ATTR_NAME_LEN)) { + *attr_value = hw->topo; + NTB_LOG(INFO, "Attribute (%s) Value (%" PRIu64 ")", + attr_name, *attr_value); + return 0; + } + + if (!strncmp(attr_name, NTB_LINK_STATUS_NAME, NTB_ATTR_NAME_LEN)) { + *attr_value = hw->link_status; + NTB_LOG(INFO, "Attribute (%s) Value (%" PRIu64 ")", + attr_name, *attr_value); + return 0; + } + + if (!strncmp(attr_name, NTB_SPEED_NAME, NTB_ATTR_NAME_LEN)) { + *attr_value = hw->link_speed; + NTB_LOG(INFO, "Attribute (%s) Value (%" PRIu64 ")", + attr_name, *attr_value); + return 0; + } + + if (!strncmp(attr_name, NTB_WIDTH_NAME, NTB_ATTR_NAME_LEN)) { + *attr_value = hw->link_width; + NTB_LOG(INFO, "Attribute (%s) Value (%" PRIu64 ")", + attr_name, *attr_value); + return 0; + } + + if (!strncmp(attr_name, NTB_MW_CNT_NAME, NTB_ATTR_NAME_LEN)) { + *attr_value = hw->mw_cnt; + NTB_LOG(INFO, "Attribute (%s) Value (%" PRIu64 ")", + attr_name, *attr_value); + return 0; + } + + if (!strncmp(attr_name, NTB_DB_CNT_NAME, NTB_ATTR_NAME_LEN)) { + *attr_value = hw->db_cnt; + NTB_LOG(INFO, "Attribute (%s) Value (%" PRIu64 ")", + attr_name, *attr_value); + return 0; + } + + if (!strncmp(attr_name, NTB_SPAD_CNT_NAME, NTB_ATTR_NAME_LEN)) { + *attr_value = hw->spad_cnt; + NTB_LOG(INFO, "Attribute (%s) Value (%" PRIu64 ")", + attr_name, *attr_value); + return 0; + } + + if (!strncmp(attr_name, NTB_SPAD_USER, NTB_SPAD_USER_LEN)) { + if (hw->ntb_ops->spad_read == NULL) + return -ENOTSUP; + index = atoi(&attr_name[NTB_SPAD_USER_LEN]); + *attr_value = (*hw->ntb_ops->spad_read)(dev, + hw->spad_user_list[index], 0); + NTB_LOG(INFO, "Attribute (%s) Value (%" PRIu64 ")", + attr_name, *attr_value); + return 0; + } + + /* Attribute not found. */ + NTB_LOG(ERR, "Attribute not found."); + return -EINVAL; +} + +static int +ntb_xstats_get(const struct rte_rawdev *dev __rte_unused, + const unsigned int ids[] __rte_unused, + uint64_t values[] __rte_unused, + unsigned int n __rte_unused) +{ + return 0; +} + +static int +ntb_xstats_get_names(const struct rte_rawdev *dev __rte_unused, + struct rte_rawdev_xstats_name *xstats_names __rte_unused, + unsigned int size __rte_unused) +{ + return 0; +} + +static uint64_t +ntb_xstats_get_by_name(const struct rte_rawdev *dev __rte_unused, + const char *name __rte_unused, + unsigned int *id __rte_unused) +{ + return 0; +} + +static int +ntb_xstats_reset(struct rte_rawdev *dev __rte_unused, + const uint32_t ids[] __rte_unused, + uint32_t nb_ids __rte_unused) +{ + return 0; +} + +static const struct rte_rawdev_ops ntb_ops = { + .dev_info_get = ntb_dev_info_get, + .dev_configure = ntb_dev_configure, + .dev_start = ntb_dev_start, + .dev_stop = ntb_dev_stop, + .dev_close = ntb_dev_close, + .dev_reset = ntb_dev_reset, + + .queue_def_conf = ntb_queue_conf_get, + .queue_setup = ntb_queue_setup, + .queue_release = ntb_queue_release, + .queue_count = ntb_queue_count, + + .enqueue_bufs = ntb_enqueue_bufs, + .dequeue_bufs = ntb_dequeue_bufs, + + .attr_get = ntb_attr_get, + .attr_set = ntb_attr_set, + + .xstats_get = ntb_xstats_get, + .xstats_get_names = ntb_xstats_get_names, + .xstats_get_by_name = ntb_xstats_get_by_name, + .xstats_reset = ntb_xstats_reset, +}; + +static int +ntb_init_hw(struct rte_rawdev *dev, struct rte_pci_device *pci_dev) +{ + struct ntb_hw *hw = dev->dev_private; + int ret; + + hw->pci_dev = pci_dev; + hw->peer_dev_up = 0; + hw->link_status = NTB_LINK_DOWN; + hw->link_speed = NTB_SPEED_NONE; + hw->link_width = NTB_WIDTH_NONE; + + switch (pci_dev->id.device_id) { + default: + NTB_LOG(ERR, "Not supported device."); + return -EINVAL; + } + + if (hw->ntb_ops->ntb_dev_init == NULL) + return -ENOTSUP; + ret = (*hw->ntb_ops->ntb_dev_init)(dev); + if (ret) { + NTB_LOG(ERR, "Unable to init ntb dev."); + return ret; + } + + if (hw->ntb_ops->set_link == NULL) + return -ENOTSUP; + ret = (*hw->ntb_ops->set_link)(dev, 1); + if (ret) + return ret; + + return ret; +} + +static int +ntb_create(struct rte_pci_device *pci_dev, int socket_id) +{ + char name[RTE_RAWDEV_NAME_MAX_LEN]; + struct rte_rawdev *rawdev = NULL; + int ret; + + if (pci_dev == NULL) { + NTB_LOG(ERR, "Invalid pci_dev."); + ret = -EINVAL; + } + + memset(name, 0, sizeof(name)); + snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "NTB:%x:%02x.%x", + pci_dev->addr.bus, pci_dev->addr.devid, + pci_dev->addr.function); + + NTB_LOG(INFO, "Init %s on NUMA node %d", name, socket_id); + + /* Allocate device structure. */ + rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct ntb_hw), + socket_id); + if (rawdev == NULL) { + NTB_LOG(ERR, "Unable to allocate rawdev."); + ret = -EINVAL; + } + + rawdev->dev_ops = &ntb_ops; + rawdev->device = &pci_dev->device; + rawdev->driver_name = pci_dev->driver->driver.name; + + ret = ntb_init_hw(rawdev, pci_dev); + if (ret < 0) { + NTB_LOG(ERR, "Unable to init ntb hw."); + goto fail; + } + + return ret; + +fail: + if (rawdev) + rte_rawdev_pmd_release(rawdev); + + return ret; +} + +static int +ntb_destroy(struct rte_pci_device *pci_dev) +{ + char name[RTE_RAWDEV_NAME_MAX_LEN]; + struct rte_rawdev *rawdev; + int ret; + + if (pci_dev == NULL) { + NTB_LOG(ERR, "Invalid pci_dev."); + ret = -EINVAL; + return ret; + } + + memset(name, 0, sizeof(name)); + snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "NTB:%x:%02x.%x", + pci_dev->addr.bus, pci_dev->addr.devid, + pci_dev->addr.function); + + NTB_LOG(INFO, "Closing %s on NUMA node %d", name, rte_socket_id()); + + rawdev = rte_rawdev_pmd_get_named_dev(name); + if (rawdev == NULL) { + NTB_LOG(ERR, "Invalid device name (%s)", name); + ret = -EINVAL; + return ret; + } + + ret = rte_rawdev_pmd_release(rawdev); + if (ret) + NTB_LOG(ERR, "Failed to destroy ntb rawdev."); + + return ret; +} + +static int +ntb_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) +{ + return ntb_create(pci_dev, rte_socket_id()); +} + +static int +ntb_remove(struct rte_pci_device *pci_dev) +{ + return ntb_destroy(pci_dev); +} + + +static struct rte_pci_driver rte_ntb_pmd = { + .id_table = pci_id_ntb_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = ntb_probe, + .remove = ntb_remove, +}; + +RTE_PMD_REGISTER_PCI(raw_ntb, rte_ntb_pmd); +RTE_PMD_REGISTER_PCI_TABLE(raw_ntb, pci_id_ntb_map); +RTE_PMD_REGISTER_KMOD_DEP(raw_ntb, "* igb_uio | uio_pci_generic | vfio-pci"); + +RTE_INIT(ntb_init_log) +{ + ntb_logtype = rte_log_register("pmd.raw.ntb"); + if (ntb_logtype >= 0) + rte_log_set_level(ntb_logtype, RTE_LOG_DEBUG); +} diff --git a/drivers/raw/ntb/ntb.h b/drivers/raw/ntb/ntb.h new file mode 100644 index 0000000000..d355231b05 --- /dev/null +++ b/drivers/raw/ntb/ntb.h @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2019 Intel Corporation. + */ + +#ifndef _NTB_RAWDEV_H_ +#define _NTB_RAWDEV_H_ + +#include + +extern int ntb_logtype; + +#define NTB_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, ntb_logtype, "%s(): " fmt "\n", \ + __func__, ##args) + +/* Vendor ID */ +#define NTB_INTEL_VENDOR_ID 0x8086 + +/* Device IDs */ +#define NTB_INTEL_DEV_ID_B2B_SKX 0x201C + +#define NTB_TOPO_NAME "topo" +#define NTB_LINK_STATUS_NAME "link_status" +#define NTB_SPEED_NAME "speed" +#define NTB_WIDTH_NAME "width" +#define NTB_MW_CNT_NAME "mw_count" +#define NTB_DB_CNT_NAME "db_count" +#define NTB_SPAD_CNT_NAME "spad_count" +/* Reserved to app to use. */ +#define NTB_SPAD_USER "spad_user_" +#define NTB_SPAD_USER_LEN (sizeof(NTB_SPAD_USER) - 1) +#define NTB_SPAD_USER_MAX_NUM 10 +#define NTB_ATTR_NAME_LEN 30 +#define NTB_ATTR_VAL_LEN 30 +#define NTB_ATTR_MAX 20 + +/* NTB Attributes */ +struct ntb_attr { + /**< Name of the attribute */ + char name[NTB_ATTR_NAME_LEN]; + /**< Value or reference of value of attribute */ + char value[NTB_ATTR_NAME_LEN]; +}; + +enum ntb_attr_idx { + NTB_TOPO_ID = 0, + NTB_LINK_STATUS_ID, + NTB_SPEED_ID, + NTB_WIDTH_ID, + NTB_MW_CNT_ID, + NTB_DB_CNT_ID, + NTB_SPAD_CNT_ID, +}; + +enum ntb_topo { + NTB_TOPO_NONE = 0, + NTB_TOPO_B2B_USD, + NTB_TOPO_B2B_DSD, +}; + +enum ntb_link { + NTB_LINK_DOWN = 0, + NTB_LINK_UP, +}; + +enum ntb_speed { + NTB_SPEED_NONE = 0, + NTB_SPEED_GEN1 = 1, + NTB_SPEED_GEN2 = 2, + NTB_SPEED_GEN3 = 3, + NTB_SPEED_GEN4 = 4, +}; + +enum ntb_width { + NTB_WIDTH_NONE = 0, + NTB_WIDTH_1 = 1, + NTB_WIDTH_2 = 2, + NTB_WIDTH_4 = 4, + NTB_WIDTH_8 = 8, + NTB_WIDTH_12 = 12, + NTB_WIDTH_16 = 16, + NTB_WIDTH_32 = 32, +}; + +/* Define spad registers usage. 0 is reserved. */ +enum ntb_spad_idx { + SPAD_NUM_MWS = 1, + SPAD_NUM_QPS, + SPAD_Q_SZ, + SPAD_MW0_SZ_H, + SPAD_MW0_SZ_L, + SPAD_MW1_SZ_H, + SPAD_MW1_SZ_L, +}; + +/** + * NTB device operations + * @ntb_dev_init: Init ntb dev. + * @get_peer_mw_addr: To get the addr of peer mw[mw_idx]. + * @mw_set_trans: Set translation of internal memory that remote can access. + * @get_link_status: get link status, link speed and link width. + * @set_link: Set local side up/down. + * @spad_read: Read local/peer spad register val. + * @spad_write: Write val to local/peer spad register. + * @db_read: Read doorbells status. + * @db_clear: Clear local doorbells. + * @db_set_mask: Set bits in db mask, preventing db interrpts generated + * for those db bits. + * @peer_db_set: Set doorbell bit to generate peer interrupt for that bit. + * @vector_bind: Bind vector source [intr] to msix vector [msix]. + */ +struct ntb_dev_ops { + int (*ntb_dev_init)(struct rte_rawdev *dev); + void *(*get_peer_mw_addr)(struct rte_rawdev *dev, int mw_idx); + int (*mw_set_trans)(struct rte_rawdev *dev, int mw_idx, + uint64_t addr, uint64_t size); + int (*get_link_status)(struct rte_rawdev *dev); + int (*set_link)(struct rte_rawdev *dev, bool up); + uint32_t (*spad_read)(struct rte_rawdev *dev, int spad, bool peer); + int (*spad_write)(struct rte_rawdev *dev, int spad, + bool peer, uint32_t spad_v); + uint64_t (*db_read)(struct rte_rawdev *dev); + int (*db_clear)(struct rte_rawdev *dev, uint64_t db_bits); + int (*db_set_mask)(struct rte_rawdev *dev, uint64_t db_mask); + int (*peer_db_set)(struct rte_rawdev *dev, uint8_t db_bit); + int (*vector_bind)(struct rte_rawdev *dev, uint8_t intr, uint8_t msix); +}; + +/* ntb private data. */ +struct ntb_hw { + uint8_t mw_cnt; + uint8_t peer_mw_cnt; + uint8_t db_cnt; + uint8_t spad_cnt; + + uint64_t db_valid_mask; + uint64_t db_mask; + + enum ntb_topo topo; + + enum ntb_link link_status; + enum ntb_speed link_speed; + enum ntb_width link_width; + + const struct ntb_dev_ops *ntb_ops; + + struct rte_pci_device *pci_dev; + char *hw_addr; + + uint64_t *mw_size; + uint64_t *peer_mw_size; + uint8_t peer_dev_up; + + uint16_t queue_pairs; + uint16_t queue_size; + + /**< mem zone to populate RX ring. */ + const struct rte_memzone **mz; + + /* Reserve several spad for app to use. */ + int spad_user_list[NTB_SPAD_USER_MAX_NUM]; +}; + +#endif /* _NTB_RAWDEV_H_ */ diff --git a/drivers/raw/ntb/rte_pmd_ntb_version.map b/drivers/raw/ntb/rte_pmd_ntb_version.map new file mode 100644 index 0000000000..8861484fb3 --- /dev/null +++ b/drivers/raw/ntb/rte_pmd_ntb_version.map @@ -0,0 +1,4 @@ +DPDK_19.08 { + + local: *; +}; diff --git a/mk/rte.app.mk b/mk/rte.app.mk index 15a6f74215..bbf9771709 100644 --- a/mk/rte.app.mk +++ b/mk/rte.app.mk @@ -311,6 +311,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IFPGA_RAWDEV) += -lrte_pmd_ifpga_rawdev _LDLIBS-$(CONFIG_RTE_LIBRTE_IPN3KE_PMD) += -lrte_pmd_ipn3ke endif # CONFIG_RTE_LIBRTE_IFPGA_BUS _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_IOAT_RAWDEV) += -lrte_pmd_ioat_rawdev +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NTB_RAWDEV) += -lrte_pmd_ntb _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX2_DMA_RAWDEV) += -lrte_pmd_octeontx2_dma endif # CONFIG_RTE_LIBRTE_RAWDEV -- 2.20.1