From e33ad06eaeebecfdb1a7851fb130cb4a69f893e8 Mon Sep 17 00:00:00 2001 From: Kevin Laatz Date: Wed, 20 Oct 2021 16:29:59 +0000 Subject: [PATCH] dma/idxd: add skeleton for VFIO based DSA device Add the basic device probe/remove skeleton code for DSA device bound to the vfio pci driver. Relevant documentation and MAINTAINERS update also included. Signed-off-by: Bruce Richardson Signed-off-by: Kevin Laatz Reviewed-by: Conor Walsh --- MAINTAINERS | 10 +++++ doc/guides/dmadevs/idxd.rst | 58 ++++++++++++++++++++++++++ doc/guides/dmadevs/index.rst | 2 + doc/guides/rel_notes/release_21_11.rst | 5 +++ drivers/dma/idxd/idxd_common.c | 11 +++++ drivers/dma/idxd/idxd_internal.h | 27 ++++++++++++ drivers/dma/idxd/idxd_pci.c | 55 ++++++++++++++++++++++++ drivers/dma/idxd/meson.build | 11 +++++ drivers/dma/idxd/version.map | 3 ++ drivers/dma/meson.build | 2 + 10 files changed, 184 insertions(+) create mode 100644 doc/guides/dmadevs/idxd.rst create mode 100644 drivers/dma/idxd/idxd_common.c create mode 100644 drivers/dma/idxd/idxd_internal.h create mode 100644 drivers/dma/idxd/idxd_pci.c create mode 100644 drivers/dma/idxd/meson.build create mode 100644 drivers/dma/idxd/version.map diff --git a/MAINTAINERS b/MAINTAINERS index d04e3d710f..9ba150059d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1201,6 +1201,16 @@ F: doc/guides/compressdevs/zlib.rst F: doc/guides/compressdevs/features/zlib.ini +DMAdev Drivers +-------------- + +Intel IDXD - EXPERIMENTAL +M: Bruce Richardson +M: Kevin Laatz +F: drivers/dma/idxd/ +F: doc/guides/dmadevs/idxd.rst + + RegEx Drivers ------------- diff --git a/doc/guides/dmadevs/idxd.rst b/doc/guides/dmadevs/idxd.rst new file mode 100644 index 0000000000..924700d17e --- /dev/null +++ b/doc/guides/dmadevs/idxd.rst @@ -0,0 +1,58 @@ +.. SPDX-License-Identifier: BSD-3-Clause + Copyright(c) 2021 Intel Corporation. + +.. include:: + +IDXD DMA Device Driver +====================== + +The ``idxd`` dmadev driver provides a poll-mode driver (PMD) for Intel\ |reg| +Data Streaming Accelerator `(Intel DSA) +`_. +This PMD can be used in conjunction with Intel\ |reg| DSA devices to offload +data operations, such as data copies, to hardware, freeing up CPU cycles for +other tasks. + +Hardware Requirements +---------------------- + +The ``dpdk-devbind.py`` script, included with DPDK, can be used to show the +presence of supported hardware. Running ``dpdk-devbind.py --status-dev dma`` +will show all the DMA devices on the system, including IDXD supported devices. +Intel\ |reg| DSA devices, are currently (at time of writing) appearing +as devices with type “0b25”, due to the absence of pci-id database entries for +them at this point. + +Compilation +------------ + +For builds using ``meson`` and ``ninja``, the driver will be built when the +target platform is x86-based. No additional compilation steps are necessary. + +Device Setup +------------- + +Devices using VFIO/UIO drivers +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The HW devices to be used will need to be bound to a user-space IO driver for use. +The ``dpdk-devbind.py`` script can be used to view the state of the devices +and to bind them to a suitable DPDK-supported driver, such as ``vfio-pci``. +For example:: + + $ dpdk-devbind.py -b vfio-pci 6a:01.0 + +Device Probing and Initialization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For devices bound to a suitable DPDK-supported VFIO/UIO driver, the HW devices will +be found as part of the device scan done at application initialization time without +the need to pass parameters to the application. + +For Intel\ |reg| DSA devices, DPDK will automatically configure the device with the +maximum number of workqueues available on it, partitioning all resources equally +among the queues. +If fewer workqueues are required, then the ``max_queues`` parameter may be passed to +the device driver on the EAL commandline, via the ``allowlist`` or ``-a`` flag e.g.:: + + $ dpdk-test -a ,max_queues=4 diff --git a/doc/guides/dmadevs/index.rst b/doc/guides/dmadevs/index.rst index 0bce29d766..5d4abf880e 100644 --- a/doc/guides/dmadevs/index.rst +++ b/doc/guides/dmadevs/index.rst @@ -10,3 +10,5 @@ an application through DMA API. .. toctree:: :maxdepth: 2 :numbered: + + idxd diff --git a/doc/guides/rel_notes/release_21_11.rst b/doc/guides/rel_notes/release_21_11.rst index c0fded3d0f..3b9f52c243 100644 --- a/doc/guides/rel_notes/release_21_11.rst +++ b/doc/guides/rel_notes/release_21_11.rst @@ -75,6 +75,11 @@ New Features operations. * Added multi-process support. +* **Added IDXD dmadev driver implementation.** + + The IDXD dmadev driver provide device drivers for the Intel DSA devices. + This device driver can be used through the generic dmadev API. + * **Added support to get all MAC addresses of a device.** Added ``rte_eth_macaddrs_get`` to allow user to retrieve all Ethernet diff --git a/drivers/dma/idxd/idxd_common.c b/drivers/dma/idxd/idxd_common.c new file mode 100644 index 0000000000..e00ddbe5ef --- /dev/null +++ b/drivers/dma/idxd/idxd_common.c @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2021 Intel Corporation + */ + +#include + +#include "idxd_internal.h" + +int idxd_pmd_logtype; + +RTE_LOG_REGISTER_DEFAULT(idxd_pmd_logtype, WARNING); diff --git a/drivers/dma/idxd/idxd_internal.h b/drivers/dma/idxd/idxd_internal.h new file mode 100644 index 0000000000..c6a7dcd72f --- /dev/null +++ b/drivers/dma/idxd/idxd_internal.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2021 Intel Corporation + */ + +#ifndef _IDXD_INTERNAL_H_ +#define _IDXD_INTERNAL_H_ + +/** + * @file idxd_internal.h + * + * Internal data structures for the idxd/DSA driver for dmadev + * + * @warning + * @b EXPERIMENTAL: these structures and APIs may change without prior notice + */ + +extern int idxd_pmd_logtype; + +#define IDXD_PMD_LOG(level, fmt, args...) rte_log(RTE_LOG_ ## level, \ + idxd_pmd_logtype, "IDXD: %s(): " fmt "\n", __func__, ##args) + +#define IDXD_PMD_DEBUG(fmt, args...) IDXD_PMD_LOG(DEBUG, fmt, ## args) +#define IDXD_PMD_INFO(fmt, args...) IDXD_PMD_LOG(INFO, fmt, ## args) +#define IDXD_PMD_ERR(fmt, args...) IDXD_PMD_LOG(ERR, fmt, ## args) +#define IDXD_PMD_WARN(fmt, args...) IDXD_PMD_LOG(WARNING, fmt, ## args) + +#endif /* _IDXD_INTERNAL_H_ */ diff --git a/drivers/dma/idxd/idxd_pci.c b/drivers/dma/idxd/idxd_pci.c new file mode 100644 index 0000000000..79e4aadcab --- /dev/null +++ b/drivers/dma/idxd/idxd_pci.c @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include + +#include "idxd_internal.h" + +#define IDXD_VENDOR_ID 0x8086 +#define IDXD_DEVICE_ID_SPR 0x0B25 + +#define IDXD_PMD_DMADEV_NAME_PCI dmadev_idxd_pci + +const struct rte_pci_id pci_id_idxd_map[] = { + { RTE_PCI_DEVICE(IDXD_VENDOR_ID, IDXD_DEVICE_ID_SPR) }, + { .vendor_id = 0, /* sentinel */ }, +}; + +static int +idxd_dmadev_probe_pci(struct rte_pci_driver *drv, struct rte_pci_device *dev) +{ + int ret = 0; + char name[PCI_PRI_STR_SIZE]; + + rte_pci_device_name(&dev->addr, name, sizeof(name)); + IDXD_PMD_INFO("Init %s on NUMA node %d", name, dev->device.numa_node); + dev->device.driver = &drv->driver; + + return ret; +} + +static int +idxd_dmadev_remove_pci(struct rte_pci_device *dev) +{ + char name[PCI_PRI_STR_SIZE]; + + rte_pci_device_name(&dev->addr, name, sizeof(name)); + + IDXD_PMD_INFO("Closing %s on NUMA node %d", + name, dev->device.numa_node); + + return 0; +} + +struct rte_pci_driver idxd_pmd_drv_pci = { + .id_table = pci_id_idxd_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING, + .probe = idxd_dmadev_probe_pci, + .remove = idxd_dmadev_remove_pci, +}; + +RTE_PMD_REGISTER_PCI(IDXD_PMD_DMADEV_NAME_PCI, idxd_pmd_drv_pci); +RTE_PMD_REGISTER_PCI_TABLE(IDXD_PMD_DMADEV_NAME_PCI, pci_id_idxd_map); +RTE_PMD_REGISTER_KMOD_DEP(IDXD_PMD_DMADEV_NAME_PCI, "vfio-pci"); +RTE_PMD_REGISTER_PARAM_STRING(dmadev_idxd_pci, "max_queues=0"); diff --git a/drivers/dma/idxd/meson.build b/drivers/dma/idxd/meson.build new file mode 100644 index 0000000000..2de32a104d --- /dev/null +++ b/drivers/dma/idxd/meson.build @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2021 Intel Corporation + +build = dpdk_conf.has('RTE_ARCH_X86') +reason = 'only supported on x86' + +deps += ['bus_pci'] +sources = files( + 'idxd_common.c', + 'idxd_pci.c', +) diff --git a/drivers/dma/idxd/version.map b/drivers/dma/idxd/version.map new file mode 100644 index 0000000000..c2e0723b4c --- /dev/null +++ b/drivers/dma/idxd/version.map @@ -0,0 +1,3 @@ +DPDK_22 { + local: *; +}; diff --git a/drivers/dma/meson.build b/drivers/dma/meson.build index d9c7ede32f..411be7a240 100644 --- a/drivers/dma/meson.build +++ b/drivers/dma/meson.build @@ -2,5 +2,7 @@ # Copyright 2021 HiSilicon Limited drivers = [ + 'idxd', 'skeleton', ] +std_deps = ['dmadev'] -- 2.20.1