From 0b5927cbcba7064445c35e5a412b460990829977 Mon Sep 17 00:00:00 2001 From: Nicolas Chautru Date: Sat, 18 Apr 2020 15:46:38 -0700 Subject: [PATCH] baseband/fpga_5gnr_fec: add PMD for FPGA 5GNR FEC Add stubs for the FPGA 5GNR FEC PMD Signed-off-by: Nicolas Chautru Acked-by: Dave Burley Acked-by: Niall Power Acked-by: Akhil Goyal --- config/common_base | 5 + doc/guides/bbdevs/fpga_5gnr_fec.rst | 146 +++++++++++++ doc/guides/bbdevs/index.rst | 1 + doc/guides/rel_notes/release_20_05.rst | 6 + drivers/baseband/Makefile | 2 + drivers/baseband/fpga_5gnr_fec/Makefile | 25 +++ .../baseband/fpga_5gnr_fec/fpga_5gnr_fec.h | 41 ++++ drivers/baseband/fpga_5gnr_fec/meson.build | 6 + .../fpga_5gnr_fec/rte_fpga_5gnr_fec.c | 193 ++++++++++++++++++ .../rte_pmd_bbdev_fpga_5gnr_fec_version.map | 3 + drivers/baseband/meson.build | 2 +- mk/rte.app.mk | 1 + 12 files changed, 430 insertions(+), 1 deletion(-) create mode 100644 doc/guides/bbdevs/fpga_5gnr_fec.rst create mode 100644 drivers/baseband/fpga_5gnr_fec/Makefile create mode 100644 drivers/baseband/fpga_5gnr_fec/fpga_5gnr_fec.h create mode 100644 drivers/baseband/fpga_5gnr_fec/meson.build create mode 100644 drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c create mode 100644 drivers/baseband/fpga_5gnr_fec/rte_pmd_bbdev_fpga_5gnr_fec_version.map diff --git a/config/common_base b/config/common_base index c31175f9d6..9ec689dde1 100644 --- a/config/common_base +++ b/config/common_base @@ -576,6 +576,11 @@ CONFIG_RTE_LIBRTE_PMD_BBDEV_TURBO_SW=y # CONFIG_RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC=y +# +# Compile PMD for Intel FPGA 5GNR FEC bbdev device +# +CONFIG_RTE_LIBRTE_PMD_BBDEV_FPGA_5GNR_FEC=y + # # Compile generic crypto device library # diff --git a/doc/guides/bbdevs/fpga_5gnr_fec.rst b/doc/guides/bbdevs/fpga_5gnr_fec.rst new file mode 100644 index 0000000000..f48a6004f0 --- /dev/null +++ b/doc/guides/bbdevs/fpga_5gnr_fec.rst @@ -0,0 +1,146 @@ +.. SPDX-License-Identifier: BSD-3-Clause + Copyright(c) 2019 Intel Corporation + +Intel(R) FPGA 5GNR FEC Poll Mode Driver +======================================= + +The BBDEV FPGA 5GNR FEC poll mode driver (PMD) supports an FPGA implementation of a VRAN +LDPC Encode / Decode 5GNR wireless acceleration function, using Intel's PCI-e and FPGA +based Vista Creek device. + +Features +-------- + +FPGA 5GNR FEC PMD supports the following features: + +- 8 VFs per PF (physical device) +- Maximum of 32 UL queues per VF +- Maximum of 32 DL queues per VF +- PCIe Gen-3 x8 Interface +- MSI-X +- SR-IOV + +Limitations +----------- + +FPGA 5GNR FEC does not support the following: + +- Scatter-Gather function + + +Installation +------------ + +Section 3 of the DPDK manual provides instuctions on installing and compiling DPDK. The +default set of bbdev compile flags may be found in config/common_base, where for example +the flag to build the FPGA 5GNR FEC device, ``CONFIG_RTE_LIBRTE_PMD_BBDEV_FPGA_5GNR_FEC``, +is already set. It is assumed DPDK has been compiled using for instance: + +.. code-block:: console + + make install T=x86_64-native-linuxapp-gcc + + +DPDK requires hugepages to be configured as detailed in section 2 of the DPDK manual. +The bbdev test application has been tested with a configuration 40 x 1GB hugepages. The +hugepage configuration of a server may be examined using: + +.. code-block:: console + + grep Huge* /proc/meminfo + + +Initialization +-------------- + +When the device first powers up, its PCI Physical Functions (PF) can be listed through this command: + +.. code-block:: console + + sudo lspci -vd8086:0d8f + +The physical and virtual functions are compatible with Linux UIO drivers: +``vfio`` and ``igb_uio``. However, in order to work the FPGA 5GNR FEC device firstly needs +to be bound to one of these linux drivers through DPDK. + + +Bind PF UIO driver(s) +~~~~~~~~~~~~~~~~~~~~~ + +Install the DPDK igb_uio driver, bind it with the PF PCI device ID and use +``lspci`` to confirm the PF device is under use by ``igb_uio`` DPDK UIO driver. + +The igb_uio driver may be bound to the PF PCI device using one of three methods: + + +1. PCI functions (physical or virtual, depending on the use case) can be bound to +the UIO driver by repeating this command for every function. + +.. code-block:: console + + cd + insmod ./build/kmod/igb_uio.ko + echo "8086 0d8f" > /sys/bus/pci/drivers/igb_uio/new_id + lspci -vd8086:0d8f + + +2. Another way to bind PF with DPDK UIO driver is by using the ``dpdk-devbind.py`` tool + +.. code-block:: console + + cd + ./usertools/dpdk-devbind.py -b igb_uio 0000:06:00.0 + +where the PCI device ID (example: 0000:06:00.0) is obtained using lspci -vd8086:0d8f + + +3. A third way to bind is to use ``dpdk-setup.sh`` tool + +.. code-block:: console + + cd + ./usertools/dpdk-setup.sh + + select 'Bind Ethernet/Crypto/Baseband device to IGB UIO module' + or + select 'Bind Ethernet/Crypto/Baseband device to VFIO module' depending on driver required + enter PCI device ID + select 'Display current Ethernet/Crypto/Baseband device settings' to confirm binding + + +In the same way the FPGA 5GNR FEC PF can be bound with vfio, but vfio driver does not +support SR-IOV configuration right out of the box, so it will need to be patched. + + +Enable Virtual Functions +~~~~~~~~~~~~~~~~~~~~~~~~ + +Now, it should be visible in the printouts that PCI PF is under igb_uio control +"``Kernel driver in use: igb_uio``" + +To show the number of available VFs on the device, read ``sriov_totalvfs`` file.. + +.. code-block:: console + + cat /sys/bus/pci/devices/0000\:\:./sriov_totalvfs + + where 0000\:\:. is the PCI device ID + + +To enable VFs via igb_uio, echo the number of virtual functions intended to +enable to ``max_vfs`` file.. + +.. code-block:: console + + echo > /sys/bus/pci/devices/0000\:\:./max_vfs + + +Afterwards, all VFs must be bound to appropriate UIO drivers as required, same +way it was done with the physical function previously. + +Enabling SR-IOV via vfio driver is pretty much the same, except that the file +name is different: + +.. code-block:: console + + echo > /sys/bus/pci/devices/0000\:\:./sriov_numvfs diff --git a/doc/guides/bbdevs/index.rst b/doc/guides/bbdevs/index.rst index 005b95e971..1a79343ff1 100644 --- a/doc/guides/bbdevs/index.rst +++ b/doc/guides/bbdevs/index.rst @@ -11,3 +11,4 @@ Baseband Device Drivers null turbo_sw fpga_lte_fec + fpga_5gnr_fec diff --git a/doc/guides/rel_notes/release_20_05.rst b/doc/guides/rel_notes/release_20_05.rst index ce885316fe..3f87203c64 100644 --- a/doc/guides/rel_notes/release_20_05.rst +++ b/doc/guides/rel_notes/release_20_05.rst @@ -104,6 +104,12 @@ New Features Supported large size code blocks which does not fit in one mbuf segment. +* **Added Intel FPGA_5GNR_FEC bbdev PMD.** + + Added a new ``fpga_5gnr_fec`` bbdev driver for the Intel\ |reg| FPGA PAC + (Programmable Acceleration Card) N3000. See the + :doc:`../bbdevs/fpga_5gnr_fec` BBDEV guide for more details on this new driver. + * **Updated ipsec-secgw sample application with following features.** * Updated ipsec-secgw application to add event based packet processing. diff --git a/drivers/baseband/Makefile b/drivers/baseband/Makefile index 91048beca4..dcc0969175 100644 --- a/drivers/baseband/Makefile +++ b/drivers/baseband/Makefile @@ -12,5 +12,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_PMD_BBDEV_TURBO_SW) += turbo_sw DEPDIRS-turbo_sw = $(core-libs) DIRS-$(CONFIG_RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC) += fpga_lte_fec DEPDIRS-fpga_lte_fec = $(core-libs) +DIRS-$(CONFIG_RTE_LIBRTE_PMD_BBDEV_FPGA_5GNR_FEC) += fpga_5gnr_fec +DEPDIRS-fpga_5gnr_fec = $(core-libs) include $(RTE_SDK)/mk/rte.subdir.mk diff --git a/drivers/baseband/fpga_5gnr_fec/Makefile b/drivers/baseband/fpga_5gnr_fec/Makefile new file mode 100644 index 0000000000..ffdd65ce98 --- /dev/null +++ b/drivers/baseband/fpga_5gnr_fec/Makefile @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2019 Intel Corporation + +include $(RTE_SDK)/mk/rte.vars.mk + +# library name +LIB = librte_pmd_bbdev_fpga_5gnr_fec.a + +# build flags +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) +LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring +LDLIBS += -lrte_bbdev +LDLIBS += -lrte_pci -lrte_bus_pci + +# versioning export map +EXPORT_MAP := rte_pmd_bbdev_fpga_5gnr_fec_version.map + +# library version +LIBABIVER := 1 + +# library source files +SRCS-$(CONFIG_RTE_LIBRTE_PMD_BBDEV_FPGA_5GNR_FEC) += rte_fpga_5gnr_fec.c + +include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/baseband/fpga_5gnr_fec/fpga_5gnr_fec.h b/drivers/baseband/fpga_5gnr_fec/fpga_5gnr_fec.h new file mode 100644 index 0000000000..0c481e251f --- /dev/null +++ b/drivers/baseband/fpga_5gnr_fec/fpga_5gnr_fec.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2020 Intel Corporation + */ + +#ifndef _FPGA_5GNR_FEC_H_ +#define _FPGA_5GNR_FEC_H_ + +#include +#include + +/* Helper macro for logging */ +#define rte_bbdev_log(level, fmt, ...) \ + rte_log(RTE_LOG_ ## level, fpga_5gnr_fec_logtype, fmt "\n", \ + ##__VA_ARGS__) + +#ifdef RTE_LIBRTE_BBDEV_DEBUG +#define rte_bbdev_log_debug(fmt, ...) \ + rte_bbdev_log(DEBUG, "fpga_5gnr_fec: " fmt, \ + ##__VA_ARGS__) +#else +#define rte_bbdev_log_debug(fmt, ...) +#endif + +/* FPGA 5GNR FEC driver names */ +#define FPGA_5GNR_FEC_PF_DRIVER_NAME intel_fpga_5gnr_fec_pf +#define FPGA_5GNR_FEC_VF_DRIVER_NAME intel_fpga_5gnr_fec_vf + +/* FPGA 5GNR FEC PCI vendor & device IDs */ +#define FPGA_5GNR_FEC_VENDOR_ID (0x8086) +#define FPGA_5GNR_FEC_PF_DEVICE_ID (0x0D8F) +#define FPGA_5GNR_FEC_VF_DEVICE_ID (0x0D90) + +/* Private data structure for each FPGA FEC device */ +struct fpga_5gnr_fec_device { + /** Base address of MMIO registers (BAR0) */ + void *mmio_base; + /** True if this is a PF FPGA FEC device */ + bool pf_device; +}; + +#endif /* _FPGA_5GNR_FEC_H_ */ diff --git a/drivers/baseband/fpga_5gnr_fec/meson.build b/drivers/baseband/fpga_5gnr_fec/meson.build new file mode 100644 index 0000000000..75c225e430 --- /dev/null +++ b/drivers/baseband/fpga_5gnr_fec/meson.build @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2020 Intel Corporation + +deps += ['bbdev', 'bus_vdev', 'ring', 'pci', 'bus_pci'] + +sources = files('rte_fpga_5gnr_fec.c') diff --git a/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c b/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c new file mode 100644 index 0000000000..ae0ec11190 --- /dev/null +++ b/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c @@ -0,0 +1,193 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2020 Intel Corporation + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "fpga_5gnr_fec.h" + +/* 5GNR SW PMD logging ID */ +static int fpga_5gnr_fec_logtype; + +static int +fpga_dev_close(struct rte_bbdev *dev __rte_unused) +{ + return 0; +} + +static const struct rte_bbdev_ops fpga_ops = { + .close = fpga_dev_close, +}; + +/* Initialization Function */ +static void +fpga_5gnr_fec_init(struct rte_bbdev *dev, struct rte_pci_driver *drv) +{ + struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); + + dev->dev_ops = &fpga_ops; + + ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->pf_device = + !strcmp(drv->driver.name, + RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME)); + ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->mmio_base = + pci_dev->mem_resource[0].addr; + + rte_bbdev_log_debug( + "Init device %s [%s] @ virtaddr %p phyaddr %#"PRIx64, + dev->device->driver->name, dev->data->name, + (void *)pci_dev->mem_resource[0].addr, + pci_dev->mem_resource[0].phys_addr); +} + +static int +fpga_5gnr_fec_probe(struct rte_pci_driver *pci_drv, + struct rte_pci_device *pci_dev) +{ + struct rte_bbdev *bbdev = NULL; + char dev_name[RTE_BBDEV_NAME_MAX_LEN]; + + if (pci_dev == NULL) { + rte_bbdev_log(ERR, "NULL PCI device"); + return -EINVAL; + } + + rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name)); + + /* Allocate memory to be used privately by drivers */ + bbdev = rte_bbdev_allocate(pci_dev->device.name); + if (bbdev == NULL) + return -ENODEV; + + /* allocate device private memory */ + bbdev->data->dev_private = rte_zmalloc_socket(dev_name, + sizeof(struct fpga_5gnr_fec_device), + RTE_CACHE_LINE_SIZE, + pci_dev->device.numa_node); + + if (bbdev->data->dev_private == NULL) { + rte_bbdev_log(CRIT, + "Allocate of %zu bytes for device \"%s\" failed", + sizeof(struct fpga_5gnr_fec_device), dev_name); + rte_bbdev_release(bbdev); + return -ENOMEM; + } + + /* Fill HW specific part of device structure */ + bbdev->device = &pci_dev->device; + bbdev->intr_handle = &pci_dev->intr_handle; + bbdev->data->socket_id = pci_dev->device.numa_node; + + /* Invoke FEC FPGA device initialization function */ + fpga_5gnr_fec_init(bbdev, pci_drv); + + rte_bbdev_log_debug("bbdev id = %u [%s]", + bbdev->data->dev_id, dev_name); + + return 0; +} + +static int +fpga_5gnr_fec_remove(struct rte_pci_device *pci_dev) +{ + struct rte_bbdev *bbdev; + int ret; + uint8_t dev_id; + + if (pci_dev == NULL) + return -EINVAL; + + /* Find device */ + bbdev = rte_bbdev_get_named_dev(pci_dev->device.name); + if (bbdev == NULL) { + rte_bbdev_log(CRIT, + "Couldn't find HW dev \"%s\" to uninitialise it", + pci_dev->device.name); + return -ENODEV; + } + dev_id = bbdev->data->dev_id; + + /* free device private memory before close */ + rte_free(bbdev->data->dev_private); + + /* Close device */ + ret = rte_bbdev_close(dev_id); + if (ret < 0) + rte_bbdev_log(ERR, + "Device %i failed to close during uninit: %i", + dev_id, ret); + + /* release bbdev from library */ + ret = rte_bbdev_release(bbdev); + if (ret) + rte_bbdev_log(ERR, "Device %i failed to uninit: %i", dev_id, + ret); + + rte_bbdev_log_debug("Destroyed bbdev = %u", dev_id); + + return 0; +} + +/* FPGA 5GNR FEC PCI PF address map */ +static struct rte_pci_id pci_id_fpga_5gnr_fec_pf_map[] = { + { + RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID, + FPGA_5GNR_FEC_PF_DEVICE_ID) + }, + {.device_id = 0}, +}; + +static struct rte_pci_driver fpga_5gnr_fec_pci_pf_driver = { + .probe = fpga_5gnr_fec_probe, + .remove = fpga_5gnr_fec_remove, + .id_table = pci_id_fpga_5gnr_fec_pf_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING +}; + +/* FPGA 5GNR FEC PCI VF address map */ +static struct rte_pci_id pci_id_fpga_5gnr_fec_vf_map[] = { + { + RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID, + FPGA_5GNR_FEC_VF_DEVICE_ID) + }, + {.device_id = 0}, +}; + +static struct rte_pci_driver fpga_5gnr_fec_pci_vf_driver = { + .probe = fpga_5gnr_fec_probe, + .remove = fpga_5gnr_fec_remove, + .id_table = pci_id_fpga_5gnr_fec_vf_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING +}; + + +RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_PF_DRIVER_NAME, fpga_5gnr_fec_pci_pf_driver); +RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_PF_DRIVER_NAME, + pci_id_fpga_5gnr_fec_pf_map); +RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_VF_DRIVER_NAME, fpga_5gnr_fec_pci_vf_driver); +RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_VF_DRIVER_NAME, + pci_id_fpga_5gnr_fec_vf_map); + +RTE_INIT(fpga_5gnr_fec_init_log) +{ + fpga_5gnr_fec_logtype = rte_log_register("pmd.bb.fpga_5gnr_fec"); + if (fpga_5gnr_fec_logtype >= 0) +#ifdef RTE_LIBRTE_BBDEV_DEBUG + rte_log_set_level(fpga_5gnr_fec_logtype, RTE_LOG_DEBUG); +#else + rte_log_set_level(fpga_5gnr_fec_logtype, RTE_LOG_NOTICE); +#endif +} diff --git a/drivers/baseband/fpga_5gnr_fec/rte_pmd_bbdev_fpga_5gnr_fec_version.map b/drivers/baseband/fpga_5gnr_fec/rte_pmd_bbdev_fpga_5gnr_fec_version.map new file mode 100644 index 0000000000..f9f17e4f6e --- /dev/null +++ b/drivers/baseband/fpga_5gnr_fec/rte_pmd_bbdev_fpga_5gnr_fec_version.map @@ -0,0 +1,3 @@ +DPDK_20.0 { + local: *; +}; diff --git a/drivers/baseband/meson.build b/drivers/baseband/meson.build index be7677f874..4d909f9a62 100644 --- a/drivers/baseband/meson.build +++ b/drivers/baseband/meson.build @@ -1,7 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2018 Luca Boccassi -drivers = ['null', 'turbo_sw', 'fpga_lte_fec'] +drivers = ['null', 'turbo_sw', 'fpga_lte_fec', 'fpga_5gnr_fec'] config_flag_fmt = 'RTE_LIBRTE_PMD_BBDEV_@0@' driver_name_fmt = 'rte_pmd_bbdev_@0@' diff --git a/mk/rte.app.mk b/mk/rte.app.mk index d295ca0a52..da12b9eec9 100644 --- a/mk/rte.app.mk +++ b/mk/rte.app.mk @@ -246,6 +246,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_NETVSC_PMD) += -lrte_pmd_netvsc ifeq ($(CONFIG_RTE_LIBRTE_BBDEV),y) _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_BBDEV_NULL) += -lrte_pmd_bbdev_null _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC) += -lrte_pmd_bbdev_fpga_lte_fec +_LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_BBDEV_FPGA_5GNR_FEC) += -lrte_pmd_bbdev_fpga_5gnr_fec # TURBO SOFTWARE PMD is dependent on the FLEXRAN library _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_BBDEV_TURBO_SW) += -lrte_pmd_bbdev_turbo_sw -- 2.20.1