From 25500d4b8076e9b84c0d69108c64418470d1f65c Mon Sep 17 00:00:00 2001 From: Jay Zhou Date: Tue, 17 Apr 2018 17:23:18 +0800 Subject: [PATCH] crypto/virtio: support device init This patch implements the initialization of the virtio crypto device. The virtio crypto device conforms to virtio-1.0, so this patch only supports modern mode operation. The cryptodev is created at the virtio crypto pci device probing stage. The function of virtio_crypto_pkt_tx_burst() is used to burst transfer packets and virtio_crypto_pkt_rx_burst() is used to burst receive packets. Signed-off-by: Jay Zhou Reviewed-by: Fan Zhang --- drivers/crypto/virtio/Makefile | 7 + drivers/crypto/virtio/meson.build | 4 +- drivers/crypto/virtio/virtio_cryptodev.c | 245 +++++++++++- drivers/crypto/virtio/virtio_cryptodev.h | 13 + drivers/crypto/virtio/virtio_logs.h | 91 +++++ drivers/crypto/virtio/virtio_pci.c | 462 +++++++++++++++++++++++ drivers/crypto/virtio/virtio_pci.h | 253 +++++++++++++ drivers/crypto/virtio/virtio_ring.h | 137 +++++++ drivers/crypto/virtio/virtio_rxtx.c | 26 ++ drivers/crypto/virtio/virtqueue.c | 43 +++ drivers/crypto/virtio/virtqueue.h | 171 +++++++++ 11 files changed, 1449 insertions(+), 3 deletions(-) create mode 100644 drivers/crypto/virtio/virtio_logs.h create mode 100644 drivers/crypto/virtio/virtio_pci.c create mode 100644 drivers/crypto/virtio/virtio_pci.h create mode 100644 drivers/crypto/virtio/virtio_ring.h create mode 100644 drivers/crypto/virtio/virtio_rxtx.c create mode 100644 drivers/crypto/virtio/virtqueue.c create mode 100644 drivers/crypto/virtio/virtqueue.h diff --git a/drivers/crypto/virtio/Makefile b/drivers/crypto/virtio/Makefile index 58f8cfb199..be7b828fe1 100644 --- a/drivers/crypto/virtio/Makefile +++ b/drivers/crypto/virtio/Makefile @@ -8,6 +8,10 @@ include $(RTE_SDK)/mk/rte.vars.mk # LIB = librte_pmd_virtio_crypto.a +# +# include virtio_crypto.h +# +CFLAGS += -I$(RTE_SDK)/lib/librte_vhost CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) @@ -18,6 +22,9 @@ LIBABIVER := 1 # # all source are stored in SRCS-y # +SRCS-$(CONFIG_RTE_LIBRTE_PMD_VIRTIO_CRYPTO) += virtqueue.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_VIRTIO_CRYPTO) += virtio_pci.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_VIRTIO_CRYPTO) += virtio_rxtx.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_VIRTIO_CRYPTO) += virtio_cryptodev.c # this lib depends upon: diff --git a/drivers/crypto/virtio/meson.build b/drivers/crypto/virtio/meson.build index 51f5b085a2..b15b3f9fab 100644 --- a/drivers/crypto/virtio/meson.build +++ b/drivers/crypto/virtio/meson.build @@ -1,6 +1,8 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD. +includes += include_directories('../../../lib/librte_vhost') deps += 'bus_pci' name = 'virtio_crypto' -sources = files('virtio_cryptodev.c') +sources = files('virtio_cryptodev.c', 'virtio_pci.c', + 'virtio_rxtx.c', 'virtqueue.c') diff --git a/drivers/crypto/virtio/virtio_cryptodev.c b/drivers/crypto/virtio/virtio_cryptodev.c index 24295d89aa..1afa3a1584 100644 --- a/drivers/crypto/virtio/virtio_cryptodev.c +++ b/drivers/crypto/virtio/virtio_cryptodev.c @@ -3,28 +3,241 @@ */ #include #include +#include #include +#include #include "virtio_cryptodev.h" +#include "virtqueue.h" + +int virtio_crypto_logtype_init; +int virtio_crypto_logtype_session; +int virtio_crypto_logtype_rx; +int virtio_crypto_logtype_tx; +int virtio_crypto_logtype_driver; + +/* + * The set of PCI devices this driver supports + */ +static const struct rte_pci_id pci_id_virtio_crypto_map[] = { + { RTE_PCI_DEVICE(VIRTIO_CRYPTO_PCI_VENDORID, + VIRTIO_CRYPTO_PCI_DEVICEID) }, + { .vendor_id = 0, /* sentinel */ }, +}; uint8_t cryptodev_virtio_driver_id; +/* + * dev_ops for virtio, bare necessities for basic operation + */ +static struct rte_cryptodev_ops virtio_crypto_dev_ops = { + /* Device related operations */ + .dev_configure = NULL, + .dev_start = NULL, + .dev_stop = NULL, + .dev_close = NULL, + .dev_infos_get = NULL, + + .stats_get = NULL, + .stats_reset = NULL, + + .queue_pair_setup = NULL, + .queue_pair_release = NULL, + .queue_pair_start = NULL, + .queue_pair_stop = NULL, + .queue_pair_count = NULL, + + /* Crypto related operations */ + .session_get_size = NULL, + .session_configure = NULL, + .session_clear = NULL, + .qp_attach_session = NULL, + .qp_detach_session = NULL +}; + +static int +virtio_negotiate_features(struct virtio_crypto_hw *hw, uint64_t req_features) +{ + uint64_t host_features; + + PMD_INIT_FUNC_TRACE(); + + /* Prepare guest_features: feature that driver wants to support */ + VIRTIO_CRYPTO_INIT_LOG_DBG("guest_features before negotiate = %" PRIx64, + req_features); + + /* Read device(host) feature bits */ + host_features = VTPCI_OPS(hw)->get_features(hw); + VIRTIO_CRYPTO_INIT_LOG_DBG("host_features before negotiate = %" PRIx64, + host_features); + + /* + * Negotiate features: Subset of device feature bits are written back + * guest feature bits. + */ + hw->guest_features = req_features; + hw->guest_features = vtpci_cryptodev_negotiate_features(hw, + host_features); + VIRTIO_CRYPTO_INIT_LOG_DBG("features after negotiate = %" PRIx64, + hw->guest_features); + + if (hw->modern) { + if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) { + VIRTIO_CRYPTO_INIT_LOG_ERR( + "VIRTIO_F_VERSION_1 features is not enabled."); + return -1; + } + vtpci_cryptodev_set_status(hw, + VIRTIO_CONFIG_STATUS_FEATURES_OK); + if (!(vtpci_cryptodev_get_status(hw) & + VIRTIO_CONFIG_STATUS_FEATURES_OK)) { + VIRTIO_CRYPTO_INIT_LOG_ERR("failed to set FEATURES_OK " + "status!"); + return -1; + } + } + + hw->req_guest_features = req_features; + + return 0; +} + +/* reset device and renegotiate features if needed */ +static int +virtio_crypto_init_device(struct rte_cryptodev *cryptodev, + uint64_t req_features) +{ + struct virtio_crypto_hw *hw = cryptodev->data->dev_private; + struct virtio_crypto_config local_config; + struct virtio_crypto_config *config = &local_config; + + PMD_INIT_FUNC_TRACE(); + + /* Reset the device although not necessary at startup */ + vtpci_cryptodev_reset(hw); + + /* Tell the host we've noticed this device. */ + vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); + + /* Tell the host we've known how to drive the device. */ + vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); + if (virtio_negotiate_features(hw, req_features) < 0) + return -1; + + /* Get status of the device */ + vtpci_read_cryptodev_config(hw, + offsetof(struct virtio_crypto_config, status), + &config->status, sizeof(config->status)); + if (config->status != VIRTIO_CRYPTO_S_HW_READY) { + VIRTIO_CRYPTO_DRV_LOG_ERR("accelerator hardware is " + "not ready"); + return -1; + } + + /* Get number of data queues */ + vtpci_read_cryptodev_config(hw, + offsetof(struct virtio_crypto_config, max_dataqueues), + &config->max_dataqueues, + sizeof(config->max_dataqueues)); + hw->max_dataqueues = config->max_dataqueues; + + VIRTIO_CRYPTO_INIT_LOG_DBG("hw->max_dataqueues=%d", + hw->max_dataqueues); + + return 0; +} + +/* + * This function is based on probe() function + * It returns 0 on success. + */ +static int +crypto_virtio_create(const char *name, struct rte_pci_device *pci_dev, + struct rte_cryptodev_pmd_init_params *init_params) +{ + struct rte_cryptodev *cryptodev; + struct virtio_crypto_hw *hw; + + PMD_INIT_FUNC_TRACE(); + + cryptodev = rte_cryptodev_pmd_create(name, &pci_dev->device, + init_params); + if (cryptodev == NULL) + return -ENODEV; + + cryptodev->driver_id = cryptodev_virtio_driver_id; + cryptodev->dev_ops = &virtio_crypto_dev_ops; + + cryptodev->enqueue_burst = virtio_crypto_pkt_tx_burst; + cryptodev->dequeue_burst = virtio_crypto_pkt_rx_burst; + + cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | + RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING; + + hw = cryptodev->data->dev_private; + hw->dev_id = cryptodev->data->dev_id; + + VIRTIO_CRYPTO_INIT_LOG_DBG("dev %d vendorID=0x%x deviceID=0x%x", + cryptodev->data->dev_id, pci_dev->id.vendor_id, + pci_dev->id.device_id); + + /* pci device init */ + if (vtpci_cryptodev_init(pci_dev, hw)) + return -1; + + if (virtio_crypto_init_device(cryptodev, + VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0) + return -1; + + return 0; +} + static int crypto_virtio_pci_probe( struct rte_pci_driver *pci_drv __rte_unused, - struct rte_pci_device *pci_dev __rte_unused) + struct rte_pci_device *pci_dev) { - return 0; + struct rte_cryptodev_pmd_init_params init_params = { + .name = "", + .socket_id = rte_socket_id(), + .private_data_size = sizeof(struct virtio_crypto_hw), + .max_nb_sessions = RTE_VIRTIO_CRYPTO_PMD_MAX_NB_SESSIONS + }; + char name[RTE_CRYPTODEV_NAME_MAX_LEN]; + + VIRTIO_CRYPTO_DRV_LOG_DBG("Found Crypto device at %02x:%02x.%x", + pci_dev->addr.bus, + pci_dev->addr.devid, + pci_dev->addr.function); + + rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); + + return crypto_virtio_create(name, pci_dev, &init_params); } static int crypto_virtio_pci_remove( struct rte_pci_device *pci_dev __rte_unused) { + struct rte_cryptodev *cryptodev; + char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; + + if (pci_dev == NULL) + return -EINVAL; + + rte_pci_device_name(&pci_dev->addr, cryptodev_name, + sizeof(cryptodev_name)); + + cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name); + if (cryptodev == NULL) + return -ENODEV; + return 0; } static struct rte_pci_driver rte_virtio_crypto_driver = { + .id_table = pci_id_virtio_crypto_map, + .drv_flags = 0, .probe = crypto_virtio_pci_probe, .remove = crypto_virtio_pci_remove }; @@ -35,3 +248,31 @@ RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_VIRTIO_PMD, rte_virtio_crypto_driver); RTE_PMD_REGISTER_CRYPTO_DRIVER(virtio_crypto_drv, rte_virtio_crypto_driver.driver, cryptodev_virtio_driver_id); + +RTE_INIT(virtio_crypto_init_log); +static void +virtio_crypto_init_log(void) +{ + virtio_crypto_logtype_init = rte_log_register("pmd.crypto.virtio.init"); + if (virtio_crypto_logtype_init >= 0) + rte_log_set_level(virtio_crypto_logtype_init, RTE_LOG_NOTICE); + + virtio_crypto_logtype_session = + rte_log_register("pmd.crypto.virtio.session"); + if (virtio_crypto_logtype_session >= 0) + rte_log_set_level(virtio_crypto_logtype_session, + RTE_LOG_NOTICE); + + virtio_crypto_logtype_rx = rte_log_register("pmd.crypto.virtio.rx"); + if (virtio_crypto_logtype_rx >= 0) + rte_log_set_level(virtio_crypto_logtype_rx, RTE_LOG_NOTICE); + + virtio_crypto_logtype_tx = rte_log_register("pmd.crypto.virtio.tx"); + if (virtio_crypto_logtype_tx >= 0) + rte_log_set_level(virtio_crypto_logtype_tx, RTE_LOG_NOTICE); + + virtio_crypto_logtype_driver = + rte_log_register("pmd.crypto.virtio.driver"); + if (virtio_crypto_logtype_driver >= 0) + rte_log_set_level(virtio_crypto_logtype_driver, RTE_LOG_NOTICE); +} diff --git a/drivers/crypto/virtio/virtio_cryptodev.h b/drivers/crypto/virtio/virtio_cryptodev.h index 44517b82ae..392db4a2b2 100644 --- a/drivers/crypto/virtio/virtio_cryptodev.h +++ b/drivers/crypto/virtio/virtio_cryptodev.h @@ -5,6 +5,19 @@ #ifndef _VIRTIO_CRYPTODEV_H_ #define _VIRTIO_CRYPTODEV_H_ +#include + +/* Features desired/implemented by this driver. */ +#define VIRTIO_CRYPTO_PMD_GUEST_FEATURES (1ULL << VIRTIO_F_VERSION_1) + #define CRYPTODEV_NAME_VIRTIO_PMD crypto_virtio +uint16_t virtio_crypto_pkt_tx_burst(void *tx_queue, + struct rte_crypto_op **tx_pkts, + uint16_t nb_pkts); + +uint16_t virtio_crypto_pkt_rx_burst(void *tx_queue, + struct rte_crypto_op **tx_pkts, + uint16_t nb_pkts); + #endif /* _VIRTIO_CRYPTODEV_H_ */ diff --git a/drivers/crypto/virtio/virtio_logs.h b/drivers/crypto/virtio/virtio_logs.h new file mode 100644 index 0000000000..26a286cf81 --- /dev/null +++ b/drivers/crypto/virtio/virtio_logs.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD. + */ + +#ifndef _VIRTIO_LOGS_H_ +#define _VIRTIO_LOGS_H_ + +#include + +#define PMD_INIT_LOG(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, RTE_LOGTYPE_PMD, \ + "PMD: %s(): " fmt "\n", __func__, ##args) + +#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>") + +extern int virtio_crypto_logtype_init; + +#define VIRTIO_CRYPTO_INIT_LOG_IMPL(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, virtio_crypto_logtype_init, \ + "INIT: %s(): " fmt "\n", __func__, ##args) + +#define VIRTIO_CRYPTO_INIT_LOG_INFO(fmt, args...) \ + VIRTIO_CRYPTO_INIT_LOG_IMPL(INFO, fmt, ## args) + +#define VIRTIO_CRYPTO_INIT_LOG_DBG(fmt, args...) \ + VIRTIO_CRYPTO_INIT_LOG_IMPL(DEBUG, fmt, ## args) + +#define VIRTIO_CRYPTO_INIT_LOG_ERR(fmt, args...) \ + VIRTIO_CRYPTO_INIT_LOG_IMPL(ERR, fmt, ## args) + +extern int virtio_crypto_logtype_session; + +#define VIRTIO_CRYPTO_SESSION_LOG_IMPL(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, virtio_crypto_logtype_session, \ + "SESSION: %s(): " fmt "\n", __func__, ##args) + +#define VIRTIO_CRYPTO_SESSION_LOG_INFO(fmt, args...) \ + VIRTIO_CRYPTO_SESSION_LOG_IMPL(INFO, fmt, ## args) + +#define VIRTIO_CRYPTO_SESSION_LOG_DBG(fmt, args...) \ + VIRTIO_CRYPTO_SESSION_LOG_IMPL(DEBUG, fmt, ## args) + +#define VIRTIO_CRYPTO_SESSION_LOG_ERR(fmt, args...) \ + VIRTIO_CRYPTO_SESSION_LOG_IMPL(ERR, fmt, ## args) + +extern int virtio_crypto_logtype_rx; + +#define VIRTIO_CRYPTO_RX_LOG_IMPL(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, virtio_crypto_logtype_rx, \ + "RX: %s(): " fmt "\n", __func__, ##args) + +#define VIRTIO_CRYPTO_RX_LOG_INFO(fmt, args...) \ + VIRTIO_CRYPTO_RX_LOG_IMPL(INFO, fmt, ## args) + +#define VIRTIO_CRYPTO_RX_LOG_DBG(fmt, args...) \ + VIRTIO_CRYPTO_RX_LOG_IMPL(DEBUG, fmt, ## args) + +#define VIRTIO_CRYPTO_RX_LOG_ERR(fmt, args...) \ + VIRTIO_CRYPTO_RX_LOG_IMPL(ERR, fmt, ## args) + +extern int virtio_crypto_logtype_tx; + +#define VIRTIO_CRYPTO_TX_LOG_IMPL(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, virtio_crypto_logtype_tx, \ + "TX: %s(): " fmt "\n", __func__, ##args) + +#define VIRTIO_CRYPTO_TX_LOG_INFO(fmt, args...) \ + VIRTIO_CRYPTO_TX_LOG_IMPL(INFO, fmt, ## args) + +#define VIRTIO_CRYPTO_TX_LOG_DBG(fmt, args...) \ + VIRTIO_CRYPTO_TX_LOG_IMPL(DEBUG, fmt, ## args) + +#define VIRTIO_CRYPTO_TX_LOG_ERR(fmt, args...) \ + VIRTIO_CRYPTO_TX_LOG_IMPL(ERR, fmt, ## args) + +extern int virtio_crypto_logtype_driver; + +#define VIRTIO_CRYPTO_DRV_LOG_IMPL(level, fmt, args...) \ + rte_log(RTE_LOG_ ## level, virtio_crypto_logtype_driver, \ + "DRIVER: %s(): " fmt "\n", __func__, ##args) + +#define VIRTIO_CRYPTO_DRV_LOG_INFO(fmt, args...) \ + VIRTIO_CRYPTO_DRV_LOG_IMPL(INFO, fmt, ## args) + +#define VIRTIO_CRYPTO_DRV_LOG_DBG(fmt, args...) \ + VIRTIO_CRYPTO_DRV_LOG_IMPL(DEBUG, fmt, ## args) + +#define VIRTIO_CRYPTO_DRV_LOG_ERR(fmt, args...) \ + VIRTIO_CRYPTO_DRV_LOG_IMPL(ERR, fmt, ## args) + +#endif /* _VIRTIO_LOGS_H_ */ diff --git a/drivers/crypto/virtio/virtio_pci.c b/drivers/crypto/virtio/virtio_pci.c new file mode 100644 index 0000000000..832c465bcf --- /dev/null +++ b/drivers/crypto/virtio/virtio_pci.c @@ -0,0 +1,462 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD. + */ + +#include + +#ifdef RTE_EXEC_ENV_LINUXAPP + #include + #include +#endif + +#include +#include + +#include "virtio_pci.h" +#include "virtqueue.h" + +/* + * Following macros are derived from linux/pci_regs.h, however, + * we can't simply include that header here, as there is no such + * file for non-Linux platform. + */ +#define PCI_CAPABILITY_LIST 0x34 +#define PCI_CAP_ID_VNDR 0x09 +#define PCI_CAP_ID_MSIX 0x11 + +/* + * The remaining space is defined by each driver as the per-driver + * configuration space. + */ +#define VIRTIO_PCI_CONFIG(hw) \ + (((hw)->use_msix == VIRTIO_MSIX_ENABLED) ? 24 : 20) + +struct virtio_hw_internal virtio_hw_internal[RTE_MAX_VIRTIO_CRYPTO]; + +static inline int +check_vq_phys_addr_ok(struct virtqueue *vq) +{ + /* Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit, + * and only accepts 32 bit page frame number. + * Check if the allocated physical memory exceeds 16TB. + */ + if ((vq->vq_ring_mem + vq->vq_ring_size - 1) >> + (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) { + VIRTIO_CRYPTO_INIT_LOG_ERR("vring address shouldn't be above 16TB!"); + return 0; + } + + return 1; +} + +static inline void +io_write64_twopart(uint64_t val, uint32_t *lo, uint32_t *hi) +{ + rte_write32(val & ((1ULL << 32) - 1), lo); + rte_write32(val >> 32, hi); +} + +static void +modern_read_dev_config(struct virtio_crypto_hw *hw, size_t offset, + void *dst, int length) +{ + int i; + uint8_t *p; + uint8_t old_gen, new_gen; + + do { + old_gen = rte_read8(&hw->common_cfg->config_generation); + + p = dst; + for (i = 0; i < length; i++) + *p++ = rte_read8((uint8_t *)hw->dev_cfg + offset + i); + + new_gen = rte_read8(&hw->common_cfg->config_generation); + } while (old_gen != new_gen); +} + +static void +modern_write_dev_config(struct virtio_crypto_hw *hw, size_t offset, + const void *src, int length) +{ + int i; + const uint8_t *p = src; + + for (i = 0; i < length; i++) + rte_write8((*p++), (((uint8_t *)hw->dev_cfg) + offset + i)); +} + +static uint64_t +modern_get_features(struct virtio_crypto_hw *hw) +{ + uint32_t features_lo, features_hi; + + rte_write32(0, &hw->common_cfg->device_feature_select); + features_lo = rte_read32(&hw->common_cfg->device_feature); + + rte_write32(1, &hw->common_cfg->device_feature_select); + features_hi = rte_read32(&hw->common_cfg->device_feature); + + return ((uint64_t)features_hi << 32) | features_lo; +} + +static void +modern_set_features(struct virtio_crypto_hw *hw, uint64_t features) +{ + rte_write32(0, &hw->common_cfg->guest_feature_select); + rte_write32(features & ((1ULL << 32) - 1), + &hw->common_cfg->guest_feature); + + rte_write32(1, &hw->common_cfg->guest_feature_select); + rte_write32(features >> 32, + &hw->common_cfg->guest_feature); +} + +static uint8_t +modern_get_status(struct virtio_crypto_hw *hw) +{ + return rte_read8(&hw->common_cfg->device_status); +} + +static void +modern_set_status(struct virtio_crypto_hw *hw, uint8_t status) +{ + rte_write8(status, &hw->common_cfg->device_status); +} + +static void +modern_reset(struct virtio_crypto_hw *hw) +{ + modern_set_status(hw, VIRTIO_CONFIG_STATUS_RESET); + modern_get_status(hw); +} + +static uint8_t +modern_get_isr(struct virtio_crypto_hw *hw) +{ + return rte_read8(hw->isr); +} + +static uint16_t +modern_set_config_irq(struct virtio_crypto_hw *hw, uint16_t vec) +{ + rte_write16(vec, &hw->common_cfg->msix_config); + return rte_read16(&hw->common_cfg->msix_config); +} + +static uint16_t +modern_set_queue_irq(struct virtio_crypto_hw *hw, struct virtqueue *vq, + uint16_t vec) +{ + rte_write16(vq->vq_queue_index, &hw->common_cfg->queue_select); + rte_write16(vec, &hw->common_cfg->queue_msix_vector); + return rte_read16(&hw->common_cfg->queue_msix_vector); +} + +static uint16_t +modern_get_queue_num(struct virtio_crypto_hw *hw, uint16_t queue_id) +{ + rte_write16(queue_id, &hw->common_cfg->queue_select); + return rte_read16(&hw->common_cfg->queue_size); +} + +static int +modern_setup_queue(struct virtio_crypto_hw *hw, struct virtqueue *vq) +{ + uint64_t desc_addr, avail_addr, used_addr; + uint16_t notify_off; + + if (!check_vq_phys_addr_ok(vq)) + return -1; + + desc_addr = vq->vq_ring_mem; + avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc); + used_addr = RTE_ALIGN_CEIL(avail_addr + offsetof(struct vring_avail, + ring[vq->vq_nentries]), + VIRTIO_PCI_VRING_ALIGN); + + rte_write16(vq->vq_queue_index, &hw->common_cfg->queue_select); + + io_write64_twopart(desc_addr, &hw->common_cfg->queue_desc_lo, + &hw->common_cfg->queue_desc_hi); + io_write64_twopart(avail_addr, &hw->common_cfg->queue_avail_lo, + &hw->common_cfg->queue_avail_hi); + io_write64_twopart(used_addr, &hw->common_cfg->queue_used_lo, + &hw->common_cfg->queue_used_hi); + + notify_off = rte_read16(&hw->common_cfg->queue_notify_off); + vq->notify_addr = (void *)((uint8_t *)hw->notify_base + + notify_off * hw->notify_off_multiplier); + + rte_write16(1, &hw->common_cfg->queue_enable); + + VIRTIO_CRYPTO_INIT_LOG_DBG("queue %u addresses:", vq->vq_queue_index); + VIRTIO_CRYPTO_INIT_LOG_DBG("\t desc_addr: %" PRIx64, desc_addr); + VIRTIO_CRYPTO_INIT_LOG_DBG("\t aval_addr: %" PRIx64, avail_addr); + VIRTIO_CRYPTO_INIT_LOG_DBG("\t used_addr: %" PRIx64, used_addr); + VIRTIO_CRYPTO_INIT_LOG_DBG("\t notify addr: %p (notify offset: %u)", + vq->notify_addr, notify_off); + + return 0; +} + +static void +modern_del_queue(struct virtio_crypto_hw *hw, struct virtqueue *vq) +{ + rte_write16(vq->vq_queue_index, &hw->common_cfg->queue_select); + + io_write64_twopart(0, &hw->common_cfg->queue_desc_lo, + &hw->common_cfg->queue_desc_hi); + io_write64_twopart(0, &hw->common_cfg->queue_avail_lo, + &hw->common_cfg->queue_avail_hi); + io_write64_twopart(0, &hw->common_cfg->queue_used_lo, + &hw->common_cfg->queue_used_hi); + + rte_write16(0, &hw->common_cfg->queue_enable); +} + +static void +modern_notify_queue(struct virtio_crypto_hw *hw __rte_unused, + struct virtqueue *vq) +{ + rte_write16(vq->vq_queue_index, vq->notify_addr); +} + +const struct virtio_pci_ops virtio_crypto_modern_ops = { + .read_dev_cfg = modern_read_dev_config, + .write_dev_cfg = modern_write_dev_config, + .reset = modern_reset, + .get_status = modern_get_status, + .set_status = modern_set_status, + .get_features = modern_get_features, + .set_features = modern_set_features, + .get_isr = modern_get_isr, + .set_config_irq = modern_set_config_irq, + .set_queue_irq = modern_set_queue_irq, + .get_queue_num = modern_get_queue_num, + .setup_queue = modern_setup_queue, + .del_queue = modern_del_queue, + .notify_queue = modern_notify_queue, +}; + +void +vtpci_read_cryptodev_config(struct virtio_crypto_hw *hw, size_t offset, + void *dst, int length) +{ + VTPCI_OPS(hw)->read_dev_cfg(hw, offset, dst, length); +} + +void +vtpci_write_cryptodev_config(struct virtio_crypto_hw *hw, size_t offset, + const void *src, int length) +{ + VTPCI_OPS(hw)->write_dev_cfg(hw, offset, src, length); +} + +uint64_t +vtpci_cryptodev_negotiate_features(struct virtio_crypto_hw *hw, + uint64_t host_features) +{ + uint64_t features; + + /* + * Limit negotiated features to what the driver, virtqueue, and + * host all support. + */ + features = host_features & hw->guest_features; + VTPCI_OPS(hw)->set_features(hw, features); + + return features; +} + +void +vtpci_cryptodev_reset(struct virtio_crypto_hw *hw) +{ + VTPCI_OPS(hw)->set_status(hw, VIRTIO_CONFIG_STATUS_RESET); + /* flush status write */ + VTPCI_OPS(hw)->get_status(hw); +} + +void +vtpci_cryptodev_reinit_complete(struct virtio_crypto_hw *hw) +{ + vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER_OK); +} + +void +vtpci_cryptodev_set_status(struct virtio_crypto_hw *hw, uint8_t status) +{ + if (status != VIRTIO_CONFIG_STATUS_RESET) + status |= VTPCI_OPS(hw)->get_status(hw); + + VTPCI_OPS(hw)->set_status(hw, status); +} + +uint8_t +vtpci_cryptodev_get_status(struct virtio_crypto_hw *hw) +{ + return VTPCI_OPS(hw)->get_status(hw); +} + +uint8_t +vtpci_cryptodev_isr(struct virtio_crypto_hw *hw) +{ + return VTPCI_OPS(hw)->get_isr(hw); +} + +static void * +get_cfg_addr(struct rte_pci_device *dev, struct virtio_pci_cap *cap) +{ + uint8_t bar = cap->bar; + uint32_t length = cap->length; + uint32_t offset = cap->offset; + uint8_t *base; + + if (bar >= PCI_MAX_RESOURCE) { + VIRTIO_CRYPTO_INIT_LOG_ERR("invalid bar: %u", bar); + return NULL; + } + + if (offset + length < offset) { + VIRTIO_CRYPTO_INIT_LOG_ERR("offset(%u) + length(%u) overflows", + offset, length); + return NULL; + } + + if (offset + length > dev->mem_resource[bar].len) { + VIRTIO_CRYPTO_INIT_LOG_ERR( + "invalid cap: overflows bar space: %u > %" PRIu64, + offset + length, dev->mem_resource[bar].len); + return NULL; + } + + base = dev->mem_resource[bar].addr; + if (base == NULL) { + VIRTIO_CRYPTO_INIT_LOG_ERR("bar %u base addr is NULL", bar); + return NULL; + } + + return base + offset; +} + +#define PCI_MSIX_ENABLE 0x8000 + +static int +virtio_read_caps(struct rte_pci_device *dev, struct virtio_crypto_hw *hw) +{ + uint8_t pos; + struct virtio_pci_cap cap; + int ret; + + if (rte_pci_map_device(dev)) { + VIRTIO_CRYPTO_INIT_LOG_DBG("failed to map pci device!"); + return -1; + } + + ret = rte_pci_read_config(dev, &pos, 1, PCI_CAPABILITY_LIST); + if (ret < 0) { + VIRTIO_CRYPTO_INIT_LOG_DBG("failed to read pci capability list"); + return -1; + } + + while (pos) { + ret = rte_pci_read_config(dev, &cap, sizeof(cap), pos); + if (ret < 0) { + VIRTIO_CRYPTO_INIT_LOG_ERR( + "failed to read pci cap at pos: %x", pos); + break; + } + + if (cap.cap_vndr == PCI_CAP_ID_MSIX) { + /* Transitional devices would also have this capability, + * that's why we also check if msix is enabled. + * 1st byte is cap ID; 2nd byte is the position of next + * cap; next two bytes are the flags. + */ + uint16_t flags = ((uint16_t *)&cap)[1]; + + if (flags & PCI_MSIX_ENABLE) + hw->use_msix = VIRTIO_MSIX_ENABLED; + else + hw->use_msix = VIRTIO_MSIX_DISABLED; + } + + if (cap.cap_vndr != PCI_CAP_ID_VNDR) { + VIRTIO_CRYPTO_INIT_LOG_DBG( + "[%2x] skipping non VNDR cap id: %02x", + pos, cap.cap_vndr); + goto next; + } + + VIRTIO_CRYPTO_INIT_LOG_DBG( + "[%2x] cfg type: %u, bar: %u, offset: %04x, len: %u", + pos, cap.cfg_type, cap.bar, cap.offset, cap.length); + + switch (cap.cfg_type) { + case VIRTIO_PCI_CAP_COMMON_CFG: + hw->common_cfg = get_cfg_addr(dev, &cap); + break; + case VIRTIO_PCI_CAP_NOTIFY_CFG: + rte_pci_read_config(dev, &hw->notify_off_multiplier, + 4, pos + sizeof(cap)); + hw->notify_base = get_cfg_addr(dev, &cap); + break; + case VIRTIO_PCI_CAP_DEVICE_CFG: + hw->dev_cfg = get_cfg_addr(dev, &cap); + break; + case VIRTIO_PCI_CAP_ISR_CFG: + hw->isr = get_cfg_addr(dev, &cap); + break; + } + +next: + pos = cap.cap_next; + } + + if (hw->common_cfg == NULL || hw->notify_base == NULL || + hw->dev_cfg == NULL || hw->isr == NULL) { + VIRTIO_CRYPTO_INIT_LOG_INFO("no modern virtio pci device found."); + return -1; + } + + VIRTIO_CRYPTO_INIT_LOG_INFO("found modern virtio pci device."); + + VIRTIO_CRYPTO_INIT_LOG_DBG("common cfg mapped at: %p", hw->common_cfg); + VIRTIO_CRYPTO_INIT_LOG_DBG("device cfg mapped at: %p", hw->dev_cfg); + VIRTIO_CRYPTO_INIT_LOG_DBG("isr cfg mapped at: %p", hw->isr); + VIRTIO_CRYPTO_INIT_LOG_DBG("notify base: %p, notify off multiplier: %u", + hw->notify_base, hw->notify_off_multiplier); + + return 0; +} + +/* + * Return -1: + * if there is error mapping with VFIO/UIO. + * if port map error when driver type is KDRV_NONE. + * if whitelisted but driver type is KDRV_UNKNOWN. + * Return 1 if kernel driver is managing the device. + * Return 0 on success. + */ +int +vtpci_cryptodev_init(struct rte_pci_device *dev, struct virtio_crypto_hw *hw) +{ + /* + * Try if we can succeed reading virtio pci caps, which exists + * only on modern pci device. If failed, we fallback to legacy + * virtio handling. + */ + if (virtio_read_caps(dev, hw) == 0) { + VIRTIO_CRYPTO_INIT_LOG_INFO("modern virtio pci detected."); + virtio_hw_internal[hw->dev_id].vtpci_ops = + &virtio_crypto_modern_ops; + hw->modern = 1; + return 0; + } + + /* + * virtio crypto conforms to virtio 1.0 and doesn't support + * legacy mode + */ + return -1; +} diff --git a/drivers/crypto/virtio/virtio_pci.h b/drivers/crypto/virtio/virtio_pci.h new file mode 100644 index 0000000000..604ec36623 --- /dev/null +++ b/drivers/crypto/virtio/virtio_pci.h @@ -0,0 +1,253 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD. + */ + +#ifndef _VIRTIO_PCI_H_ +#define _VIRTIO_PCI_H_ + +#include + +#include +#include +#include + +#include "virtio_crypto.h" + +struct virtqueue; + +/* VirtIO PCI vendor/device ID. */ +#define VIRTIO_CRYPTO_PCI_VENDORID 0x1AF4 +#define VIRTIO_CRYPTO_PCI_DEVICEID 0x1054 + +/* VirtIO ABI version, this must match exactly. */ +#define VIRTIO_PCI_ABI_VERSION 0 + +/* + * VirtIO Header, located in BAR 0. + */ +#define VIRTIO_PCI_HOST_FEATURES 0 /* host's supported features (32bit, RO)*/ +#define VIRTIO_PCI_GUEST_FEATURES 4 /* guest's supported features (32, RW) */ +#define VIRTIO_PCI_QUEUE_PFN 8 /* physical address of VQ (32, RW) */ +#define VIRTIO_PCI_QUEUE_NUM 12 /* number of ring entries (16, RO) */ +#define VIRTIO_PCI_QUEUE_SEL 14 /* current VQ selection (16, RW) */ +#define VIRTIO_PCI_QUEUE_NOTIFY 16 /* notify host regarding VQ (16, RW) */ +#define VIRTIO_PCI_STATUS 18 /* device status register (8, RW) */ +#define VIRTIO_PCI_ISR 19 /* interrupt status register, reading + * also clears the register (8, RO) + */ +/* Only if MSIX is enabled: */ + +/* configuration change vector (16, RW) */ +#define VIRTIO_MSI_CONFIG_VECTOR 20 +/* vector for selected VQ notifications */ +#define VIRTIO_MSI_QUEUE_VECTOR 22 + +/* The bit of the ISR which indicates a device has an interrupt. */ +#define VIRTIO_PCI_ISR_INTR 0x1 +/* The bit of the ISR which indicates a device configuration change. */ +#define VIRTIO_PCI_ISR_CONFIG 0x2 +/* Vector value used to disable MSI for queue. */ +#define VIRTIO_MSI_NO_VECTOR 0xFFFF + +/* Status byte for guest to report progress. */ +#define VIRTIO_CONFIG_STATUS_RESET 0x00 +#define VIRTIO_CONFIG_STATUS_ACK 0x01 +#define VIRTIO_CONFIG_STATUS_DRIVER 0x02 +#define VIRTIO_CONFIG_STATUS_DRIVER_OK 0x04 +#define VIRTIO_CONFIG_STATUS_FEATURES_OK 0x08 +#define VIRTIO_CONFIG_STATUS_FAILED 0x80 + +/* + * Each virtqueue indirect descriptor list must be physically contiguous. + * To allow us to malloc(9) each list individually, limit the number + * supported to what will fit in one page. With 4KB pages, this is a limit + * of 256 descriptors. If there is ever a need for more, we can switch to + * contigmalloc(9) for the larger allocations, similar to what + * bus_dmamem_alloc(9) does. + * + * Note the sizeof(struct vring_desc) is 16 bytes. + */ +#define VIRTIO_MAX_INDIRECT ((int) (PAGE_SIZE / 16)) + +/* Do we get callbacks when the ring is completely used, even if we've + * suppressed them? + */ +#define VIRTIO_F_NOTIFY_ON_EMPTY 24 + +/* Can the device handle any descriptor layout? */ +#define VIRTIO_F_ANY_LAYOUT 27 + +/* We support indirect buffer descriptors */ +#define VIRTIO_RING_F_INDIRECT_DESC 28 + +#define VIRTIO_F_VERSION_1 32 +#define VIRTIO_F_IOMMU_PLATFORM 33 + +/* The Guest publishes the used index for which it expects an interrupt + * at the end of the avail ring. Host should ignore the avail->flags field. + */ +/* The Host publishes the avail index for which it expects a kick + * at the end of the used ring. Guest should ignore the used->flags field. + */ +#define VIRTIO_RING_F_EVENT_IDX 29 + +/* Common configuration */ +#define VIRTIO_PCI_CAP_COMMON_CFG 1 +/* Notifications */ +#define VIRTIO_PCI_CAP_NOTIFY_CFG 2 +/* ISR Status */ +#define VIRTIO_PCI_CAP_ISR_CFG 3 +/* Device specific configuration */ +#define VIRTIO_PCI_CAP_DEVICE_CFG 4 +/* PCI configuration access */ +#define VIRTIO_PCI_CAP_PCI_CFG 5 + +/* This is the PCI capability header: */ +struct virtio_pci_cap { + uint8_t cap_vndr; /* Generic PCI field: PCI_CAP_ID_VNDR */ + uint8_t cap_next; /* Generic PCI field: next ptr. */ + uint8_t cap_len; /* Generic PCI field: capability length */ + uint8_t cfg_type; /* Identifies the structure. */ + uint8_t bar; /* Where to find it. */ + uint8_t padding[3]; /* Pad to full dword. */ + uint32_t offset; /* Offset within bar. */ + uint32_t length; /* Length of the structure, in bytes. */ +}; + +struct virtio_pci_notify_cap { + struct virtio_pci_cap cap; + uint32_t notify_off_multiplier; /* Multiplier for queue_notify_off. */ +}; + +/* Fields in VIRTIO_PCI_CAP_COMMON_CFG: */ +struct virtio_pci_common_cfg { + /* About the whole device. */ + uint32_t device_feature_select; /* read-write */ + uint32_t device_feature; /* read-only */ + uint32_t guest_feature_select; /* read-write */ + uint32_t guest_feature; /* read-write */ + uint16_t msix_config; /* read-write */ + uint16_t num_queues; /* read-only */ + uint8_t device_status; /* read-write */ + uint8_t config_generation; /* read-only */ + + /* About a specific virtqueue. */ + uint16_t queue_select; /* read-write */ + uint16_t queue_size; /* read-write, power of 2. */ + uint16_t queue_msix_vector; /* read-write */ + uint16_t queue_enable; /* read-write */ + uint16_t queue_notify_off; /* read-only */ + uint32_t queue_desc_lo; /* read-write */ + uint32_t queue_desc_hi; /* read-write */ + uint32_t queue_avail_lo; /* read-write */ + uint32_t queue_avail_hi; /* read-write */ + uint32_t queue_used_lo; /* read-write */ + uint32_t queue_used_hi; /* read-write */ +}; + +struct virtio_crypto_hw; + +struct virtio_pci_ops { + void (*read_dev_cfg)(struct virtio_crypto_hw *hw, size_t offset, + void *dst, int len); + void (*write_dev_cfg)(struct virtio_crypto_hw *hw, size_t offset, + const void *src, int len); + void (*reset)(struct virtio_crypto_hw *hw); + + uint8_t (*get_status)(struct virtio_crypto_hw *hw); + void (*set_status)(struct virtio_crypto_hw *hw, uint8_t status); + + uint64_t (*get_features)(struct virtio_crypto_hw *hw); + void (*set_features)(struct virtio_crypto_hw *hw, uint64_t features); + + uint8_t (*get_isr)(struct virtio_crypto_hw *hw); + + uint16_t (*set_config_irq)(struct virtio_crypto_hw *hw, uint16_t vec); + + uint16_t (*set_queue_irq)(struct virtio_crypto_hw *hw, + struct virtqueue *vq, uint16_t vec); + + uint16_t (*get_queue_num)(struct virtio_crypto_hw *hw, + uint16_t queue_id); + int (*setup_queue)(struct virtio_crypto_hw *hw, struct virtqueue *vq); + void (*del_queue)(struct virtio_crypto_hw *hw, struct virtqueue *vq); + void (*notify_queue)(struct virtio_crypto_hw *hw, struct virtqueue *vq); +}; + +struct virtio_crypto_hw { + /* control queue */ + struct virtqueue *cvq; + uint16_t dev_id; + uint16_t max_dataqueues; + uint64_t req_guest_features; + uint64_t guest_features; + uint8_t use_msix; + uint8_t modern; + uint32_t notify_off_multiplier; + uint8_t *isr; + uint16_t *notify_base; + struct virtio_pci_common_cfg *common_cfg; + struct virtio_crypto_config *dev_cfg; + const struct rte_cryptodev_capabilities *virtio_dev_capabilities; +}; + +/* + * While virtio_crypto_hw is stored in shared memory, this structure stores + * some infos that may vary in the multiple process model locally. + * For example, the vtpci_ops pointer. + */ +struct virtio_hw_internal { + const struct virtio_pci_ops *vtpci_ops; + struct rte_pci_ioport io; +}; + +#define VTPCI_OPS(hw) (virtio_hw_internal[(hw)->dev_id].vtpci_ops) +#define VTPCI_IO(hw) (&virtio_hw_internal[(hw)->dev_id].io) + +extern struct virtio_hw_internal virtio_hw_internal[RTE_MAX_VIRTIO_CRYPTO]; + +/* + * How many bits to shift physical queue address written to QUEUE_PFN. + * 12 is historical, and due to x86 page size. + */ +#define VIRTIO_PCI_QUEUE_ADDR_SHIFT 12 + +/* The alignment to use between consumer and producer parts of vring. */ +#define VIRTIO_PCI_VRING_ALIGN 4096 + +enum virtio_msix_status { + VIRTIO_MSIX_NONE = 0, + VIRTIO_MSIX_DISABLED = 1, + VIRTIO_MSIX_ENABLED = 2 +}; + +static inline int +vtpci_with_feature(struct virtio_crypto_hw *hw, uint64_t bit) +{ + return (hw->guest_features & (1ULL << bit)) != 0; +} + +/* + * Function declaration from virtio_pci.c + */ +int vtpci_cryptodev_init(struct rte_pci_device *dev, + struct virtio_crypto_hw *hw); +void vtpci_cryptodev_reset(struct virtio_crypto_hw *hw); + +void vtpci_cryptodev_reinit_complete(struct virtio_crypto_hw *hw); + +uint8_t vtpci_cryptodev_get_status(struct virtio_crypto_hw *hw); +void vtpci_cryptodev_set_status(struct virtio_crypto_hw *hw, uint8_t status); + +uint64_t vtpci_cryptodev_negotiate_features(struct virtio_crypto_hw *hw, + uint64_t host_features); + +void vtpci_write_cryptodev_config(struct virtio_crypto_hw *hw, size_t offset, + const void *src, int length); + +void vtpci_read_cryptodev_config(struct virtio_crypto_hw *hw, size_t offset, + void *dst, int length); + +uint8_t vtpci_cryptodev_isr(struct virtio_crypto_hw *hw); + +#endif /* _VIRTIO_PCI_H_ */ diff --git a/drivers/crypto/virtio/virtio_ring.h b/drivers/crypto/virtio/virtio_ring.h new file mode 100644 index 0000000000..ee30674566 --- /dev/null +++ b/drivers/crypto/virtio/virtio_ring.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD. + */ + +#ifndef _VIRTIO_RING_H_ +#define _VIRTIO_RING_H_ + +#include + +#include + +/* This marks a buffer as continuing via the next field. */ +#define VRING_DESC_F_NEXT 1 +/* This marks a buffer as write-only (otherwise read-only). */ +#define VRING_DESC_F_WRITE 2 +/* This means the buffer contains a list of buffer descriptors. */ +#define VRING_DESC_F_INDIRECT 4 + +/* The Host uses this in used->flags to advise the Guest: don't kick me + * when you add a buffer. It's unreliable, so it's simply an + * optimization. Guest will still kick if it's out of buffers. + */ +#define VRING_USED_F_NO_NOTIFY 1 +/* The Guest uses this in avail->flags to advise the Host: don't + * interrupt me when you consume a buffer. It's unreliable, so it's + * simply an optimization. + */ +#define VRING_AVAIL_F_NO_INTERRUPT 1 + +/* VirtIO ring descriptors: 16 bytes. + * These can chain together via "next". + */ +struct vring_desc { + uint64_t addr; /* Address (guest-physical). */ + uint32_t len; /* Length. */ + uint16_t flags; /* The flags as indicated above. */ + uint16_t next; /* We chain unused descriptors via this. */ +}; + +struct vring_avail { + uint16_t flags; + uint16_t idx; + uint16_t ring[0]; +}; + +/* id is a 16bit index. uint32_t is used here for ids for padding reasons. */ +struct vring_used_elem { + /* Index of start of used descriptor chain. */ + uint32_t id; + /* Total length of the descriptor chain which was written to. */ + uint32_t len; +}; + +struct vring_used { + uint16_t flags; + volatile uint16_t idx; + struct vring_used_elem ring[0]; +}; + +struct vring { + unsigned int num; + struct vring_desc *desc; + struct vring_avail *avail; + struct vring_used *used; +}; + +/* The standard layout for the ring is a continuous chunk of memory which + * looks like this. We assume num is a power of 2. + * + * struct vring { + * // The actual descriptors (16 bytes each) + * struct vring_desc desc[num]; + * + * // A ring of available descriptor heads with free-running index. + * __u16 avail_flags; + * __u16 avail_idx; + * __u16 available[num]; + * __u16 used_event_idx; + * + * // Padding to the next align boundary. + * char pad[]; + * + * // A ring of used descriptor heads with free-running index. + * __u16 used_flags; + * __u16 used_idx; + * struct vring_used_elem used[num]; + * __u16 avail_event_idx; + * }; + * + * NOTE: for VirtIO PCI, align is 4096. + */ + +/* + * We publish the used event index at the end of the available ring, and vice + * versa. They are at the end for backwards compatibility. + */ +#define vring_used_event(vr) ((vr)->avail->ring[(vr)->num]) +#define vring_avail_event(vr) (*(uint16_t *)&(vr)->used->ring[(vr)->num]) + +static inline size_t +vring_size(unsigned int num, unsigned long align) +{ + size_t size; + + size = num * sizeof(struct vring_desc); + size += sizeof(struct vring_avail) + (num * sizeof(uint16_t)); + size = RTE_ALIGN_CEIL(size, align); + size += sizeof(struct vring_used) + + (num * sizeof(struct vring_used_elem)); + return size; +} + +static inline void +vring_init(struct vring *vr, unsigned int num, uint8_t *p, + unsigned long align) +{ + vr->num = num; + vr->desc = (struct vring_desc *) p; + vr->avail = (struct vring_avail *) (p + + num * sizeof(struct vring_desc)); + vr->used = (void *) + RTE_ALIGN_CEIL((uintptr_t)(&vr->avail->ring[num]), align); +} + +/* + * The following is used with VIRTIO_RING_F_EVENT_IDX. + * Assuming a given event_idx value from the other size, if we have + * just incremented index from old to new_idx, should we trigger an + * event? + */ +static inline int +vring_need_event(uint16_t event_idx, uint16_t new_idx, uint16_t old) +{ + return (uint16_t)(new_idx - event_idx - 1) < (uint16_t)(new_idx - old); +} + +#endif /* _VIRTIO_RING_H_ */ diff --git a/drivers/crypto/virtio/virtio_rxtx.c b/drivers/crypto/virtio/virtio_rxtx.c new file mode 100644 index 0000000000..51f6e09359 --- /dev/null +++ b/drivers/crypto/virtio/virtio_rxtx.c @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD. + */ +#include "virtio_cryptodev.h" + +uint16_t +virtio_crypto_pkt_rx_burst( + void *tx_queue __rte_unused, + struct rte_crypto_op **rx_pkts __rte_unused, + uint16_t nb_pkts __rte_unused) +{ + uint16_t nb_rx = 0; + + return nb_rx; +} + +uint16_t +virtio_crypto_pkt_tx_burst( + void *tx_queue __rte_unused, + struct rte_crypto_op **tx_pkts __rte_unused, + uint16_t nb_pkts __rte_unused) +{ + uint16_t nb_tx = 0; + + return nb_tx; +} diff --git a/drivers/crypto/virtio/virtqueue.c b/drivers/crypto/virtio/virtqueue.c new file mode 100644 index 0000000000..fd8be581c9 --- /dev/null +++ b/drivers/crypto/virtio/virtqueue.c @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD. + */ + +#include + +#include +#include +#include + +#include "virtqueue.h" + +void +virtqueue_disable_intr(struct virtqueue *vq) +{ + /* + * Set VRING_AVAIL_F_NO_INTERRUPT to hint host + * not to interrupt when it consumes packets + * Note: this is only considered a hint to the host + */ + vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; +} + +void +virtqueue_detatch_unused(struct virtqueue *vq) +{ + struct rte_crypto_op *cop = NULL; + + int idx; + + if (vq != NULL) + for (idx = 0; idx < vq->vq_nentries; idx++) { + cop = vq->vq_descx[idx].crypto_op; + if (cop) { + if (cop->sym->m_src) + rte_pktmbuf_free(cop->sym->m_src); + if (cop->sym->m_dst) + rte_pktmbuf_free(cop->sym->m_dst); + rte_crypto_op_free(cop); + vq->vq_descx[idx].crypto_op = NULL; + } + } +} diff --git a/drivers/crypto/virtio/virtqueue.h b/drivers/crypto/virtio/virtqueue.h new file mode 100644 index 0000000000..bf10c6579b --- /dev/null +++ b/drivers/crypto/virtio/virtqueue.h @@ -0,0 +1,171 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD. + */ + +#ifndef _VIRTQUEUE_H_ +#define _VIRTQUEUE_H_ + +#include + +#include +#include +#include +#include + +#include "virtio_pci.h" +#include "virtio_ring.h" +#include "virtio_logs.h" +#include "virtio_crypto.h" + +struct rte_mbuf; + +/* + * Per virtio_config.h in Linux. + * For virtio_pci on SMP, we don't need to order with respect to MMIO + * accesses through relaxed memory I/O windows, so smp_mb() et al are + * sufficient. + * + */ +#define virtio_mb() rte_smp_mb() +#define virtio_rmb() rte_smp_rmb() +#define virtio_wmb() rte_smp_wmb() + +#define VIRTQUEUE_MAX_NAME_SZ 32 + +enum { VTCRYPTO_DATAQ = 0, VTCRYPTO_CTRLQ = 1 }; + +/** + * The maximum virtqueue size is 2^15. Use that value as the end of + * descriptor chain terminator since it will never be a valid index + * in the descriptor table. This is used to verify we are correctly + * handling vq_free_cnt. + */ +#define VQ_RING_DESC_CHAIN_END 32768 + +struct vq_desc_extra { + void *crypto_op; + void *cookie; + uint16_t ndescs; +}; + +struct virtqueue { + /**< virtio_crypto_hw structure pointer. */ + struct virtio_crypto_hw *hw; + /**< mem zone to populate RX ring. */ + const struct rte_memzone *mz; + /**< memzone to populate hdr and request. */ + struct rte_mempool *mpool; + uint8_t dev_id; /**< Device identifier. */ + uint16_t vq_queue_index; /**< PCI queue index */ + + void *vq_ring_virt_mem; /**< linear address of vring*/ + unsigned int vq_ring_size; + phys_addr_t vq_ring_mem; /**< physical address of vring */ + + struct vring vq_ring; /**< vring keeping desc, used and avail */ + uint16_t vq_free_cnt; /**< num of desc available */ + uint16_t vq_nentries; /**< vring desc numbers */ + + /** + * Head of the free chain in the descriptor table. If + * there are no free descriptors, this will be set to + * VQ_RING_DESC_CHAIN_END. + */ + uint16_t vq_desc_head_idx; + uint16_t vq_desc_tail_idx; + /** + * Last consumed descriptor in the used table, + * trails vq_ring.used->idx. + */ + uint16_t vq_used_cons_idx; + uint16_t vq_avail_idx; + + /* Statistics */ + uint64_t packets_sent_total; + uint64_t packets_sent_failed; + uint64_t packets_received_total; + uint64_t packets_received_failed; + + uint16_t *notify_addr; + + struct vq_desc_extra vq_descx[0]; +}; + +/** + * Tell the backend not to interrupt us. + */ +void virtqueue_disable_intr(struct virtqueue *vq); + +/** + * Get all mbufs to be freed. + */ +void virtqueue_detatch_unused(struct virtqueue *vq); + +static inline int +virtqueue_full(const struct virtqueue *vq) +{ + return vq->vq_free_cnt == 0; +} + +#define VIRTQUEUE_NUSED(vq) \ + ((uint16_t)((vq)->vq_ring.used->idx - (vq)->vq_used_cons_idx)) + +static inline void +vq_update_avail_idx(struct virtqueue *vq) +{ + virtio_wmb(); + vq->vq_ring.avail->idx = vq->vq_avail_idx; +} + +static inline void +vq_update_avail_ring(struct virtqueue *vq, uint16_t desc_idx) +{ + uint16_t avail_idx; + /* + * Place the head of the descriptor chain into the next slot and make + * it usable to the host. The chain is made available now rather than + * deferring to virtqueue_notify() in the hopes that if the host is + * currently running on another CPU, we can keep it processing the new + * descriptor. + */ + avail_idx = (uint16_t)(vq->vq_avail_idx & (vq->vq_nentries - 1)); + if (unlikely(vq->vq_ring.avail->ring[avail_idx] != desc_idx)) + vq->vq_ring.avail->ring[avail_idx] = desc_idx; + vq->vq_avail_idx++; +} + +static inline int +virtqueue_kick_prepare(struct virtqueue *vq) +{ + return !(vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY); +} + +static inline void +virtqueue_notify(struct virtqueue *vq) +{ + /* + * Ensure updated avail->idx is visible to host. + * For virtio on IA, the notificaiton is through io port operation + * which is a serialization instruction itself. + */ + VTPCI_OPS(vq->hw)->notify_queue(vq->hw, vq); +} + +/** + * Dump virtqueue internal structures, for debug purpose only. + */ +#define VIRTQUEUE_DUMP(vq) do { \ + uint16_t used_idx, nused; \ + used_idx = (vq)->vq_ring.used->idx; \ + nused = (uint16_t)(used_idx - (vq)->vq_used_cons_idx); \ + VIRTIO_CRYPTO_INIT_LOG_DBG(\ + "VQ: - size=%d; free=%d; used=%d; desc_head_idx=%d;" \ + " avail.idx=%d; used_cons_idx=%d; used.idx=%d;" \ + " avail.flags=0x%x; used.flags=0x%x", \ + (vq)->vq_nentries, (vq)->vq_free_cnt, nused, \ + (vq)->vq_desc_head_idx, (vq)->vq_ring.avail->idx, \ + (vq)->vq_used_cons_idx, (vq)->vq_ring.used->idx, \ + (vq)->vq_ring.avail->flags, (vq)->vq_ring.used->flags); \ +} while (0) + +#endif /* _VIRTQUEUE_H_ */ -- 2.20.1