crypto/virtio: support basic PMD ops
authorJay Zhou <jianjay.zhou@huawei.com>
Tue, 17 Apr 2018 09:23:19 +0000 (17:23 +0800)
committerPablo de Lara <pablo.de.lara.guarch@intel.com>
Mon, 23 Apr 2018 17:20:10 +0000 (18:20 +0100)
This patch implements the basic operations of virtio crypto PMD, which
includes start, stop, close, information getting, queue setup and
release of the device.
The virtio crypto device has two types of queues, data queue and
control queue. It has one data queue at least and has one and only one
control queue. For example, if a virtio crypto device has N queues,
then [0, N-2] is the data queue index, N-1 is the control
queue index.

Signed-off-by: Jay Zhou <jianjay.zhou@huawei.com>
Reviewed-by: Fan Zhang <roy.fan.zhang@intel.com>
drivers/crypto/virtio/virtio_cryptodev.c
drivers/crypto/virtio/virtio_cryptodev.h
drivers/crypto/virtio/virtio_rxtx.c

index 1afa3a1..e23b827 100644 (file)
@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
  */
+#include <rte_errno.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
 #include <rte_cryptodev.h>
@@ -16,6 +17,22 @@ int virtio_crypto_logtype_rx;
 int virtio_crypto_logtype_tx;
 int virtio_crypto_logtype_driver;
 
+static int virtio_crypto_dev_configure(struct rte_cryptodev *dev,
+               struct rte_cryptodev_config *config);
+static int virtio_crypto_dev_start(struct rte_cryptodev *dev);
+static void virtio_crypto_dev_stop(struct rte_cryptodev *dev);
+static int virtio_crypto_dev_close(struct rte_cryptodev *dev);
+static void virtio_crypto_dev_info_get(struct rte_cryptodev *dev,
+               struct rte_cryptodev_info *dev_info);
+static int virtio_crypto_qp_setup(struct rte_cryptodev *dev,
+               uint16_t queue_pair_id,
+               const struct rte_cryptodev_qp_conf *qp_conf,
+               int socket_id,
+               struct rte_mempool *session_pool);
+static int virtio_crypto_qp_release(struct rte_cryptodev *dev,
+               uint16_t queue_pair_id);
+static void virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev);
+
 /*
  * The set of PCI devices this driver supports
  */
@@ -27,22 +44,251 @@ static const struct rte_pci_id pci_id_virtio_crypto_map[] = {
 
 uint8_t cryptodev_virtio_driver_id;
 
+void
+virtio_crypto_queue_release(struct virtqueue *vq)
+{
+       struct virtio_crypto_hw *hw;
+
+       PMD_INIT_FUNC_TRACE();
+
+       if (vq) {
+               hw = vq->hw;
+               /* Select and deactivate the queue */
+               VTPCI_OPS(hw)->del_queue(hw, vq);
+
+               rte_memzone_free(vq->mz);
+               rte_mempool_free(vq->mpool);
+               rte_free(vq);
+       }
+}
+
+#define MPOOL_MAX_NAME_SZ 32
+
+int
+virtio_crypto_queue_setup(struct rte_cryptodev *dev,
+               int queue_type,
+               uint16_t vtpci_queue_idx,
+               uint16_t nb_desc,
+               int socket_id,
+               struct virtqueue **pvq)
+{
+       char vq_name[VIRTQUEUE_MAX_NAME_SZ];
+       char mpool_name[MPOOL_MAX_NAME_SZ];
+       const struct rte_memzone *mz;
+       unsigned int vq_size, size;
+       struct virtio_crypto_hw *hw = dev->data->dev_private;
+       struct virtqueue *vq = NULL;
+       uint32_t i = 0;
+       uint32_t j;
+
+       PMD_INIT_FUNC_TRACE();
+
+       VIRTIO_CRYPTO_INIT_LOG_DBG("setting up queue: %u", vtpci_queue_idx);
+
+       /*
+        * Read the virtqueue size from the Queue Size field
+        * Always power of 2 and if 0 virtqueue does not exist
+        */
+       vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx);
+       if (vq_size == 0) {
+               VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue does not exist");
+               return -EINVAL;
+       }
+       VIRTIO_CRYPTO_INIT_LOG_DBG("vq_size: %u", vq_size);
+
+       if (!rte_is_power_of_2(vq_size)) {
+               VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue size is not powerof 2");
+               return -EINVAL;
+       }
+
+       if (queue_type == VTCRYPTO_DATAQ) {
+               snprintf(vq_name, sizeof(vq_name), "dev%d_dataqueue%d",
+                               dev->data->dev_id, vtpci_queue_idx);
+               snprintf(mpool_name, sizeof(mpool_name),
+                               "dev%d_dataqueue%d_mpool",
+                               dev->data->dev_id, vtpci_queue_idx);
+       } else if (queue_type == VTCRYPTO_CTRLQ) {
+               snprintf(vq_name, sizeof(vq_name), "dev%d_controlqueue",
+                               dev->data->dev_id);
+               snprintf(mpool_name, sizeof(mpool_name),
+                               "dev%d_controlqueue_mpool",
+                               dev->data->dev_id);
+       }
+       size = RTE_ALIGN_CEIL(sizeof(*vq) +
+                               vq_size * sizeof(struct vq_desc_extra),
+                               RTE_CACHE_LINE_SIZE);
+       vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE,
+                               socket_id);
+       if (vq == NULL) {
+               VIRTIO_CRYPTO_INIT_LOG_ERR("Can not allocate virtqueue");
+               return -ENOMEM;
+       }
+
+       if (queue_type == VTCRYPTO_DATAQ) {
+               /* pre-allocate a mempool and use it in the data plane to
+                * improve performance
+                */
+               vq->mpool = rte_mempool_lookup(mpool_name);
+               if (vq->mpool == NULL)
+                       vq->mpool = rte_mempool_create(mpool_name,
+                                       vq_size,
+                                       sizeof(struct virtio_crypto_op_cookie),
+                                       RTE_CACHE_LINE_SIZE, 0,
+                                       NULL, NULL, NULL, NULL, socket_id,
+                                       0);
+               if (!vq->mpool) {
+                       VIRTIO_CRYPTO_DRV_LOG_ERR("Virtio Crypto PMD "
+                                       "Cannot create mempool");
+                       goto mpool_create_err;
+               }
+               for (i = 0; i < vq_size; i++) {
+                       vq->vq_descx[i].cookie =
+                               rte_zmalloc("crypto PMD op cookie pointer",
+                                       sizeof(struct virtio_crypto_op_cookie),
+                                       RTE_CACHE_LINE_SIZE);
+                       if (vq->vq_descx[i].cookie == NULL) {
+                               VIRTIO_CRYPTO_DRV_LOG_ERR("Failed to "
+                                               "alloc mem for cookie");
+                               goto cookie_alloc_err;
+                       }
+               }
+       }
+
+       vq->hw = hw;
+       vq->dev_id = dev->data->dev_id;
+       vq->vq_queue_index = vtpci_queue_idx;
+       vq->vq_nentries = vq_size;
+
+       /*
+        * Using part of the vring entries is permitted, but the maximum
+        * is vq_size
+        */
+       if (nb_desc == 0 || nb_desc > vq_size)
+               nb_desc = vq_size;
+       vq->vq_free_cnt = nb_desc;
+
+       /*
+        * Reserve a memzone for vring elements
+        */
+       size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN);
+       vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);
+       VIRTIO_CRYPTO_INIT_LOG_DBG("%s vring_size: %d, rounded_vring_size: %d",
+                       (queue_type == VTCRYPTO_DATAQ) ? "dataq" : "ctrlq",
+                       size, vq->vq_ring_size);
+
+       mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
+                       socket_id, 0, VIRTIO_PCI_VRING_ALIGN);
+       if (mz == NULL) {
+               if (rte_errno == EEXIST)
+                       mz = rte_memzone_lookup(vq_name);
+               if (mz == NULL) {
+                       VIRTIO_CRYPTO_INIT_LOG_ERR("not enough memory");
+                       goto mz_reserve_err;
+               }
+       }
+
+       /*
+        * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit,
+        * and only accepts 32 bit page frame number.
+        * Check if the allocated physical memory exceeds 16TB.
+        */
+       if ((mz->phys_addr + vq->vq_ring_size - 1)
+                               >> (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) {
+               VIRTIO_CRYPTO_INIT_LOG_ERR("vring address shouldn't be "
+                                       "above 16TB!");
+               goto vring_addr_err;
+       }
+
+       memset(mz->addr, 0, sizeof(mz->len));
+       vq->mz = mz;
+       vq->vq_ring_mem = mz->phys_addr;
+       vq->vq_ring_virt_mem = mz->addr;
+       VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_mem(physical): 0x%"PRIx64,
+                                       (uint64_t)mz->phys_addr);
+       VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_virt_mem: 0x%"PRIx64,
+                                       (uint64_t)(uintptr_t)mz->addr);
+
+       *pvq = vq;
+
+       return 0;
+
+vring_addr_err:
+       rte_memzone_free(mz);
+mz_reserve_err:
+cookie_alloc_err:
+       rte_mempool_free(vq->mpool);
+       if (i != 0) {
+               for (j = 0; j < i; j++)
+                       rte_free(vq->vq_descx[j].cookie);
+       }
+mpool_create_err:
+       rte_free(vq);
+       return -ENOMEM;
+}
+
+static int
+virtio_crypto_ctrlq_setup(struct rte_cryptodev *dev, uint16_t queue_idx)
+{
+       int ret;
+       struct virtqueue *vq;
+       struct virtio_crypto_hw *hw = dev->data->dev_private;
+
+       /* if virtio device has started, do not touch the virtqueues */
+       if (dev->data->dev_started)
+               return 0;
+
+       PMD_INIT_FUNC_TRACE();
+
+       ret = virtio_crypto_queue_setup(dev, VTCRYPTO_CTRLQ, queue_idx,
+                       0, SOCKET_ID_ANY, &vq);
+       if (ret < 0) {
+               VIRTIO_CRYPTO_INIT_LOG_ERR("control vq initialization failed");
+               return ret;
+       }
+
+       hw->cvq = vq;
+
+       return 0;
+}
+
+static void
+virtio_crypto_free_queues(struct rte_cryptodev *dev)
+{
+       unsigned int i;
+       struct virtio_crypto_hw *hw = dev->data->dev_private;
+
+       PMD_INIT_FUNC_TRACE();
+
+       /* control queue release */
+       virtio_crypto_queue_release(hw->cvq);
+
+       /* data queue release */
+       for (i = 0; i < hw->max_dataqueues; i++)
+               virtio_crypto_queue_release(dev->data->queue_pairs[i]);
+}
+
+static int
+virtio_crypto_dev_close(struct rte_cryptodev *dev __rte_unused)
+{
+       return 0;
+}
+
 /*
  * dev_ops for virtio, bare necessities for basic operation
  */
 static struct rte_cryptodev_ops virtio_crypto_dev_ops = {
        /* Device related operations */
-       .dev_configure                   = NULL,
-       .dev_start                       = NULL,
-       .dev_stop                        = NULL,
-       .dev_close                       = NULL,
-       .dev_infos_get                   = NULL,
+       .dev_configure                   = virtio_crypto_dev_configure,
+       .dev_start                       = virtio_crypto_dev_start,
+       .dev_stop                        = virtio_crypto_dev_stop,
+       .dev_close                       = virtio_crypto_dev_close,
+       .dev_infos_get                   = virtio_crypto_dev_info_get,
 
        .stats_get                       = NULL,
        .stats_reset                     = NULL,
 
-       .queue_pair_setup                = NULL,
-       .queue_pair_release              = NULL,
+       .queue_pair_setup                = virtio_crypto_qp_setup,
+       .queue_pair_release              = virtio_crypto_qp_release,
        .queue_pair_start                = NULL,
        .queue_pair_stop                 = NULL,
        .queue_pair_count                = NULL,
@@ -55,6 +301,51 @@ static struct rte_cryptodev_ops virtio_crypto_dev_ops = {
        .qp_detach_session = NULL
 };
 
+static int
+virtio_crypto_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id,
+               const struct rte_cryptodev_qp_conf *qp_conf,
+               int socket_id,
+               struct rte_mempool *session_pool __rte_unused)
+{
+       int ret;
+       struct virtqueue *vq;
+
+       PMD_INIT_FUNC_TRACE();
+
+       /* if virtio dev is started, do not touch the virtqueues */
+       if (dev->data->dev_started)
+               return 0;
+
+       ret = virtio_crypto_queue_setup(dev, VTCRYPTO_DATAQ, queue_pair_id,
+                       qp_conf->nb_descriptors, socket_id, &vq);
+       if (ret < 0) {
+               VIRTIO_CRYPTO_INIT_LOG_ERR(
+                       "virtio crypto data queue initialization failed\n");
+               return ret;
+       }
+
+       dev->data->queue_pairs[queue_pair_id] = vq;
+
+       return 0;
+}
+
+static int
+virtio_crypto_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
+{
+       struct virtqueue *vq
+               = (struct virtqueue *)dev->data->queue_pairs[queue_pair_id];
+
+       PMD_INIT_FUNC_TRACE();
+
+       if (vq == NULL) {
+               VIRTIO_CRYPTO_DRV_LOG_DBG("vq already freed");
+               return 0;
+       }
+
+       virtio_crypto_queue_release(vq);
+       return 0;
+}
+
 static int
 virtio_negotiate_features(struct virtio_crypto_hw *hw, uint64_t req_features)
 {
@@ -192,6 +483,136 @@ crypto_virtio_create(const char *name, struct rte_pci_device *pci_dev,
        return 0;
 }
 
+static int
+virtio_crypto_dev_uninit(struct rte_cryptodev *cryptodev)
+{
+       struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
+
+       PMD_INIT_FUNC_TRACE();
+
+       if (rte_eal_process_type() == RTE_PROC_SECONDARY)
+               return -EPERM;
+
+       if (cryptodev->data->dev_started) {
+               virtio_crypto_dev_stop(cryptodev);
+               virtio_crypto_dev_close(cryptodev);
+       }
+
+       cryptodev->dev_ops = NULL;
+       cryptodev->enqueue_burst = NULL;
+       cryptodev->dequeue_burst = NULL;
+
+       /* release control queue */
+       virtio_crypto_queue_release(hw->cvq);
+
+       rte_free(cryptodev->data);
+       cryptodev->data = NULL;
+
+       VIRTIO_CRYPTO_DRV_LOG_INFO("dev_uninit completed");
+
+       return 0;
+}
+
+static int
+virtio_crypto_dev_configure(struct rte_cryptodev *cryptodev,
+       struct rte_cryptodev_config *config __rte_unused)
+{
+       struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
+
+       PMD_INIT_FUNC_TRACE();
+
+       if (virtio_crypto_init_device(cryptodev,
+                       VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0)
+               return -1;
+
+       /* setup control queue
+        * [0, 1, ... ,(config->max_dataqueues - 1)] are data queues
+        * config->max_dataqueues is the control queue
+        */
+       if (virtio_crypto_ctrlq_setup(cryptodev, hw->max_dataqueues) < 0) {
+               VIRTIO_CRYPTO_INIT_LOG_ERR("control queue setup error");
+               return -1;
+       }
+       virtio_crypto_ctrlq_start(cryptodev);
+
+       return 0;
+}
+
+static void
+virtio_crypto_dev_stop(struct rte_cryptodev *dev)
+{
+       struct virtio_crypto_hw *hw = dev->data->dev_private;
+
+       PMD_INIT_FUNC_TRACE();
+       VIRTIO_CRYPTO_DRV_LOG_DBG("virtio_dev_stop");
+
+       vtpci_cryptodev_reset(hw);
+
+       virtio_crypto_dev_free_mbufs(dev);
+       virtio_crypto_free_queues(dev);
+
+       dev->data->dev_started = 0;
+}
+
+static int
+virtio_crypto_dev_start(struct rte_cryptodev *dev)
+{
+       struct virtio_crypto_hw *hw = dev->data->dev_private;
+
+       if (dev->data->dev_started)
+               return 0;
+
+       /* Do final configuration before queue engine starts */
+       virtio_crypto_dataq_start(dev);
+       vtpci_cryptodev_reinit_complete(hw);
+
+       dev->data->dev_started = 1;
+
+       return 0;
+}
+
+static void
+virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev)
+{
+       uint32_t i;
+       struct virtio_crypto_hw *hw = dev->data->dev_private;
+
+       for (i = 0; i < hw->max_dataqueues; i++) {
+               VIRTIO_CRYPTO_INIT_LOG_DBG("Before freeing dataq[%d] used "
+                       "and unused buf", i);
+               VIRTQUEUE_DUMP((struct virtqueue *)
+                       dev->data->queue_pairs[i]);
+
+               VIRTIO_CRYPTO_INIT_LOG_DBG("queue_pairs[%d]=%p",
+                               i, dev->data->queue_pairs[i]);
+
+               virtqueue_detatch_unused(dev->data->queue_pairs[i]);
+
+               VIRTIO_CRYPTO_INIT_LOG_DBG("After freeing dataq[%d] used and "
+                                       "unused buf", i);
+               VIRTQUEUE_DUMP(
+                       (struct virtqueue *)dev->data->queue_pairs[i]);
+       }
+}
+
+static void
+virtio_crypto_dev_info_get(struct rte_cryptodev *dev,
+               struct rte_cryptodev_info *info)
+{
+       struct virtio_crypto_hw *hw = dev->data->dev_private;
+
+       PMD_INIT_FUNC_TRACE();
+
+       if (info != NULL) {
+               info->driver_id = cryptodev_virtio_driver_id;
+               info->pci_dev = RTE_DEV_TO_PCI(dev->device);
+               info->feature_flags = dev->feature_flags;
+               info->max_nb_queue_pairs = hw->max_dataqueues;
+               info->sym.max_nb_sessions =
+                       RTE_VIRTIO_CRYPTO_PMD_MAX_NB_SESSIONS;
+       }
+}
+
 static int
 crypto_virtio_pci_probe(
        struct rte_pci_driver *pci_drv __rte_unused,
@@ -232,7 +653,7 @@ crypto_virtio_pci_remove(
        if (cryptodev == NULL)
                return -ENODEV;
 
-       return 0;
+       return virtio_crypto_dev_uninit(cryptodev);
 }
 
 static struct rte_pci_driver rte_virtio_crypto_driver = {
index 392db4a..4f33684 100644 (file)
@@ -5,13 +5,42 @@
 #ifndef _VIRTIO_CRYPTODEV_H_
 #define _VIRTIO_CRYPTODEV_H_
 
-#include <rte_cryptodev.h>
+#include "virtio_crypto.h"
+#include "virtio_pci.h"
+#include "virtio_ring.h"
 
 /* Features desired/implemented by this driver. */
 #define VIRTIO_CRYPTO_PMD_GUEST_FEATURES (1ULL << VIRTIO_F_VERSION_1)
 
 #define CRYPTODEV_NAME_VIRTIO_PMD crypto_virtio
 
+#define NUM_ENTRY_VIRTIO_CRYPTO_OP 7
+
+struct virtio_crypto_op_cookie {
+       struct virtio_crypto_op_data_req data_req;
+       struct virtio_crypto_inhdr inhdr;
+       struct vring_desc desc[NUM_ENTRY_VIRTIO_CRYPTO_OP];
+};
+
+/*
+ * Control queue function prototype
+ */
+void virtio_crypto_ctrlq_start(struct rte_cryptodev *dev);
+
+/*
+ * Data queue function prototype
+ */
+void virtio_crypto_dataq_start(struct rte_cryptodev *dev);
+
+int virtio_crypto_queue_setup(struct rte_cryptodev *dev,
+               int queue_type,
+               uint16_t vtpci_queue_idx,
+               uint16_t nb_desc,
+               int socket_id,
+               struct virtqueue **pvq);
+
+void virtio_crypto_queue_release(struct virtqueue *vq);
+
 uint16_t virtio_crypto_pkt_tx_burst(void *tx_queue,
                struct rte_crypto_op **tx_pkts,
                uint16_t nb_pkts);
index 51f6e09..3b038a7 100644 (file)
@@ -1,8 +1,76 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
  */
+#include "virtqueue.h"
 #include "virtio_cryptodev.h"
 
+static int
+virtio_crypto_vring_start(struct virtqueue *vq)
+{
+       struct virtio_crypto_hw *hw = vq->hw;
+       int i, size = vq->vq_nentries;
+       struct vring *vr = &vq->vq_ring;
+       uint8_t *ring_mem = vq->vq_ring_virt_mem;
+
+       PMD_INIT_FUNC_TRACE();
+
+       vring_init(vr, size, ring_mem, VIRTIO_PCI_VRING_ALIGN);
+       vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
+       vq->vq_free_cnt = vq->vq_nentries;
+
+       /* Chain all the descriptors in the ring with an END */
+       for (i = 0; i < size - 1; i++)
+               vr->desc[i].next = (uint16_t)(i + 1);
+       vr->desc[i].next = VQ_RING_DESC_CHAIN_END;
+
+       /*
+        * Disable device(host) interrupting guest
+        */
+       virtqueue_disable_intr(vq);
+
+       /*
+        * Set guest physical address of the virtqueue
+        * in VIRTIO_PCI_QUEUE_PFN config register of device
+        * to share with the backend
+        */
+       if (VTPCI_OPS(hw)->setup_queue(hw, vq) < 0) {
+               VIRTIO_CRYPTO_INIT_LOG_ERR("setup_queue failed");
+               return -EINVAL;
+       }
+
+       return 0;
+}
+
+void
+virtio_crypto_ctrlq_start(struct rte_cryptodev *dev)
+{
+       struct virtio_crypto_hw *hw = dev->data->dev_private;
+
+       if (hw->cvq) {
+               virtio_crypto_vring_start(hw->cvq);
+               VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq);
+       }
+}
+
+void
+virtio_crypto_dataq_start(struct rte_cryptodev *dev)
+{
+       /*
+        * Start data vrings
+        * -    Setup vring structure for data queues
+        */
+       uint16_t i;
+       struct virtio_crypto_hw *hw = dev->data->dev_private;
+
+       PMD_INIT_FUNC_TRACE();
+
+       /* Start data vring. */
+       for (i = 0; i < hw->max_dataqueues; i++) {
+               virtio_crypto_vring_start(dev->data->queue_pairs[i]);
+               VIRTQUEUE_DUMP((struct virtqueue *)dev->data->queue_pairs[i]);
+       }
+}
+
 uint16_t
 virtio_crypto_pkt_rx_burst(
        void *tx_queue __rte_unused,