1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
7 #include <rte_common.h>
10 #include <rte_bus_pci.h>
11 #include <rte_cryptodev.h>
12 #include <cryptodev_pmd.h>
15 #include "virtio_cryptodev.h"
16 #include "virtqueue.h"
17 #include "virtio_crypto_algs.h"
18 #include "virtio_crypto_capabilities.h"
20 static int virtio_crypto_dev_configure(struct rte_cryptodev *dev,
21 struct rte_cryptodev_config *config);
22 static int virtio_crypto_dev_start(struct rte_cryptodev *dev);
23 static void virtio_crypto_dev_stop(struct rte_cryptodev *dev);
24 static int virtio_crypto_dev_close(struct rte_cryptodev *dev);
25 static void virtio_crypto_dev_info_get(struct rte_cryptodev *dev,
26 struct rte_cryptodev_info *dev_info);
27 static void virtio_crypto_dev_stats_get(struct rte_cryptodev *dev,
28 struct rte_cryptodev_stats *stats);
29 static void virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev);
30 static int virtio_crypto_qp_setup(struct rte_cryptodev *dev,
31 uint16_t queue_pair_id,
32 const struct rte_cryptodev_qp_conf *qp_conf,
34 static int virtio_crypto_qp_release(struct rte_cryptodev *dev,
35 uint16_t queue_pair_id);
36 static void virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev);
37 static unsigned int virtio_crypto_sym_get_session_private_size(
38 struct rte_cryptodev *dev);
39 static void virtio_crypto_sym_clear_session(struct rte_cryptodev *dev,
40 struct rte_cryptodev_sym_session *sess);
41 static int virtio_crypto_sym_configure_session(struct rte_cryptodev *dev,
42 struct rte_crypto_sym_xform *xform,
43 struct rte_cryptodev_sym_session *session,
44 struct rte_mempool *mp);
47 * The set of PCI devices this driver supports
49 static const struct rte_pci_id pci_id_virtio_crypto_map[] = {
50 { RTE_PCI_DEVICE(VIRTIO_CRYPTO_PCI_VENDORID,
51 VIRTIO_CRYPTO_PCI_DEVICEID) },
52 { .vendor_id = 0, /* sentinel */ },
55 static const struct rte_cryptodev_capabilities virtio_capabilities[] = {
56 VIRTIO_SYM_CAPABILITIES,
57 RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
60 uint8_t cryptodev_virtio_driver_id;
62 #define NUM_ENTRY_SYM_CREATE_SESSION 4
65 virtio_crypto_send_command(struct virtqueue *vq,
66 struct virtio_crypto_op_ctrl_req *ctrl, uint8_t *cipher_key,
67 uint8_t *auth_key, struct virtio_crypto_session *session)
72 uint32_t len_cipher_key = 0;
73 uint32_t len_auth_key = 0;
74 uint32_t len_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req);
75 uint32_t len_session_input = sizeof(struct virtio_crypto_session_input);
76 uint32_t len_total = 0;
77 uint32_t input_offset = 0;
78 void *virt_addr_started = NULL;
79 phys_addr_t phys_addr_started;
80 struct vring_desc *desc;
82 struct virtio_crypto_session_input *input;
85 PMD_INIT_FUNC_TRACE();
87 if (session == NULL) {
88 VIRTIO_CRYPTO_SESSION_LOG_ERR("session is NULL.");
91 /* cipher only is supported, it is available if auth_key is NULL */
93 VIRTIO_CRYPTO_SESSION_LOG_ERR("cipher key is NULL.");
97 head = vq->vq_desc_head_idx;
98 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_desc_head_idx = %d, vq = %p",
101 if (vq->vq_free_cnt < needed) {
102 VIRTIO_CRYPTO_SESSION_LOG_ERR("Not enough entry");
106 /* calculate the length of cipher key */
108 switch (ctrl->u.sym_create_session.op_type) {
109 case VIRTIO_CRYPTO_SYM_OP_CIPHER:
111 = ctrl->u.sym_create_session.u.cipher
114 case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING:
116 = ctrl->u.sym_create_session.u.chain
117 .para.cipher_param.keylen;
120 VIRTIO_CRYPTO_SESSION_LOG_ERR("invalid op type");
125 /* calculate the length of auth key */
128 ctrl->u.sym_create_session.u.chain.para.u.mac_param
133 * malloc memory to store indirect vring_desc entries, including
134 * ctrl request, cipher key, auth key, session input and desc vring
136 desc_offset = len_ctrl_req + len_cipher_key + len_auth_key
138 virt_addr_started = rte_malloc(NULL,
139 desc_offset + NUM_ENTRY_SYM_CREATE_SESSION
140 * sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE);
141 if (virt_addr_started == NULL) {
142 VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap memory");
145 phys_addr_started = rte_malloc_virt2iova(virt_addr_started);
147 /* address to store indirect vring desc entries */
148 desc = (struct vring_desc *)
149 ((uint8_t *)virt_addr_started + desc_offset);
152 memcpy(virt_addr_started, ctrl, len_ctrl_req);
153 desc[idx].addr = phys_addr_started;
154 desc[idx].len = len_ctrl_req;
155 desc[idx].flags = VRING_DESC_F_NEXT;
156 desc[idx].next = idx + 1;
158 len_total += len_ctrl_req;
159 input_offset += len_ctrl_req;
161 /* cipher key part */
162 if (len_cipher_key > 0) {
163 memcpy((uint8_t *)virt_addr_started + len_total,
164 cipher_key, len_cipher_key);
166 desc[idx].addr = phys_addr_started + len_total;
167 desc[idx].len = len_cipher_key;
168 desc[idx].flags = VRING_DESC_F_NEXT;
169 desc[idx].next = idx + 1;
171 len_total += len_cipher_key;
172 input_offset += len_cipher_key;
176 if (len_auth_key > 0) {
177 memcpy((uint8_t *)virt_addr_started + len_total,
178 auth_key, len_auth_key);
180 desc[idx].addr = phys_addr_started + len_total;
181 desc[idx].len = len_auth_key;
182 desc[idx].flags = VRING_DESC_F_NEXT;
183 desc[idx].next = idx + 1;
185 len_total += len_auth_key;
186 input_offset += len_auth_key;
190 input = (struct virtio_crypto_session_input *)
191 ((uint8_t *)virt_addr_started + input_offset);
192 input->status = VIRTIO_CRYPTO_ERR;
193 input->session_id = ~0ULL;
194 desc[idx].addr = phys_addr_started + len_total;
195 desc[idx].len = len_session_input;
196 desc[idx].flags = VRING_DESC_F_WRITE;
199 /* use a single desc entry */
200 vq->vq_ring.desc[head].addr = phys_addr_started + desc_offset;
201 vq->vq_ring.desc[head].len = idx * sizeof(struct vring_desc);
202 vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT;
205 vq->vq_desc_head_idx = vq->vq_ring.desc[head].next;
207 vq_update_avail_ring(vq, head);
208 vq_update_avail_idx(vq);
210 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d",
213 virtqueue_notify(vq);
216 while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) {
221 while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) {
222 uint32_t idx, desc_idx, used_idx;
223 struct vring_used_elem *uep;
225 used_idx = (uint32_t)(vq->vq_used_cons_idx
226 & (vq->vq_nentries - 1));
227 uep = &vq->vq_ring.used->ring[used_idx];
228 idx = (uint32_t) uep->id;
231 while (vq->vq_ring.desc[desc_idx].flags & VRING_DESC_F_NEXT) {
232 desc_idx = vq->vq_ring.desc[desc_idx].next;
236 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx;
237 vq->vq_desc_head_idx = idx;
239 vq->vq_used_cons_idx++;
243 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n"
244 "vq->vq_desc_head_idx=%d",
245 vq->vq_free_cnt, vq->vq_desc_head_idx);
248 if (input->status != VIRTIO_CRYPTO_OK) {
249 VIRTIO_CRYPTO_SESSION_LOG_ERR("Something wrong on backend! "
250 "status=%u, session_id=%" PRIu64 "",
251 input->status, input->session_id);
252 rte_free(virt_addr_started);
255 session->session_id = input->session_id;
257 VIRTIO_CRYPTO_SESSION_LOG_INFO("Create session successfully, "
258 "session_id=%" PRIu64 "", input->session_id);
259 rte_free(virt_addr_started);
267 virtio_crypto_queue_release(struct virtqueue *vq)
269 struct virtio_crypto_hw *hw;
271 PMD_INIT_FUNC_TRACE();
275 /* Select and deactivate the queue */
276 VTPCI_OPS(hw)->del_queue(hw, vq);
278 rte_memzone_free(vq->mz);
279 rte_mempool_free(vq->mpool);
284 #define MPOOL_MAX_NAME_SZ 32
287 virtio_crypto_queue_setup(struct rte_cryptodev *dev,
289 uint16_t vtpci_queue_idx,
292 struct virtqueue **pvq)
294 char vq_name[VIRTQUEUE_MAX_NAME_SZ];
295 char mpool_name[MPOOL_MAX_NAME_SZ];
296 const struct rte_memzone *mz;
297 unsigned int vq_size, size;
298 struct virtio_crypto_hw *hw = dev->data->dev_private;
299 struct virtqueue *vq = NULL;
303 PMD_INIT_FUNC_TRACE();
305 VIRTIO_CRYPTO_INIT_LOG_DBG("setting up queue: %u", vtpci_queue_idx);
308 * Read the virtqueue size from the Queue Size field
309 * Always power of 2 and if 0 virtqueue does not exist
311 vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx);
313 VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue does not exist");
316 VIRTIO_CRYPTO_INIT_LOG_DBG("vq_size: %u", vq_size);
318 if (!rte_is_power_of_2(vq_size)) {
319 VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue size is not powerof 2");
323 if (queue_type == VTCRYPTO_DATAQ) {
324 snprintf(vq_name, sizeof(vq_name), "dev%d_dataqueue%d",
325 dev->data->dev_id, vtpci_queue_idx);
326 snprintf(mpool_name, sizeof(mpool_name),
327 "dev%d_dataqueue%d_mpool",
328 dev->data->dev_id, vtpci_queue_idx);
329 } else if (queue_type == VTCRYPTO_CTRLQ) {
330 snprintf(vq_name, sizeof(vq_name), "dev%d_controlqueue",
332 snprintf(mpool_name, sizeof(mpool_name),
333 "dev%d_controlqueue_mpool",
336 size = RTE_ALIGN_CEIL(sizeof(*vq) +
337 vq_size * sizeof(struct vq_desc_extra),
338 RTE_CACHE_LINE_SIZE);
339 vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE,
342 VIRTIO_CRYPTO_INIT_LOG_ERR("Can not allocate virtqueue");
346 if (queue_type == VTCRYPTO_DATAQ) {
347 /* pre-allocate a mempool and use it in the data plane to
348 * improve performance
350 vq->mpool = rte_mempool_lookup(mpool_name);
351 if (vq->mpool == NULL)
352 vq->mpool = rte_mempool_create(mpool_name,
354 sizeof(struct virtio_crypto_op_cookie),
355 RTE_CACHE_LINE_SIZE, 0,
356 NULL, NULL, NULL, NULL, socket_id,
359 VIRTIO_CRYPTO_DRV_LOG_ERR("Virtio Crypto PMD "
360 "Cannot create mempool");
361 goto mpool_create_err;
363 for (i = 0; i < vq_size; i++) {
364 vq->vq_descx[i].cookie =
365 rte_zmalloc("crypto PMD op cookie pointer",
366 sizeof(struct virtio_crypto_op_cookie),
367 RTE_CACHE_LINE_SIZE);
368 if (vq->vq_descx[i].cookie == NULL) {
369 VIRTIO_CRYPTO_DRV_LOG_ERR("Failed to "
370 "alloc mem for cookie");
371 goto cookie_alloc_err;
377 vq->dev_id = dev->data->dev_id;
378 vq->vq_queue_index = vtpci_queue_idx;
379 vq->vq_nentries = vq_size;
382 * Using part of the vring entries is permitted, but the maximum
385 if (nb_desc == 0 || nb_desc > vq_size)
387 vq->vq_free_cnt = nb_desc;
390 * Reserve a memzone for vring elements
392 size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN);
393 vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);
394 VIRTIO_CRYPTO_INIT_LOG_DBG("%s vring_size: %d, rounded_vring_size: %d",
395 (queue_type == VTCRYPTO_DATAQ) ? "dataq" : "ctrlq",
396 size, vq->vq_ring_size);
398 mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
399 socket_id, 0, VIRTIO_PCI_VRING_ALIGN);
401 if (rte_errno == EEXIST)
402 mz = rte_memzone_lookup(vq_name);
404 VIRTIO_CRYPTO_INIT_LOG_ERR("not enough memory");
410 * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit,
411 * and only accepts 32 bit page frame number.
412 * Check if the allocated physical memory exceeds 16TB.
414 if ((mz->iova + vq->vq_ring_size - 1)
415 >> (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) {
416 VIRTIO_CRYPTO_INIT_LOG_ERR("vring address shouldn't be "
421 memset(mz->addr, 0, sizeof(mz->len));
423 vq->vq_ring_mem = mz->iova;
424 vq->vq_ring_virt_mem = mz->addr;
425 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_mem(physical): 0x%"PRIx64,
427 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_virt_mem: 0x%"PRIx64,
428 (uint64_t)(uintptr_t)mz->addr);
435 rte_memzone_free(mz);
438 rte_mempool_free(vq->mpool);
440 for (j = 0; j < i; j++)
441 rte_free(vq->vq_descx[j].cookie);
449 virtio_crypto_ctrlq_setup(struct rte_cryptodev *dev, uint16_t queue_idx)
452 struct virtqueue *vq;
453 struct virtio_crypto_hw *hw = dev->data->dev_private;
455 /* if virtio device has started, do not touch the virtqueues */
456 if (dev->data->dev_started)
459 PMD_INIT_FUNC_TRACE();
461 ret = virtio_crypto_queue_setup(dev, VTCRYPTO_CTRLQ, queue_idx,
462 0, SOCKET_ID_ANY, &vq);
464 VIRTIO_CRYPTO_INIT_LOG_ERR("control vq initialization failed");
474 virtio_crypto_free_queues(struct rte_cryptodev *dev)
477 struct virtio_crypto_hw *hw = dev->data->dev_private;
479 PMD_INIT_FUNC_TRACE();
481 /* control queue release */
482 virtio_crypto_queue_release(hw->cvq);
484 /* data queue release */
485 for (i = 0; i < hw->max_dataqueues; i++)
486 virtio_crypto_queue_release(dev->data->queue_pairs[i]);
490 virtio_crypto_dev_close(struct rte_cryptodev *dev __rte_unused)
496 * dev_ops for virtio, bare necessities for basic operation
498 static struct rte_cryptodev_ops virtio_crypto_dev_ops = {
499 /* Device related operations */
500 .dev_configure = virtio_crypto_dev_configure,
501 .dev_start = virtio_crypto_dev_start,
502 .dev_stop = virtio_crypto_dev_stop,
503 .dev_close = virtio_crypto_dev_close,
504 .dev_infos_get = virtio_crypto_dev_info_get,
506 .stats_get = virtio_crypto_dev_stats_get,
507 .stats_reset = virtio_crypto_dev_stats_reset,
509 .queue_pair_setup = virtio_crypto_qp_setup,
510 .queue_pair_release = virtio_crypto_qp_release,
512 /* Crypto related operations */
513 .sym_session_get_size = virtio_crypto_sym_get_session_private_size,
514 .sym_session_configure = virtio_crypto_sym_configure_session,
515 .sym_session_clear = virtio_crypto_sym_clear_session
519 virtio_crypto_update_stats(struct rte_cryptodev *dev,
520 struct rte_cryptodev_stats *stats)
523 struct virtio_crypto_hw *hw = dev->data->dev_private;
525 PMD_INIT_FUNC_TRACE();
528 VIRTIO_CRYPTO_DRV_LOG_ERR("invalid pointer");
532 for (i = 0; i < hw->max_dataqueues; i++) {
533 const struct virtqueue *data_queue
534 = dev->data->queue_pairs[i];
535 if (data_queue == NULL)
538 stats->enqueued_count += data_queue->packets_sent_total;
539 stats->enqueue_err_count += data_queue->packets_sent_failed;
541 stats->dequeued_count += data_queue->packets_received_total;
542 stats->dequeue_err_count
543 += data_queue->packets_received_failed;
548 virtio_crypto_dev_stats_get(struct rte_cryptodev *dev,
549 struct rte_cryptodev_stats *stats)
551 PMD_INIT_FUNC_TRACE();
553 virtio_crypto_update_stats(dev, stats);
557 virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev)
560 struct virtio_crypto_hw *hw = dev->data->dev_private;
562 PMD_INIT_FUNC_TRACE();
564 for (i = 0; i < hw->max_dataqueues; i++) {
565 struct virtqueue *data_queue = dev->data->queue_pairs[i];
566 if (data_queue == NULL)
569 data_queue->packets_sent_total = 0;
570 data_queue->packets_sent_failed = 0;
572 data_queue->packets_received_total = 0;
573 data_queue->packets_received_failed = 0;
578 virtio_crypto_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id,
579 const struct rte_cryptodev_qp_conf *qp_conf,
583 struct virtqueue *vq;
585 PMD_INIT_FUNC_TRACE();
587 /* if virtio dev is started, do not touch the virtqueues */
588 if (dev->data->dev_started)
591 ret = virtio_crypto_queue_setup(dev, VTCRYPTO_DATAQ, queue_pair_id,
592 qp_conf->nb_descriptors, socket_id, &vq);
594 VIRTIO_CRYPTO_INIT_LOG_ERR(
595 "virtio crypto data queue initialization failed\n");
599 dev->data->queue_pairs[queue_pair_id] = vq;
605 virtio_crypto_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
608 = (struct virtqueue *)dev->data->queue_pairs[queue_pair_id];
610 PMD_INIT_FUNC_TRACE();
613 VIRTIO_CRYPTO_DRV_LOG_DBG("vq already freed");
617 virtio_crypto_queue_release(vq);
622 virtio_negotiate_features(struct virtio_crypto_hw *hw, uint64_t req_features)
624 uint64_t host_features;
626 PMD_INIT_FUNC_TRACE();
628 /* Prepare guest_features: feature that driver wants to support */
629 VIRTIO_CRYPTO_INIT_LOG_DBG("guest_features before negotiate = %" PRIx64,
632 /* Read device(host) feature bits */
633 host_features = VTPCI_OPS(hw)->get_features(hw);
634 VIRTIO_CRYPTO_INIT_LOG_DBG("host_features before negotiate = %" PRIx64,
638 * Negotiate features: Subset of device feature bits are written back
639 * guest feature bits.
641 hw->guest_features = req_features;
642 hw->guest_features = vtpci_cryptodev_negotiate_features(hw,
644 VIRTIO_CRYPTO_INIT_LOG_DBG("features after negotiate = %" PRIx64,
648 if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) {
649 VIRTIO_CRYPTO_INIT_LOG_ERR(
650 "VIRTIO_F_VERSION_1 features is not enabled.");
653 vtpci_cryptodev_set_status(hw,
654 VIRTIO_CONFIG_STATUS_FEATURES_OK);
655 if (!(vtpci_cryptodev_get_status(hw) &
656 VIRTIO_CONFIG_STATUS_FEATURES_OK)) {
657 VIRTIO_CRYPTO_INIT_LOG_ERR("failed to set FEATURES_OK "
663 hw->req_guest_features = req_features;
668 /* reset device and renegotiate features if needed */
670 virtio_crypto_init_device(struct rte_cryptodev *cryptodev,
671 uint64_t req_features)
673 struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
674 struct virtio_crypto_config local_config;
675 struct virtio_crypto_config *config = &local_config;
677 PMD_INIT_FUNC_TRACE();
679 /* Reset the device although not necessary at startup */
680 vtpci_cryptodev_reset(hw);
682 /* Tell the host we've noticed this device. */
683 vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
685 /* Tell the host we've known how to drive the device. */
686 vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
687 if (virtio_negotiate_features(hw, req_features) < 0)
690 /* Get status of the device */
691 vtpci_read_cryptodev_config(hw,
692 offsetof(struct virtio_crypto_config, status),
693 &config->status, sizeof(config->status));
694 if (config->status != VIRTIO_CRYPTO_S_HW_READY) {
695 VIRTIO_CRYPTO_DRV_LOG_ERR("accelerator hardware is "
700 /* Get number of data queues */
701 vtpci_read_cryptodev_config(hw,
702 offsetof(struct virtio_crypto_config, max_dataqueues),
703 &config->max_dataqueues,
704 sizeof(config->max_dataqueues));
705 hw->max_dataqueues = config->max_dataqueues;
707 VIRTIO_CRYPTO_INIT_LOG_DBG("hw->max_dataqueues=%d",
714 * This function is based on probe() function
715 * It returns 0 on success.
718 crypto_virtio_create(const char *name, struct rte_pci_device *pci_dev,
719 struct rte_cryptodev_pmd_init_params *init_params)
721 struct rte_cryptodev *cryptodev;
722 struct virtio_crypto_hw *hw;
724 PMD_INIT_FUNC_TRACE();
726 cryptodev = rte_cryptodev_pmd_create(name, &pci_dev->device,
728 if (cryptodev == NULL)
731 cryptodev->driver_id = cryptodev_virtio_driver_id;
732 cryptodev->dev_ops = &virtio_crypto_dev_ops;
734 cryptodev->enqueue_burst = virtio_crypto_pkt_tx_burst;
735 cryptodev->dequeue_burst = virtio_crypto_pkt_rx_burst;
737 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
738 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
739 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
741 hw = cryptodev->data->dev_private;
742 hw->dev_id = cryptodev->data->dev_id;
743 hw->virtio_dev_capabilities = virtio_capabilities;
745 VIRTIO_CRYPTO_INIT_LOG_DBG("dev %d vendorID=0x%x deviceID=0x%x",
746 cryptodev->data->dev_id, pci_dev->id.vendor_id,
747 pci_dev->id.device_id);
749 /* pci device init */
750 if (vtpci_cryptodev_init(pci_dev, hw))
753 if (virtio_crypto_init_device(cryptodev,
754 VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0)
757 rte_cryptodev_pmd_probing_finish(cryptodev);
763 virtio_crypto_dev_uninit(struct rte_cryptodev *cryptodev)
765 struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
767 PMD_INIT_FUNC_TRACE();
769 if (rte_eal_process_type() == RTE_PROC_SECONDARY)
772 if (cryptodev->data->dev_started) {
773 virtio_crypto_dev_stop(cryptodev);
774 virtio_crypto_dev_close(cryptodev);
777 cryptodev->dev_ops = NULL;
778 cryptodev->enqueue_burst = NULL;
779 cryptodev->dequeue_burst = NULL;
781 /* release control queue */
782 virtio_crypto_queue_release(hw->cvq);
784 rte_free(cryptodev->data);
785 cryptodev->data = NULL;
787 VIRTIO_CRYPTO_DRV_LOG_INFO("dev_uninit completed");
793 virtio_crypto_dev_configure(struct rte_cryptodev *cryptodev,
794 struct rte_cryptodev_config *config __rte_unused)
796 struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
798 PMD_INIT_FUNC_TRACE();
800 if (virtio_crypto_init_device(cryptodev,
801 VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0)
804 /* setup control queue
805 * [0, 1, ... ,(config->max_dataqueues - 1)] are data queues
806 * config->max_dataqueues is the control queue
808 if (virtio_crypto_ctrlq_setup(cryptodev, hw->max_dataqueues) < 0) {
809 VIRTIO_CRYPTO_INIT_LOG_ERR("control queue setup error");
812 virtio_crypto_ctrlq_start(cryptodev);
818 virtio_crypto_dev_stop(struct rte_cryptodev *dev)
820 struct virtio_crypto_hw *hw = dev->data->dev_private;
822 PMD_INIT_FUNC_TRACE();
823 VIRTIO_CRYPTO_DRV_LOG_DBG("virtio_dev_stop");
825 vtpci_cryptodev_reset(hw);
827 virtio_crypto_dev_free_mbufs(dev);
828 virtio_crypto_free_queues(dev);
830 dev->data->dev_started = 0;
834 virtio_crypto_dev_start(struct rte_cryptodev *dev)
836 struct virtio_crypto_hw *hw = dev->data->dev_private;
838 if (dev->data->dev_started)
841 /* Do final configuration before queue engine starts */
842 virtio_crypto_dataq_start(dev);
843 vtpci_cryptodev_reinit_complete(hw);
845 dev->data->dev_started = 1;
851 virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev)
854 struct virtio_crypto_hw *hw = dev->data->dev_private;
856 for (i = 0; i < hw->max_dataqueues; i++) {
857 VIRTIO_CRYPTO_INIT_LOG_DBG("Before freeing dataq[%d] used "
858 "and unused buf", i);
859 VIRTQUEUE_DUMP((struct virtqueue *)
860 dev->data->queue_pairs[i]);
862 VIRTIO_CRYPTO_INIT_LOG_DBG("queue_pairs[%d]=%p",
863 i, dev->data->queue_pairs[i]);
865 virtqueue_detatch_unused(dev->data->queue_pairs[i]);
867 VIRTIO_CRYPTO_INIT_LOG_DBG("After freeing dataq[%d] used and "
870 (struct virtqueue *)dev->data->queue_pairs[i]);
875 virtio_crypto_sym_get_session_private_size(
876 struct rte_cryptodev *dev __rte_unused)
878 PMD_INIT_FUNC_TRACE();
880 return RTE_ALIGN_CEIL(sizeof(struct virtio_crypto_session), 16);
884 virtio_crypto_check_sym_session_paras(
885 struct rte_cryptodev *dev)
887 struct virtio_crypto_hw *hw;
889 PMD_INIT_FUNC_TRACE();
891 if (unlikely(dev == NULL)) {
892 VIRTIO_CRYPTO_SESSION_LOG_ERR("dev is NULL");
895 if (unlikely(dev->data == NULL)) {
896 VIRTIO_CRYPTO_SESSION_LOG_ERR("dev->data is NULL");
899 hw = dev->data->dev_private;
900 if (unlikely(hw == NULL)) {
901 VIRTIO_CRYPTO_SESSION_LOG_ERR("hw is NULL");
904 if (unlikely(hw->cvq == NULL)) {
905 VIRTIO_CRYPTO_SESSION_LOG_ERR("vq is NULL");
913 virtio_crypto_check_sym_clear_session_paras(
914 struct rte_cryptodev *dev,
915 struct rte_cryptodev_sym_session *sess)
917 PMD_INIT_FUNC_TRACE();
920 VIRTIO_CRYPTO_SESSION_LOG_ERR("sym_session is NULL");
924 return virtio_crypto_check_sym_session_paras(dev);
927 #define NUM_ENTRY_SYM_CLEAR_SESSION 2
930 virtio_crypto_sym_clear_session(
931 struct rte_cryptodev *dev,
932 struct rte_cryptodev_sym_session *sess)
934 struct virtio_crypto_hw *hw;
935 struct virtqueue *vq;
936 struct virtio_crypto_session *session;
937 struct virtio_crypto_op_ctrl_req *ctrl;
938 struct vring_desc *desc;
942 uint8_t *malloc_virt_addr;
943 uint64_t malloc_phys_addr;
944 uint8_t len_inhdr = sizeof(struct virtio_crypto_inhdr);
945 uint32_t len_op_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req);
946 uint32_t desc_offset = len_op_ctrl_req + len_inhdr;
948 PMD_INIT_FUNC_TRACE();
950 if (virtio_crypto_check_sym_clear_session_paras(dev, sess) < 0)
953 hw = dev->data->dev_private;
955 session = (struct virtio_crypto_session *)get_sym_session_private_data(
956 sess, cryptodev_virtio_driver_id);
957 if (session == NULL) {
958 VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid session parameter");
962 VIRTIO_CRYPTO_SESSION_LOG_INFO("vq->vq_desc_head_idx = %d, "
963 "vq = %p", vq->vq_desc_head_idx, vq);
965 if (vq->vq_free_cnt < needed) {
966 VIRTIO_CRYPTO_SESSION_LOG_ERR(
967 "vq->vq_free_cnt = %d is less than %d, "
968 "not enough", vq->vq_free_cnt, needed);
973 * malloc memory to store information of ctrl request op,
974 * returned status and desc vring
976 malloc_virt_addr = rte_malloc(NULL, len_op_ctrl_req + len_inhdr
977 + NUM_ENTRY_SYM_CLEAR_SESSION
978 * sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE);
979 if (malloc_virt_addr == NULL) {
980 VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap room");
983 malloc_phys_addr = rte_malloc_virt2iova(malloc_virt_addr);
985 /* assign ctrl request op part */
986 ctrl = (struct virtio_crypto_op_ctrl_req *)malloc_virt_addr;
987 ctrl->header.opcode = VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION;
988 /* default data virtqueue is 0 */
989 ctrl->header.queue_id = 0;
990 ctrl->u.destroy_session.session_id = session->session_id;
993 status = &(((struct virtio_crypto_inhdr *)
994 ((uint8_t *)malloc_virt_addr + len_op_ctrl_req))->status);
995 *status = VIRTIO_CRYPTO_ERR;
997 /* indirect desc vring part */
998 desc = (struct vring_desc *)((uint8_t *)malloc_virt_addr
1001 /* ctrl request part */
1002 desc[0].addr = malloc_phys_addr;
1003 desc[0].len = len_op_ctrl_req;
1004 desc[0].flags = VRING_DESC_F_NEXT;
1008 desc[1].addr = malloc_phys_addr + len_op_ctrl_req;
1009 desc[1].len = len_inhdr;
1010 desc[1].flags = VRING_DESC_F_WRITE;
1012 /* use only a single desc entry */
1013 head = vq->vq_desc_head_idx;
1014 vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT;
1015 vq->vq_ring.desc[head].addr = malloc_phys_addr + desc_offset;
1016 vq->vq_ring.desc[head].len
1017 = NUM_ENTRY_SYM_CLEAR_SESSION
1018 * sizeof(struct vring_desc);
1019 vq->vq_free_cnt -= needed;
1021 vq->vq_desc_head_idx = vq->vq_ring.desc[head].next;
1023 vq_update_avail_ring(vq, head);
1024 vq_update_avail_idx(vq);
1026 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d",
1027 vq->vq_queue_index);
1029 virtqueue_notify(vq);
1032 while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) {
1037 while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) {
1038 uint32_t idx, desc_idx, used_idx;
1039 struct vring_used_elem *uep;
1041 used_idx = (uint32_t)(vq->vq_used_cons_idx
1042 & (vq->vq_nentries - 1));
1043 uep = &vq->vq_ring.used->ring[used_idx];
1044 idx = (uint32_t) uep->id;
1046 while (vq->vq_ring.desc[desc_idx].flags
1047 & VRING_DESC_F_NEXT) {
1048 desc_idx = vq->vq_ring.desc[desc_idx].next;
1052 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx;
1053 vq->vq_desc_head_idx = idx;
1054 vq->vq_used_cons_idx++;
1058 if (*status != VIRTIO_CRYPTO_OK) {
1059 VIRTIO_CRYPTO_SESSION_LOG_ERR("Close session failed "
1060 "status=%"PRIu32", session_id=%"PRIu64"",
1061 *status, session->session_id);
1062 rte_free(malloc_virt_addr);
1066 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n"
1067 "vq->vq_desc_head_idx=%d",
1068 vq->vq_free_cnt, vq->vq_desc_head_idx);
1070 VIRTIO_CRYPTO_SESSION_LOG_INFO("Close session %"PRIu64" successfully ",
1071 session->session_id);
1073 memset(session, 0, sizeof(struct virtio_crypto_session));
1074 struct rte_mempool *sess_mp = rte_mempool_from_obj(session);
1075 set_sym_session_private_data(sess, cryptodev_virtio_driver_id, NULL);
1076 rte_mempool_put(sess_mp, session);
1077 rte_free(malloc_virt_addr);
1080 static struct rte_crypto_cipher_xform *
1081 virtio_crypto_get_cipher_xform(struct rte_crypto_sym_xform *xform)
1084 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
1085 return &xform->cipher;
1087 xform = xform->next;
1093 static struct rte_crypto_auth_xform *
1094 virtio_crypto_get_auth_xform(struct rte_crypto_sym_xform *xform)
1097 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
1098 return &xform->auth;
1100 xform = xform->next;
1106 /** Get xform chain order */
1108 virtio_crypto_get_chain_order(struct rte_crypto_sym_xform *xform)
1114 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1115 xform->next == NULL)
1116 return VIRTIO_CRYPTO_CMD_CIPHER;
1118 /* Authentication Only */
1119 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1120 xform->next == NULL)
1121 return VIRTIO_CRYPTO_CMD_AUTH;
1123 /* Authenticate then Cipher */
1124 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1125 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
1126 return VIRTIO_CRYPTO_CMD_HASH_CIPHER;
1128 /* Cipher then Authenticate */
1129 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1130 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
1131 return VIRTIO_CRYPTO_CMD_CIPHER_HASH;
1137 virtio_crypto_sym_pad_cipher_param(
1138 struct virtio_crypto_cipher_session_para *para,
1139 struct rte_crypto_cipher_xform *cipher_xform)
1141 switch (cipher_xform->algo) {
1142 case RTE_CRYPTO_CIPHER_AES_CBC:
1143 para->algo = VIRTIO_CRYPTO_CIPHER_AES_CBC;
1146 VIRTIO_CRYPTO_SESSION_LOG_ERR("Crypto: Unsupported "
1147 "Cipher alg %u", cipher_xform->algo);
1151 para->keylen = cipher_xform->key.length;
1152 switch (cipher_xform->op) {
1153 case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
1154 para->op = VIRTIO_CRYPTO_OP_ENCRYPT;
1156 case RTE_CRYPTO_CIPHER_OP_DECRYPT:
1157 para->op = VIRTIO_CRYPTO_OP_DECRYPT;
1160 VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported cipher operation "
1169 virtio_crypto_sym_pad_auth_param(
1170 struct virtio_crypto_op_ctrl_req *ctrl,
1171 struct rte_crypto_auth_xform *auth_xform)
1174 struct virtio_crypto_alg_chain_session_para *para =
1175 &(ctrl->u.sym_create_session.u.chain.para);
1177 switch (ctrl->u.sym_create_session.u.chain.para.hash_mode) {
1178 case VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN:
1179 algo = &(para->u.hash_param.algo);
1181 case VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH:
1182 algo = &(para->u.mac_param.algo);
1185 VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported hash mode %u "
1187 ctrl->u.sym_create_session.u.chain.para.hash_mode);
1191 switch (auth_xform->algo) {
1192 case RTE_CRYPTO_AUTH_SHA1_HMAC:
1193 *algo = VIRTIO_CRYPTO_MAC_HMAC_SHA1;
1196 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1197 "Crypto: Undefined Hash algo %u specified",
1206 virtio_crypto_sym_pad_op_ctrl_req(
1207 struct virtio_crypto_op_ctrl_req *ctrl,
1208 struct rte_crypto_sym_xform *xform, bool is_chainned,
1209 uint8_t *cipher_key_data, uint8_t *auth_key_data,
1210 struct virtio_crypto_session *session)
1213 struct rte_crypto_auth_xform *auth_xform = NULL;
1214 struct rte_crypto_cipher_xform *cipher_xform = NULL;
1216 /* Get cipher xform from crypto xform chain */
1217 cipher_xform = virtio_crypto_get_cipher_xform(xform);
1219 if (cipher_xform->key.length > VIRTIO_CRYPTO_MAX_KEY_SIZE) {
1220 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1221 "cipher key size cannot be longer than %u",
1222 VIRTIO_CRYPTO_MAX_KEY_SIZE);
1225 if (cipher_xform->iv.length > VIRTIO_CRYPTO_MAX_IV_SIZE) {
1226 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1227 "cipher IV size cannot be longer than %u",
1228 VIRTIO_CRYPTO_MAX_IV_SIZE);
1232 ret = virtio_crypto_sym_pad_cipher_param(
1233 &ctrl->u.sym_create_session.u.chain.para
1234 .cipher_param, cipher_xform);
1236 ret = virtio_crypto_sym_pad_cipher_param(
1237 &ctrl->u.sym_create_session.u.cipher.para,
1241 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1242 "pad cipher parameter failed");
1246 memcpy(cipher_key_data, cipher_xform->key.data,
1247 cipher_xform->key.length);
1249 session->iv.offset = cipher_xform->iv.offset;
1250 session->iv.length = cipher_xform->iv.length;
1253 /* Get auth xform from crypto xform chain */
1254 auth_xform = virtio_crypto_get_auth_xform(xform);
1256 /* FIXME: support VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */
1257 struct virtio_crypto_alg_chain_session_para *para =
1258 &(ctrl->u.sym_create_session.u.chain.para);
1259 if (auth_xform->key.length) {
1260 if (auth_xform->key.length >
1261 VIRTIO_CRYPTO_MAX_KEY_SIZE) {
1262 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1263 "auth key size cannot be longer than %u",
1264 VIRTIO_CRYPTO_MAX_KEY_SIZE);
1267 para->hash_mode = VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH;
1268 para->u.mac_param.auth_key_len =
1269 (uint32_t)auth_xform->key.length;
1270 para->u.mac_param.hash_result_len =
1271 auth_xform->digest_length;
1272 memcpy(auth_key_data, auth_xform->key.data,
1273 auth_xform->key.length);
1275 para->hash_mode = VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN;
1276 para->u.hash_param.hash_result_len =
1277 auth_xform->digest_length;
1280 ret = virtio_crypto_sym_pad_auth_param(ctrl, auth_xform);
1282 VIRTIO_CRYPTO_SESSION_LOG_ERR("pad auth parameter "
1292 virtio_crypto_check_sym_configure_session_paras(
1293 struct rte_cryptodev *dev,
1294 struct rte_crypto_sym_xform *xform,
1295 struct rte_cryptodev_sym_session *sym_sess,
1296 struct rte_mempool *mempool)
1298 if (unlikely(xform == NULL) || unlikely(sym_sess == NULL) ||
1299 unlikely(mempool == NULL)) {
1300 VIRTIO_CRYPTO_SESSION_LOG_ERR("NULL pointer");
1304 if (virtio_crypto_check_sym_session_paras(dev) < 0)
1311 virtio_crypto_sym_configure_session(
1312 struct rte_cryptodev *dev,
1313 struct rte_crypto_sym_xform *xform,
1314 struct rte_cryptodev_sym_session *sess,
1315 struct rte_mempool *mempool)
1318 struct virtio_crypto_session crypto_sess;
1319 void *session_private = &crypto_sess;
1320 struct virtio_crypto_session *session;
1321 struct virtio_crypto_op_ctrl_req *ctrl_req;
1322 enum virtio_crypto_cmd_id cmd_id;
1323 uint8_t cipher_key_data[VIRTIO_CRYPTO_MAX_KEY_SIZE] = {0};
1324 uint8_t auth_key_data[VIRTIO_CRYPTO_MAX_KEY_SIZE] = {0};
1325 struct virtio_crypto_hw *hw;
1326 struct virtqueue *control_vq;
1328 PMD_INIT_FUNC_TRACE();
1330 ret = virtio_crypto_check_sym_configure_session_paras(dev, xform,
1333 VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid parameters");
1337 if (rte_mempool_get(mempool, &session_private)) {
1338 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1339 "Couldn't get object from session mempool");
1343 session = (struct virtio_crypto_session *)session_private;
1344 memset(session, 0, sizeof(struct virtio_crypto_session));
1345 ctrl_req = &session->ctrl;
1346 ctrl_req->header.opcode = VIRTIO_CRYPTO_CIPHER_CREATE_SESSION;
1347 /* FIXME: support multiqueue */
1348 ctrl_req->header.queue_id = 0;
1350 hw = dev->data->dev_private;
1351 control_vq = hw->cvq;
1353 cmd_id = virtio_crypto_get_chain_order(xform);
1354 if (cmd_id == VIRTIO_CRYPTO_CMD_CIPHER_HASH)
1355 ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order
1356 = VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
1357 if (cmd_id == VIRTIO_CRYPTO_CMD_HASH_CIPHER)
1358 ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order
1359 = VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
1362 case VIRTIO_CRYPTO_CMD_CIPHER_HASH:
1363 case VIRTIO_CRYPTO_CMD_HASH_CIPHER:
1364 ctrl_req->u.sym_create_session.op_type
1365 = VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING;
1367 ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req,
1368 xform, true, cipher_key_data, auth_key_data, session);
1370 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1371 "padding sym op ctrl req failed");
1374 ret = virtio_crypto_send_command(control_vq, ctrl_req,
1375 cipher_key_data, auth_key_data, session);
1377 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1378 "create session failed: %d", ret);
1382 case VIRTIO_CRYPTO_CMD_CIPHER:
1383 ctrl_req->u.sym_create_session.op_type
1384 = VIRTIO_CRYPTO_SYM_OP_CIPHER;
1385 ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req, xform,
1386 false, cipher_key_data, auth_key_data, session);
1388 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1389 "padding sym op ctrl req failed");
1392 ret = virtio_crypto_send_command(control_vq, ctrl_req,
1393 cipher_key_data, NULL, session);
1395 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1396 "create session failed: %d", ret);
1401 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1402 "Unsupported operation chain order parameter");
1406 set_sym_session_private_data(sess, dev->driver_id,
1416 virtio_crypto_dev_info_get(struct rte_cryptodev *dev,
1417 struct rte_cryptodev_info *info)
1419 struct virtio_crypto_hw *hw = dev->data->dev_private;
1421 PMD_INIT_FUNC_TRACE();
1424 info->driver_id = cryptodev_virtio_driver_id;
1425 info->feature_flags = dev->feature_flags;
1426 info->max_nb_queue_pairs = hw->max_dataqueues;
1427 /* No limit of number of sessions */
1428 info->sym.max_nb_sessions = 0;
1429 info->capabilities = hw->virtio_dev_capabilities;
1434 crypto_virtio_pci_probe(
1435 struct rte_pci_driver *pci_drv __rte_unused,
1436 struct rte_pci_device *pci_dev)
1438 struct rte_cryptodev_pmd_init_params init_params = {
1440 .socket_id = pci_dev->device.numa_node,
1441 .private_data_size = sizeof(struct virtio_crypto_hw)
1443 char name[RTE_CRYPTODEV_NAME_MAX_LEN];
1445 VIRTIO_CRYPTO_DRV_LOG_DBG("Found Crypto device at %02x:%02x.%x",
1447 pci_dev->addr.devid,
1448 pci_dev->addr.function);
1450 rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
1452 return crypto_virtio_create(name, pci_dev, &init_params);
1456 crypto_virtio_pci_remove(
1457 struct rte_pci_device *pci_dev __rte_unused)
1459 struct rte_cryptodev *cryptodev;
1460 char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
1462 if (pci_dev == NULL)
1465 rte_pci_device_name(&pci_dev->addr, cryptodev_name,
1466 sizeof(cryptodev_name));
1468 cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name);
1469 if (cryptodev == NULL)
1472 return virtio_crypto_dev_uninit(cryptodev);
1475 static struct rte_pci_driver rte_virtio_crypto_driver = {
1476 .id_table = pci_id_virtio_crypto_map,
1478 .probe = crypto_virtio_pci_probe,
1479 .remove = crypto_virtio_pci_remove
1482 static struct cryptodev_driver virtio_crypto_drv;
1484 RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_VIRTIO_PMD, rte_virtio_crypto_driver);
1485 RTE_PMD_REGISTER_CRYPTO_DRIVER(virtio_crypto_drv,
1486 rte_virtio_crypto_driver.driver,
1487 cryptodev_virtio_driver_id);
1488 RTE_LOG_REGISTER_SUFFIX(virtio_crypto_logtype_init, init, NOTICE);
1489 RTE_LOG_REGISTER_SUFFIX(virtio_crypto_logtype_session, session, NOTICE);
1490 RTE_LOG_REGISTER_SUFFIX(virtio_crypto_logtype_rx, rx, NOTICE);
1491 RTE_LOG_REGISTER_SUFFIX(virtio_crypto_logtype_tx, tx, NOTICE);
1492 RTE_LOG_REGISTER_SUFFIX(virtio_crypto_logtype_driver, driver, NOTICE);