1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
7 #include <rte_common.h>
10 #include <rte_bus_pci.h>
11 #include <rte_cryptodev.h>
12 #include <cryptodev_pmd.h>
15 #include "virtio_cryptodev.h"
16 #include "virtqueue.h"
17 #include "virtio_crypto_algs.h"
18 #include "virtio_crypto_capabilities.h"
20 static int virtio_crypto_dev_configure(struct rte_cryptodev *dev,
21 struct rte_cryptodev_config *config);
22 static int virtio_crypto_dev_start(struct rte_cryptodev *dev);
23 static void virtio_crypto_dev_stop(struct rte_cryptodev *dev);
24 static int virtio_crypto_dev_close(struct rte_cryptodev *dev);
25 static void virtio_crypto_dev_info_get(struct rte_cryptodev *dev,
26 struct rte_cryptodev_info *dev_info);
27 static void virtio_crypto_dev_stats_get(struct rte_cryptodev *dev,
28 struct rte_cryptodev_stats *stats);
29 static void virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev);
30 static int virtio_crypto_qp_setup(struct rte_cryptodev *dev,
31 uint16_t queue_pair_id,
32 const struct rte_cryptodev_qp_conf *qp_conf,
34 static int virtio_crypto_qp_release(struct rte_cryptodev *dev,
35 uint16_t queue_pair_id);
36 static void virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev);
37 static unsigned int virtio_crypto_sym_get_session_private_size(
38 struct rte_cryptodev *dev);
39 static void virtio_crypto_sym_clear_session(struct rte_cryptodev *dev,
40 struct rte_cryptodev_sym_session *sess);
41 static int virtio_crypto_sym_configure_session(struct rte_cryptodev *dev,
42 struct rte_crypto_sym_xform *xform,
43 struct rte_cryptodev_sym_session *session,
44 struct rte_mempool *mp);
47 * The set of PCI devices this driver supports
49 static const struct rte_pci_id pci_id_virtio_crypto_map[] = {
50 { RTE_PCI_DEVICE(VIRTIO_CRYPTO_PCI_VENDORID,
51 VIRTIO_CRYPTO_PCI_DEVICEID) },
52 { .vendor_id = 0, /* sentinel */ },
55 static const struct rte_cryptodev_capabilities virtio_capabilities[] = {
56 VIRTIO_SYM_CAPABILITIES,
57 RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
60 uint8_t cryptodev_virtio_driver_id;
62 #define NUM_ENTRY_SYM_CREATE_SESSION 4
65 virtio_crypto_send_command(struct virtqueue *vq,
66 struct virtio_crypto_op_ctrl_req *ctrl, uint8_t *cipher_key,
67 uint8_t *auth_key, struct virtio_crypto_session *session)
72 uint32_t len_cipher_key = 0;
73 uint32_t len_auth_key = 0;
74 uint32_t len_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req);
75 uint32_t len_session_input = sizeof(struct virtio_crypto_session_input);
76 uint32_t len_total = 0;
77 uint32_t input_offset = 0;
78 void *virt_addr_started = NULL;
79 phys_addr_t phys_addr_started;
80 struct vring_desc *desc;
82 struct virtio_crypto_session_input *input;
85 PMD_INIT_FUNC_TRACE();
87 if (session == NULL) {
88 VIRTIO_CRYPTO_SESSION_LOG_ERR("session is NULL.");
91 /* cipher only is supported, it is available if auth_key is NULL */
93 VIRTIO_CRYPTO_SESSION_LOG_ERR("cipher key is NULL.");
97 head = vq->vq_desc_head_idx;
98 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_desc_head_idx = %d, vq = %p",
101 if (vq->vq_free_cnt < needed) {
102 VIRTIO_CRYPTO_SESSION_LOG_ERR("Not enough entry");
106 /* calculate the length of cipher key */
108 switch (ctrl->u.sym_create_session.op_type) {
109 case VIRTIO_CRYPTO_SYM_OP_CIPHER:
111 = ctrl->u.sym_create_session.u.cipher
114 case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING:
116 = ctrl->u.sym_create_session.u.chain
117 .para.cipher_param.keylen;
120 VIRTIO_CRYPTO_SESSION_LOG_ERR("invalid op type");
125 /* calculate the length of auth key */
128 ctrl->u.sym_create_session.u.chain.para.u.mac_param
133 * malloc memory to store indirect vring_desc entries, including
134 * ctrl request, cipher key, auth key, session input and desc vring
136 desc_offset = len_ctrl_req + len_cipher_key + len_auth_key
138 virt_addr_started = rte_malloc(NULL,
139 desc_offset + NUM_ENTRY_SYM_CREATE_SESSION
140 * sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE);
141 if (virt_addr_started == NULL) {
142 VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap memory");
145 phys_addr_started = rte_malloc_virt2iova(virt_addr_started);
147 /* address to store indirect vring desc entries */
148 desc = (struct vring_desc *)
149 ((uint8_t *)virt_addr_started + desc_offset);
152 memcpy(virt_addr_started, ctrl, len_ctrl_req);
153 desc[idx].addr = phys_addr_started;
154 desc[idx].len = len_ctrl_req;
155 desc[idx].flags = VRING_DESC_F_NEXT;
156 desc[idx].next = idx + 1;
158 len_total += len_ctrl_req;
159 input_offset += len_ctrl_req;
161 /* cipher key part */
162 if (len_cipher_key > 0) {
163 memcpy((uint8_t *)virt_addr_started + len_total,
164 cipher_key, len_cipher_key);
166 desc[idx].addr = phys_addr_started + len_total;
167 desc[idx].len = len_cipher_key;
168 desc[idx].flags = VRING_DESC_F_NEXT;
169 desc[idx].next = idx + 1;
171 len_total += len_cipher_key;
172 input_offset += len_cipher_key;
176 if (len_auth_key > 0) {
177 memcpy((uint8_t *)virt_addr_started + len_total,
178 auth_key, len_auth_key);
180 desc[idx].addr = phys_addr_started + len_total;
181 desc[idx].len = len_auth_key;
182 desc[idx].flags = VRING_DESC_F_NEXT;
183 desc[idx].next = idx + 1;
185 len_total += len_auth_key;
186 input_offset += len_auth_key;
190 input = (struct virtio_crypto_session_input *)
191 ((uint8_t *)virt_addr_started + input_offset);
192 input->status = VIRTIO_CRYPTO_ERR;
193 input->session_id = ~0ULL;
194 desc[idx].addr = phys_addr_started + len_total;
195 desc[idx].len = len_session_input;
196 desc[idx].flags = VRING_DESC_F_WRITE;
199 /* use a single desc entry */
200 vq->vq_ring.desc[head].addr = phys_addr_started + desc_offset;
201 vq->vq_ring.desc[head].len = idx * sizeof(struct vring_desc);
202 vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT;
205 vq->vq_desc_head_idx = vq->vq_ring.desc[head].next;
207 vq_update_avail_ring(vq, head);
208 vq_update_avail_idx(vq);
210 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d",
213 virtqueue_notify(vq);
216 while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) {
221 while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) {
222 uint32_t idx, desc_idx, used_idx;
223 struct vring_used_elem *uep;
225 used_idx = (uint32_t)(vq->vq_used_cons_idx
226 & (vq->vq_nentries - 1));
227 uep = &vq->vq_ring.used->ring[used_idx];
228 idx = (uint32_t) uep->id;
231 while (vq->vq_ring.desc[desc_idx].flags & VRING_DESC_F_NEXT) {
232 desc_idx = vq->vq_ring.desc[desc_idx].next;
236 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx;
237 vq->vq_desc_head_idx = idx;
239 vq->vq_used_cons_idx++;
243 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n"
244 "vq->vq_desc_head_idx=%d",
245 vq->vq_free_cnt, vq->vq_desc_head_idx);
248 if (input->status != VIRTIO_CRYPTO_OK) {
249 VIRTIO_CRYPTO_SESSION_LOG_ERR("Something wrong on backend! "
250 "status=%u, session_id=%" PRIu64 "",
251 input->status, input->session_id);
252 rte_free(virt_addr_started);
255 session->session_id = input->session_id;
257 VIRTIO_CRYPTO_SESSION_LOG_INFO("Create session successfully, "
258 "session_id=%" PRIu64 "", input->session_id);
259 rte_free(virt_addr_started);
267 virtio_crypto_queue_release(struct virtqueue *vq)
269 struct virtio_crypto_hw *hw;
271 PMD_INIT_FUNC_TRACE();
275 /* Select and deactivate the queue */
276 VTPCI_OPS(hw)->del_queue(hw, vq);
278 rte_memzone_free(vq->mz);
279 rte_mempool_free(vq->mpool);
284 #define MPOOL_MAX_NAME_SZ 32
287 virtio_crypto_queue_setup(struct rte_cryptodev *dev,
289 uint16_t vtpci_queue_idx,
292 struct virtqueue **pvq)
294 char vq_name[VIRTQUEUE_MAX_NAME_SZ];
295 char mpool_name[MPOOL_MAX_NAME_SZ];
296 const struct rte_memzone *mz;
297 unsigned int vq_size, size;
298 struct virtio_crypto_hw *hw = dev->data->dev_private;
299 struct virtqueue *vq = NULL;
303 PMD_INIT_FUNC_TRACE();
305 VIRTIO_CRYPTO_INIT_LOG_DBG("setting up queue: %u", vtpci_queue_idx);
308 * Read the virtqueue size from the Queue Size field
309 * Always power of 2 and if 0 virtqueue does not exist
311 vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx);
313 VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue does not exist");
316 VIRTIO_CRYPTO_INIT_LOG_DBG("vq_size: %u", vq_size);
318 if (!rte_is_power_of_2(vq_size)) {
319 VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue size is not powerof 2");
323 if (queue_type == VTCRYPTO_DATAQ) {
324 snprintf(vq_name, sizeof(vq_name), "dev%d_dataqueue%d",
325 dev->data->dev_id, vtpci_queue_idx);
326 snprintf(mpool_name, sizeof(mpool_name),
327 "dev%d_dataqueue%d_mpool",
328 dev->data->dev_id, vtpci_queue_idx);
329 } else if (queue_type == VTCRYPTO_CTRLQ) {
330 snprintf(vq_name, sizeof(vq_name), "dev%d_controlqueue",
332 snprintf(mpool_name, sizeof(mpool_name),
333 "dev%d_controlqueue_mpool",
336 size = RTE_ALIGN_CEIL(sizeof(*vq) +
337 vq_size * sizeof(struct vq_desc_extra),
338 RTE_CACHE_LINE_SIZE);
339 vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE,
342 VIRTIO_CRYPTO_INIT_LOG_ERR("Can not allocate virtqueue");
346 if (queue_type == VTCRYPTO_DATAQ) {
347 /* pre-allocate a mempool and use it in the data plane to
348 * improve performance
350 vq->mpool = rte_mempool_lookup(mpool_name);
351 if (vq->mpool == NULL)
352 vq->mpool = rte_mempool_create(mpool_name,
354 sizeof(struct virtio_crypto_op_cookie),
355 RTE_CACHE_LINE_SIZE, 0,
356 NULL, NULL, NULL, NULL, socket_id,
359 VIRTIO_CRYPTO_DRV_LOG_ERR("Virtio Crypto PMD "
360 "Cannot create mempool");
361 goto mpool_create_err;
363 for (i = 0; i < vq_size; i++) {
364 vq->vq_descx[i].cookie =
365 rte_zmalloc("crypto PMD op cookie pointer",
366 sizeof(struct virtio_crypto_op_cookie),
367 RTE_CACHE_LINE_SIZE);
368 if (vq->vq_descx[i].cookie == NULL) {
369 VIRTIO_CRYPTO_DRV_LOG_ERR("Failed to "
370 "alloc mem for cookie");
371 goto cookie_alloc_err;
377 vq->dev_id = dev->data->dev_id;
378 vq->vq_queue_index = vtpci_queue_idx;
379 vq->vq_nentries = vq_size;
382 * Using part of the vring entries is permitted, but the maximum
385 if (nb_desc == 0 || nb_desc > vq_size)
387 vq->vq_free_cnt = nb_desc;
390 * Reserve a memzone for vring elements
392 size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN);
393 vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);
394 VIRTIO_CRYPTO_INIT_LOG_DBG("%s vring_size: %d, rounded_vring_size: %d",
395 (queue_type == VTCRYPTO_DATAQ) ? "dataq" : "ctrlq",
396 size, vq->vq_ring_size);
398 mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
399 socket_id, 0, VIRTIO_PCI_VRING_ALIGN);
401 if (rte_errno == EEXIST)
402 mz = rte_memzone_lookup(vq_name);
404 VIRTIO_CRYPTO_INIT_LOG_ERR("not enough memory");
410 * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit,
411 * and only accepts 32 bit page frame number.
412 * Check if the allocated physical memory exceeds 16TB.
414 if ((mz->iova + vq->vq_ring_size - 1)
415 >> (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) {
416 VIRTIO_CRYPTO_INIT_LOG_ERR("vring address shouldn't be "
421 memset(mz->addr, 0, sizeof(mz->len));
423 vq->vq_ring_mem = mz->iova;
424 vq->vq_ring_virt_mem = mz->addr;
425 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_mem(physical): 0x%"PRIx64,
427 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_virt_mem: 0x%"PRIx64,
428 (uint64_t)(uintptr_t)mz->addr);
435 rte_memzone_free(mz);
438 rte_mempool_free(vq->mpool);
440 for (j = 0; j < i; j++)
441 rte_free(vq->vq_descx[j].cookie);
449 virtio_crypto_ctrlq_setup(struct rte_cryptodev *dev, uint16_t queue_idx)
452 struct virtqueue *vq;
453 struct virtio_crypto_hw *hw = dev->data->dev_private;
455 /* if virtio device has started, do not touch the virtqueues */
456 if (dev->data->dev_started)
459 PMD_INIT_FUNC_TRACE();
461 ret = virtio_crypto_queue_setup(dev, VTCRYPTO_CTRLQ, queue_idx,
462 0, SOCKET_ID_ANY, &vq);
464 VIRTIO_CRYPTO_INIT_LOG_ERR("control vq initialization failed");
474 virtio_crypto_free_queues(struct rte_cryptodev *dev)
477 struct virtio_crypto_hw *hw = dev->data->dev_private;
479 PMD_INIT_FUNC_TRACE();
481 /* control queue release */
482 virtio_crypto_queue_release(hw->cvq);
484 /* data queue release */
485 for (i = 0; i < hw->max_dataqueues; i++)
486 virtio_crypto_queue_release(dev->data->queue_pairs[i]);
490 virtio_crypto_dev_close(struct rte_cryptodev *dev __rte_unused)
496 * dev_ops for virtio, bare necessities for basic operation
498 static struct rte_cryptodev_ops virtio_crypto_dev_ops = {
499 /* Device related operations */
500 .dev_configure = virtio_crypto_dev_configure,
501 .dev_start = virtio_crypto_dev_start,
502 .dev_stop = virtio_crypto_dev_stop,
503 .dev_close = virtio_crypto_dev_close,
504 .dev_infos_get = virtio_crypto_dev_info_get,
506 .stats_get = virtio_crypto_dev_stats_get,
507 .stats_reset = virtio_crypto_dev_stats_reset,
509 .queue_pair_setup = virtio_crypto_qp_setup,
510 .queue_pair_release = virtio_crypto_qp_release,
512 /* Crypto related operations */
513 .sym_session_get_size = virtio_crypto_sym_get_session_private_size,
514 .sym_session_configure = virtio_crypto_sym_configure_session,
515 .sym_session_clear = virtio_crypto_sym_clear_session
519 virtio_crypto_update_stats(struct rte_cryptodev *dev,
520 struct rte_cryptodev_stats *stats)
523 struct virtio_crypto_hw *hw = dev->data->dev_private;
525 PMD_INIT_FUNC_TRACE();
528 VIRTIO_CRYPTO_DRV_LOG_ERR("invalid pointer");
532 for (i = 0; i < hw->max_dataqueues; i++) {
533 const struct virtqueue *data_queue
534 = dev->data->queue_pairs[i];
535 if (data_queue == NULL)
538 stats->enqueued_count += data_queue->packets_sent_total;
539 stats->enqueue_err_count += data_queue->packets_sent_failed;
541 stats->dequeued_count += data_queue->packets_received_total;
542 stats->dequeue_err_count
543 += data_queue->packets_received_failed;
548 virtio_crypto_dev_stats_get(struct rte_cryptodev *dev,
549 struct rte_cryptodev_stats *stats)
551 PMD_INIT_FUNC_TRACE();
553 virtio_crypto_update_stats(dev, stats);
557 virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev)
560 struct virtio_crypto_hw *hw = dev->data->dev_private;
562 PMD_INIT_FUNC_TRACE();
564 for (i = 0; i < hw->max_dataqueues; i++) {
565 struct virtqueue *data_queue = dev->data->queue_pairs[i];
566 if (data_queue == NULL)
569 data_queue->packets_sent_total = 0;
570 data_queue->packets_sent_failed = 0;
572 data_queue->packets_received_total = 0;
573 data_queue->packets_received_failed = 0;
578 virtio_crypto_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id,
579 const struct rte_cryptodev_qp_conf *qp_conf,
583 struct virtqueue *vq;
585 PMD_INIT_FUNC_TRACE();
587 /* if virtio dev is started, do not touch the virtqueues */
588 if (dev->data->dev_started)
591 ret = virtio_crypto_queue_setup(dev, VTCRYPTO_DATAQ, queue_pair_id,
592 qp_conf->nb_descriptors, socket_id, &vq);
594 VIRTIO_CRYPTO_INIT_LOG_ERR(
595 "virtio crypto data queue initialization failed\n");
599 dev->data->queue_pairs[queue_pair_id] = vq;
605 virtio_crypto_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
608 = (struct virtqueue *)dev->data->queue_pairs[queue_pair_id];
610 PMD_INIT_FUNC_TRACE();
613 VIRTIO_CRYPTO_DRV_LOG_DBG("vq already freed");
617 virtio_crypto_queue_release(vq);
622 virtio_negotiate_features(struct virtio_crypto_hw *hw, uint64_t req_features)
624 uint64_t host_features;
626 PMD_INIT_FUNC_TRACE();
628 /* Prepare guest_features: feature that driver wants to support */
629 VIRTIO_CRYPTO_INIT_LOG_DBG("guest_features before negotiate = %" PRIx64,
632 /* Read device(host) feature bits */
633 host_features = VTPCI_OPS(hw)->get_features(hw);
634 VIRTIO_CRYPTO_INIT_LOG_DBG("host_features before negotiate = %" PRIx64,
638 * Negotiate features: Subset of device feature bits are written back
639 * guest feature bits.
641 hw->guest_features = req_features;
642 hw->guest_features = vtpci_cryptodev_negotiate_features(hw,
644 VIRTIO_CRYPTO_INIT_LOG_DBG("features after negotiate = %" PRIx64,
648 if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) {
649 VIRTIO_CRYPTO_INIT_LOG_ERR(
650 "VIRTIO_F_VERSION_1 features is not enabled.");
653 vtpci_cryptodev_set_status(hw,
654 VIRTIO_CONFIG_STATUS_FEATURES_OK);
655 if (!(vtpci_cryptodev_get_status(hw) &
656 VIRTIO_CONFIG_STATUS_FEATURES_OK)) {
657 VIRTIO_CRYPTO_INIT_LOG_ERR("failed to set FEATURES_OK "
663 hw->req_guest_features = req_features;
668 /* reset device and renegotiate features if needed */
670 virtio_crypto_init_device(struct rte_cryptodev *cryptodev,
671 uint64_t req_features)
673 struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
674 struct virtio_crypto_config local_config;
675 struct virtio_crypto_config *config = &local_config;
677 PMD_INIT_FUNC_TRACE();
679 /* Reset the device although not necessary at startup */
680 vtpci_cryptodev_reset(hw);
682 /* Tell the host we've noticed this device. */
683 vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
685 /* Tell the host we've known how to drive the device. */
686 vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
687 if (virtio_negotiate_features(hw, req_features) < 0)
690 /* Get status of the device */
691 vtpci_read_cryptodev_config(hw,
692 offsetof(struct virtio_crypto_config, status),
693 &config->status, sizeof(config->status));
694 if (config->status != VIRTIO_CRYPTO_S_HW_READY) {
695 VIRTIO_CRYPTO_DRV_LOG_ERR("accelerator hardware is "
700 /* Get number of data queues */
701 vtpci_read_cryptodev_config(hw,
702 offsetof(struct virtio_crypto_config, max_dataqueues),
703 &config->max_dataqueues,
704 sizeof(config->max_dataqueues));
705 hw->max_dataqueues = config->max_dataqueues;
707 VIRTIO_CRYPTO_INIT_LOG_DBG("hw->max_dataqueues=%d",
714 * This function is based on probe() function
715 * It returns 0 on success.
718 crypto_virtio_create(const char *name, struct rte_pci_device *pci_dev,
719 struct rte_cryptodev_pmd_init_params *init_params)
721 struct rte_cryptodev *cryptodev;
722 struct virtio_crypto_hw *hw;
724 PMD_INIT_FUNC_TRACE();
726 cryptodev = rte_cryptodev_pmd_create(name, &pci_dev->device,
728 if (cryptodev == NULL)
731 cryptodev->driver_id = cryptodev_virtio_driver_id;
732 cryptodev->dev_ops = &virtio_crypto_dev_ops;
734 cryptodev->enqueue_burst = virtio_crypto_pkt_tx_burst;
735 cryptodev->dequeue_burst = virtio_crypto_pkt_rx_burst;
737 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
738 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
739 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
741 hw = cryptodev->data->dev_private;
742 hw->dev_id = cryptodev->data->dev_id;
743 hw->virtio_dev_capabilities = virtio_capabilities;
745 VIRTIO_CRYPTO_INIT_LOG_DBG("dev %d vendorID=0x%x deviceID=0x%x",
746 cryptodev->data->dev_id, pci_dev->id.vendor_id,
747 pci_dev->id.device_id);
749 /* pci device init */
750 if (vtpci_cryptodev_init(pci_dev, hw))
753 if (virtio_crypto_init_device(cryptodev,
754 VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0)
761 virtio_crypto_dev_uninit(struct rte_cryptodev *cryptodev)
763 struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
765 PMD_INIT_FUNC_TRACE();
767 if (rte_eal_process_type() == RTE_PROC_SECONDARY)
770 if (cryptodev->data->dev_started) {
771 virtio_crypto_dev_stop(cryptodev);
772 virtio_crypto_dev_close(cryptodev);
775 cryptodev->dev_ops = NULL;
776 cryptodev->enqueue_burst = NULL;
777 cryptodev->dequeue_burst = NULL;
779 /* release control queue */
780 virtio_crypto_queue_release(hw->cvq);
782 rte_free(cryptodev->data);
783 cryptodev->data = NULL;
785 VIRTIO_CRYPTO_DRV_LOG_INFO("dev_uninit completed");
791 virtio_crypto_dev_configure(struct rte_cryptodev *cryptodev,
792 struct rte_cryptodev_config *config __rte_unused)
794 struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
796 PMD_INIT_FUNC_TRACE();
798 if (virtio_crypto_init_device(cryptodev,
799 VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0)
802 /* setup control queue
803 * [0, 1, ... ,(config->max_dataqueues - 1)] are data queues
804 * config->max_dataqueues is the control queue
806 if (virtio_crypto_ctrlq_setup(cryptodev, hw->max_dataqueues) < 0) {
807 VIRTIO_CRYPTO_INIT_LOG_ERR("control queue setup error");
810 virtio_crypto_ctrlq_start(cryptodev);
816 virtio_crypto_dev_stop(struct rte_cryptodev *dev)
818 struct virtio_crypto_hw *hw = dev->data->dev_private;
820 PMD_INIT_FUNC_TRACE();
821 VIRTIO_CRYPTO_DRV_LOG_DBG("virtio_dev_stop");
823 vtpci_cryptodev_reset(hw);
825 virtio_crypto_dev_free_mbufs(dev);
826 virtio_crypto_free_queues(dev);
828 dev->data->dev_started = 0;
832 virtio_crypto_dev_start(struct rte_cryptodev *dev)
834 struct virtio_crypto_hw *hw = dev->data->dev_private;
836 if (dev->data->dev_started)
839 /* Do final configuration before queue engine starts */
840 virtio_crypto_dataq_start(dev);
841 vtpci_cryptodev_reinit_complete(hw);
843 dev->data->dev_started = 1;
849 virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev)
852 struct virtio_crypto_hw *hw = dev->data->dev_private;
854 for (i = 0; i < hw->max_dataqueues; i++) {
855 VIRTIO_CRYPTO_INIT_LOG_DBG("Before freeing dataq[%d] used "
856 "and unused buf", i);
857 VIRTQUEUE_DUMP((struct virtqueue *)
858 dev->data->queue_pairs[i]);
860 VIRTIO_CRYPTO_INIT_LOG_DBG("queue_pairs[%d]=%p",
861 i, dev->data->queue_pairs[i]);
863 virtqueue_detatch_unused(dev->data->queue_pairs[i]);
865 VIRTIO_CRYPTO_INIT_LOG_DBG("After freeing dataq[%d] used and "
868 (struct virtqueue *)dev->data->queue_pairs[i]);
873 virtio_crypto_sym_get_session_private_size(
874 struct rte_cryptodev *dev __rte_unused)
876 PMD_INIT_FUNC_TRACE();
878 return RTE_ALIGN_CEIL(sizeof(struct virtio_crypto_session), 16);
882 virtio_crypto_check_sym_session_paras(
883 struct rte_cryptodev *dev)
885 struct virtio_crypto_hw *hw;
887 PMD_INIT_FUNC_TRACE();
889 if (unlikely(dev == NULL)) {
890 VIRTIO_CRYPTO_SESSION_LOG_ERR("dev is NULL");
893 if (unlikely(dev->data == NULL)) {
894 VIRTIO_CRYPTO_SESSION_LOG_ERR("dev->data is NULL");
897 hw = dev->data->dev_private;
898 if (unlikely(hw == NULL)) {
899 VIRTIO_CRYPTO_SESSION_LOG_ERR("hw is NULL");
902 if (unlikely(hw->cvq == NULL)) {
903 VIRTIO_CRYPTO_SESSION_LOG_ERR("vq is NULL");
911 virtio_crypto_check_sym_clear_session_paras(
912 struct rte_cryptodev *dev,
913 struct rte_cryptodev_sym_session *sess)
915 PMD_INIT_FUNC_TRACE();
918 VIRTIO_CRYPTO_SESSION_LOG_ERR("sym_session is NULL");
922 return virtio_crypto_check_sym_session_paras(dev);
925 #define NUM_ENTRY_SYM_CLEAR_SESSION 2
928 virtio_crypto_sym_clear_session(
929 struct rte_cryptodev *dev,
930 struct rte_cryptodev_sym_session *sess)
932 struct virtio_crypto_hw *hw;
933 struct virtqueue *vq;
934 struct virtio_crypto_session *session;
935 struct virtio_crypto_op_ctrl_req *ctrl;
936 struct vring_desc *desc;
940 uint8_t *malloc_virt_addr;
941 uint64_t malloc_phys_addr;
942 uint8_t len_inhdr = sizeof(struct virtio_crypto_inhdr);
943 uint32_t len_op_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req);
944 uint32_t desc_offset = len_op_ctrl_req + len_inhdr;
946 PMD_INIT_FUNC_TRACE();
948 if (virtio_crypto_check_sym_clear_session_paras(dev, sess) < 0)
951 hw = dev->data->dev_private;
953 session = (struct virtio_crypto_session *)get_sym_session_private_data(
954 sess, cryptodev_virtio_driver_id);
955 if (session == NULL) {
956 VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid session parameter");
960 VIRTIO_CRYPTO_SESSION_LOG_INFO("vq->vq_desc_head_idx = %d, "
961 "vq = %p", vq->vq_desc_head_idx, vq);
963 if (vq->vq_free_cnt < needed) {
964 VIRTIO_CRYPTO_SESSION_LOG_ERR(
965 "vq->vq_free_cnt = %d is less than %d, "
966 "not enough", vq->vq_free_cnt, needed);
971 * malloc memory to store information of ctrl request op,
972 * returned status and desc vring
974 malloc_virt_addr = rte_malloc(NULL, len_op_ctrl_req + len_inhdr
975 + NUM_ENTRY_SYM_CLEAR_SESSION
976 * sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE);
977 if (malloc_virt_addr == NULL) {
978 VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap room");
981 malloc_phys_addr = rte_malloc_virt2iova(malloc_virt_addr);
983 /* assign ctrl request op part */
984 ctrl = (struct virtio_crypto_op_ctrl_req *)malloc_virt_addr;
985 ctrl->header.opcode = VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION;
986 /* default data virtqueue is 0 */
987 ctrl->header.queue_id = 0;
988 ctrl->u.destroy_session.session_id = session->session_id;
991 status = &(((struct virtio_crypto_inhdr *)
992 ((uint8_t *)malloc_virt_addr + len_op_ctrl_req))->status);
993 *status = VIRTIO_CRYPTO_ERR;
995 /* indirect desc vring part */
996 desc = (struct vring_desc *)((uint8_t *)malloc_virt_addr
999 /* ctrl request part */
1000 desc[0].addr = malloc_phys_addr;
1001 desc[0].len = len_op_ctrl_req;
1002 desc[0].flags = VRING_DESC_F_NEXT;
1006 desc[1].addr = malloc_phys_addr + len_op_ctrl_req;
1007 desc[1].len = len_inhdr;
1008 desc[1].flags = VRING_DESC_F_WRITE;
1010 /* use only a single desc entry */
1011 head = vq->vq_desc_head_idx;
1012 vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT;
1013 vq->vq_ring.desc[head].addr = malloc_phys_addr + desc_offset;
1014 vq->vq_ring.desc[head].len
1015 = NUM_ENTRY_SYM_CLEAR_SESSION
1016 * sizeof(struct vring_desc);
1017 vq->vq_free_cnt -= needed;
1019 vq->vq_desc_head_idx = vq->vq_ring.desc[head].next;
1021 vq_update_avail_ring(vq, head);
1022 vq_update_avail_idx(vq);
1024 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d",
1025 vq->vq_queue_index);
1027 virtqueue_notify(vq);
1030 while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) {
1035 while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) {
1036 uint32_t idx, desc_idx, used_idx;
1037 struct vring_used_elem *uep;
1039 used_idx = (uint32_t)(vq->vq_used_cons_idx
1040 & (vq->vq_nentries - 1));
1041 uep = &vq->vq_ring.used->ring[used_idx];
1042 idx = (uint32_t) uep->id;
1044 while (vq->vq_ring.desc[desc_idx].flags
1045 & VRING_DESC_F_NEXT) {
1046 desc_idx = vq->vq_ring.desc[desc_idx].next;
1050 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx;
1051 vq->vq_desc_head_idx = idx;
1052 vq->vq_used_cons_idx++;
1056 if (*status != VIRTIO_CRYPTO_OK) {
1057 VIRTIO_CRYPTO_SESSION_LOG_ERR("Close session failed "
1058 "status=%"PRIu32", session_id=%"PRIu64"",
1059 *status, session->session_id);
1060 rte_free(malloc_virt_addr);
1064 VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n"
1065 "vq->vq_desc_head_idx=%d",
1066 vq->vq_free_cnt, vq->vq_desc_head_idx);
1068 VIRTIO_CRYPTO_SESSION_LOG_INFO("Close session %"PRIu64" successfully ",
1069 session->session_id);
1071 memset(session, 0, sizeof(struct virtio_crypto_session));
1072 struct rte_mempool *sess_mp = rte_mempool_from_obj(session);
1073 set_sym_session_private_data(sess, cryptodev_virtio_driver_id, NULL);
1074 rte_mempool_put(sess_mp, session);
1075 rte_free(malloc_virt_addr);
1078 static struct rte_crypto_cipher_xform *
1079 virtio_crypto_get_cipher_xform(struct rte_crypto_sym_xform *xform)
1082 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
1083 return &xform->cipher;
1085 xform = xform->next;
1091 static struct rte_crypto_auth_xform *
1092 virtio_crypto_get_auth_xform(struct rte_crypto_sym_xform *xform)
1095 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
1096 return &xform->auth;
1098 xform = xform->next;
1104 /** Get xform chain order */
1106 virtio_crypto_get_chain_order(struct rte_crypto_sym_xform *xform)
1112 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1113 xform->next == NULL)
1114 return VIRTIO_CRYPTO_CMD_CIPHER;
1116 /* Authentication Only */
1117 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1118 xform->next == NULL)
1119 return VIRTIO_CRYPTO_CMD_AUTH;
1121 /* Authenticate then Cipher */
1122 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1123 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
1124 return VIRTIO_CRYPTO_CMD_HASH_CIPHER;
1126 /* Cipher then Authenticate */
1127 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1128 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
1129 return VIRTIO_CRYPTO_CMD_CIPHER_HASH;
1135 virtio_crypto_sym_pad_cipher_param(
1136 struct virtio_crypto_cipher_session_para *para,
1137 struct rte_crypto_cipher_xform *cipher_xform)
1139 switch (cipher_xform->algo) {
1140 case RTE_CRYPTO_CIPHER_AES_CBC:
1141 para->algo = VIRTIO_CRYPTO_CIPHER_AES_CBC;
1144 VIRTIO_CRYPTO_SESSION_LOG_ERR("Crypto: Unsupported "
1145 "Cipher alg %u", cipher_xform->algo);
1149 para->keylen = cipher_xform->key.length;
1150 switch (cipher_xform->op) {
1151 case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
1152 para->op = VIRTIO_CRYPTO_OP_ENCRYPT;
1154 case RTE_CRYPTO_CIPHER_OP_DECRYPT:
1155 para->op = VIRTIO_CRYPTO_OP_DECRYPT;
1158 VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported cipher operation "
1167 virtio_crypto_sym_pad_auth_param(
1168 struct virtio_crypto_op_ctrl_req *ctrl,
1169 struct rte_crypto_auth_xform *auth_xform)
1172 struct virtio_crypto_alg_chain_session_para *para =
1173 &(ctrl->u.sym_create_session.u.chain.para);
1175 switch (ctrl->u.sym_create_session.u.chain.para.hash_mode) {
1176 case VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN:
1177 algo = &(para->u.hash_param.algo);
1179 case VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH:
1180 algo = &(para->u.mac_param.algo);
1183 VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported hash mode %u "
1185 ctrl->u.sym_create_session.u.chain.para.hash_mode);
1189 switch (auth_xform->algo) {
1190 case RTE_CRYPTO_AUTH_SHA1_HMAC:
1191 *algo = VIRTIO_CRYPTO_MAC_HMAC_SHA1;
1194 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1195 "Crypto: Undefined Hash algo %u specified",
1204 virtio_crypto_sym_pad_op_ctrl_req(
1205 struct virtio_crypto_op_ctrl_req *ctrl,
1206 struct rte_crypto_sym_xform *xform, bool is_chainned,
1207 uint8_t *cipher_key_data, uint8_t *auth_key_data,
1208 struct virtio_crypto_session *session)
1211 struct rte_crypto_auth_xform *auth_xform = NULL;
1212 struct rte_crypto_cipher_xform *cipher_xform = NULL;
1214 /* Get cipher xform from crypto xform chain */
1215 cipher_xform = virtio_crypto_get_cipher_xform(xform);
1217 if (cipher_xform->key.length > VIRTIO_CRYPTO_MAX_KEY_SIZE) {
1218 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1219 "cipher key size cannot be longer than %u",
1220 VIRTIO_CRYPTO_MAX_KEY_SIZE);
1223 if (cipher_xform->iv.length > VIRTIO_CRYPTO_MAX_IV_SIZE) {
1224 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1225 "cipher IV size cannot be longer than %u",
1226 VIRTIO_CRYPTO_MAX_IV_SIZE);
1230 ret = virtio_crypto_sym_pad_cipher_param(
1231 &ctrl->u.sym_create_session.u.chain.para
1232 .cipher_param, cipher_xform);
1234 ret = virtio_crypto_sym_pad_cipher_param(
1235 &ctrl->u.sym_create_session.u.cipher.para,
1239 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1240 "pad cipher parameter failed");
1244 memcpy(cipher_key_data, cipher_xform->key.data,
1245 cipher_xform->key.length);
1247 session->iv.offset = cipher_xform->iv.offset;
1248 session->iv.length = cipher_xform->iv.length;
1251 /* Get auth xform from crypto xform chain */
1252 auth_xform = virtio_crypto_get_auth_xform(xform);
1254 /* FIXME: support VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */
1255 struct virtio_crypto_alg_chain_session_para *para =
1256 &(ctrl->u.sym_create_session.u.chain.para);
1257 if (auth_xform->key.length) {
1258 if (auth_xform->key.length >
1259 VIRTIO_CRYPTO_MAX_KEY_SIZE) {
1260 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1261 "auth key size cannot be longer than %u",
1262 VIRTIO_CRYPTO_MAX_KEY_SIZE);
1265 para->hash_mode = VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH;
1266 para->u.mac_param.auth_key_len =
1267 (uint32_t)auth_xform->key.length;
1268 para->u.mac_param.hash_result_len =
1269 auth_xform->digest_length;
1270 memcpy(auth_key_data, auth_xform->key.data,
1271 auth_xform->key.length);
1273 para->hash_mode = VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN;
1274 para->u.hash_param.hash_result_len =
1275 auth_xform->digest_length;
1278 ret = virtio_crypto_sym_pad_auth_param(ctrl, auth_xform);
1280 VIRTIO_CRYPTO_SESSION_LOG_ERR("pad auth parameter "
1290 virtio_crypto_check_sym_configure_session_paras(
1291 struct rte_cryptodev *dev,
1292 struct rte_crypto_sym_xform *xform,
1293 struct rte_cryptodev_sym_session *sym_sess,
1294 struct rte_mempool *mempool)
1296 if (unlikely(xform == NULL) || unlikely(sym_sess == NULL) ||
1297 unlikely(mempool == NULL)) {
1298 VIRTIO_CRYPTO_SESSION_LOG_ERR("NULL pointer");
1302 if (virtio_crypto_check_sym_session_paras(dev) < 0)
1309 virtio_crypto_sym_configure_session(
1310 struct rte_cryptodev *dev,
1311 struct rte_crypto_sym_xform *xform,
1312 struct rte_cryptodev_sym_session *sess,
1313 struct rte_mempool *mempool)
1316 struct virtio_crypto_session crypto_sess;
1317 void *session_private = &crypto_sess;
1318 struct virtio_crypto_session *session;
1319 struct virtio_crypto_op_ctrl_req *ctrl_req;
1320 enum virtio_crypto_cmd_id cmd_id;
1321 uint8_t cipher_key_data[VIRTIO_CRYPTO_MAX_KEY_SIZE] = {0};
1322 uint8_t auth_key_data[VIRTIO_CRYPTO_MAX_KEY_SIZE] = {0};
1323 struct virtio_crypto_hw *hw;
1324 struct virtqueue *control_vq;
1326 PMD_INIT_FUNC_TRACE();
1328 ret = virtio_crypto_check_sym_configure_session_paras(dev, xform,
1331 VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid parameters");
1335 if (rte_mempool_get(mempool, &session_private)) {
1336 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1337 "Couldn't get object from session mempool");
1341 session = (struct virtio_crypto_session *)session_private;
1342 memset(session, 0, sizeof(struct virtio_crypto_session));
1343 ctrl_req = &session->ctrl;
1344 ctrl_req->header.opcode = VIRTIO_CRYPTO_CIPHER_CREATE_SESSION;
1345 /* FIXME: support multiqueue */
1346 ctrl_req->header.queue_id = 0;
1348 hw = dev->data->dev_private;
1349 control_vq = hw->cvq;
1351 cmd_id = virtio_crypto_get_chain_order(xform);
1352 if (cmd_id == VIRTIO_CRYPTO_CMD_CIPHER_HASH)
1353 ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order
1354 = VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
1355 if (cmd_id == VIRTIO_CRYPTO_CMD_HASH_CIPHER)
1356 ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order
1357 = VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
1360 case VIRTIO_CRYPTO_CMD_CIPHER_HASH:
1361 case VIRTIO_CRYPTO_CMD_HASH_CIPHER:
1362 ctrl_req->u.sym_create_session.op_type
1363 = VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING;
1365 ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req,
1366 xform, true, cipher_key_data, auth_key_data, session);
1368 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1369 "padding sym op ctrl req failed");
1372 ret = virtio_crypto_send_command(control_vq, ctrl_req,
1373 cipher_key_data, auth_key_data, session);
1375 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1376 "create session failed: %d", ret);
1380 case VIRTIO_CRYPTO_CMD_CIPHER:
1381 ctrl_req->u.sym_create_session.op_type
1382 = VIRTIO_CRYPTO_SYM_OP_CIPHER;
1383 ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req, xform,
1384 false, cipher_key_data, auth_key_data, session);
1386 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1387 "padding sym op ctrl req failed");
1390 ret = virtio_crypto_send_command(control_vq, ctrl_req,
1391 cipher_key_data, NULL, session);
1393 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1394 "create session failed: %d", ret);
1399 VIRTIO_CRYPTO_SESSION_LOG_ERR(
1400 "Unsupported operation chain order parameter");
1404 set_sym_session_private_data(sess, dev->driver_id,
1414 virtio_crypto_dev_info_get(struct rte_cryptodev *dev,
1415 struct rte_cryptodev_info *info)
1417 struct virtio_crypto_hw *hw = dev->data->dev_private;
1419 PMD_INIT_FUNC_TRACE();
1422 info->driver_id = cryptodev_virtio_driver_id;
1423 info->feature_flags = dev->feature_flags;
1424 info->max_nb_queue_pairs = hw->max_dataqueues;
1425 /* No limit of number of sessions */
1426 info->sym.max_nb_sessions = 0;
1427 info->capabilities = hw->virtio_dev_capabilities;
1432 crypto_virtio_pci_probe(
1433 struct rte_pci_driver *pci_drv __rte_unused,
1434 struct rte_pci_device *pci_dev)
1436 struct rte_cryptodev_pmd_init_params init_params = {
1438 .socket_id = pci_dev->device.numa_node,
1439 .private_data_size = sizeof(struct virtio_crypto_hw)
1441 char name[RTE_CRYPTODEV_NAME_MAX_LEN];
1443 VIRTIO_CRYPTO_DRV_LOG_DBG("Found Crypto device at %02x:%02x.%x",
1445 pci_dev->addr.devid,
1446 pci_dev->addr.function);
1448 rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
1450 return crypto_virtio_create(name, pci_dev, &init_params);
1454 crypto_virtio_pci_remove(
1455 struct rte_pci_device *pci_dev __rte_unused)
1457 struct rte_cryptodev *cryptodev;
1458 char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
1460 if (pci_dev == NULL)
1463 rte_pci_device_name(&pci_dev->addr, cryptodev_name,
1464 sizeof(cryptodev_name));
1466 cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name);
1467 if (cryptodev == NULL)
1470 return virtio_crypto_dev_uninit(cryptodev);
1473 static struct rte_pci_driver rte_virtio_crypto_driver = {
1474 .id_table = pci_id_virtio_crypto_map,
1476 .probe = crypto_virtio_pci_probe,
1477 .remove = crypto_virtio_pci_remove
1480 static struct cryptodev_driver virtio_crypto_drv;
1482 RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_VIRTIO_PMD, rte_virtio_crypto_driver);
1483 RTE_PMD_REGISTER_CRYPTO_DRIVER(virtio_crypto_drv,
1484 rte_virtio_crypto_driver.driver,
1485 cryptodev_virtio_driver_id);
1486 RTE_LOG_REGISTER_SUFFIX(virtio_crypto_logtype_init, init, NOTICE);
1487 RTE_LOG_REGISTER_SUFFIX(virtio_crypto_logtype_session, session, NOTICE);
1488 RTE_LOG_REGISTER_SUFFIX(virtio_crypto_logtype_rx, rx, NOTICE);
1489 RTE_LOG_REGISTER_SUFFIX(virtio_crypto_logtype_tx, tx, NOTICE);
1490 RTE_LOG_REGISTER_SUFFIX(virtio_crypto_logtype_driver, driver, NOTICE);