1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2018 Intel Corporation
4 #include <rte_malloc.h>
7 #include <rte_cryptodev.h>
9 #include "rte_vhost_crypto.h"
11 #include "vhost_user.h"
12 #include "virtio_crypto.h"
14 #define INHDR_LEN (sizeof(struct virtio_crypto_inhdr))
15 #define IV_OFFSET (sizeof(struct rte_crypto_op) + \
16 sizeof(struct rte_crypto_sym_op))
18 #ifdef RTE_LIBRTE_VHOST_DEBUG
19 #define VC_LOG_ERR(fmt, args...) \
20 RTE_LOG(ERR, USER1, "[%s] %s() line %u: " fmt "\n", \
21 "Vhost-Crypto", __func__, __LINE__, ## args)
22 #define VC_LOG_INFO(fmt, args...) \
23 RTE_LOG(INFO, USER1, "[%s] %s() line %u: " fmt "\n", \
24 "Vhost-Crypto", __func__, __LINE__, ## args)
26 #define VC_LOG_DBG(fmt, args...) \
27 RTE_LOG(DEBUG, USER1, "[%s] %s() line %u: " fmt "\n", \
28 "Vhost-Crypto", __func__, __LINE__, ## args)
30 #define VC_LOG_ERR(fmt, args...) \
31 RTE_LOG(ERR, USER1, "[VHOST-Crypto]: " fmt "\n", ## args)
32 #define VC_LOG_INFO(fmt, args...) \
33 RTE_LOG(INFO, USER1, "[VHOST-Crypto]: " fmt "\n", ## args)
34 #define VC_LOG_DBG(fmt, args...)
37 #define GPA_TO_VVA(t, m, a) ((t)(uintptr_t)rte_vhost_gpa_to_vva(m, a))
40 cipher_algo_transform(uint32_t virtio_cipher_algo)
44 switch (virtio_cipher_algo) {
45 case VIRTIO_CRYPTO_CIPHER_AES_CBC:
46 ret = RTE_CRYPTO_CIPHER_AES_CBC;
48 case VIRTIO_CRYPTO_CIPHER_AES_CTR:
49 ret = RTE_CRYPTO_CIPHER_AES_CTR;
51 case VIRTIO_CRYPTO_CIPHER_DES_ECB:
52 ret = -VIRTIO_CRYPTO_NOTSUPP;
54 case VIRTIO_CRYPTO_CIPHER_DES_CBC:
55 ret = RTE_CRYPTO_CIPHER_DES_CBC;
57 case VIRTIO_CRYPTO_CIPHER_3DES_ECB:
58 ret = RTE_CRYPTO_CIPHER_3DES_ECB;
60 case VIRTIO_CRYPTO_CIPHER_3DES_CBC:
61 ret = RTE_CRYPTO_CIPHER_3DES_CBC;
63 case VIRTIO_CRYPTO_CIPHER_3DES_CTR:
64 ret = RTE_CRYPTO_CIPHER_3DES_CTR;
66 case VIRTIO_CRYPTO_CIPHER_KASUMI_F8:
67 ret = RTE_CRYPTO_CIPHER_KASUMI_F8;
69 case VIRTIO_CRYPTO_CIPHER_SNOW3G_UEA2:
70 ret = RTE_CRYPTO_CIPHER_SNOW3G_UEA2;
72 case VIRTIO_CRYPTO_CIPHER_AES_F8:
73 ret = RTE_CRYPTO_CIPHER_AES_F8;
75 case VIRTIO_CRYPTO_CIPHER_AES_XTS:
76 ret = RTE_CRYPTO_CIPHER_AES_XTS;
78 case VIRTIO_CRYPTO_CIPHER_ZUC_EEA3:
79 ret = RTE_CRYPTO_CIPHER_ZUC_EEA3;
82 ret = -VIRTIO_CRYPTO_BADMSG;
90 auth_algo_transform(uint32_t virtio_auth_algo)
94 switch (virtio_auth_algo) {
96 case VIRTIO_CRYPTO_NO_MAC:
97 ret = RTE_CRYPTO_AUTH_NULL;
99 case VIRTIO_CRYPTO_MAC_HMAC_MD5:
100 ret = RTE_CRYPTO_AUTH_MD5_HMAC;
102 case VIRTIO_CRYPTO_MAC_HMAC_SHA1:
103 ret = RTE_CRYPTO_AUTH_SHA1_HMAC;
105 case VIRTIO_CRYPTO_MAC_HMAC_SHA_224:
106 ret = RTE_CRYPTO_AUTH_SHA224_HMAC;
108 case VIRTIO_CRYPTO_MAC_HMAC_SHA_256:
109 ret = RTE_CRYPTO_AUTH_SHA256_HMAC;
111 case VIRTIO_CRYPTO_MAC_HMAC_SHA_384:
112 ret = RTE_CRYPTO_AUTH_SHA384_HMAC;
114 case VIRTIO_CRYPTO_MAC_HMAC_SHA_512:
115 ret = RTE_CRYPTO_AUTH_SHA512_HMAC;
117 case VIRTIO_CRYPTO_MAC_CMAC_3DES:
118 ret = -VIRTIO_CRYPTO_NOTSUPP;
120 case VIRTIO_CRYPTO_MAC_CMAC_AES:
121 ret = RTE_CRYPTO_AUTH_AES_CMAC;
123 case VIRTIO_CRYPTO_MAC_KASUMI_F9:
124 ret = RTE_CRYPTO_AUTH_KASUMI_F9;
126 case VIRTIO_CRYPTO_MAC_SNOW3G_UIA2:
127 ret = RTE_CRYPTO_AUTH_SNOW3G_UIA2;
129 case VIRTIO_CRYPTO_MAC_GMAC_AES:
130 ret = RTE_CRYPTO_AUTH_AES_GMAC;
132 case VIRTIO_CRYPTO_MAC_GMAC_TWOFISH:
133 ret = -VIRTIO_CRYPTO_NOTSUPP;
135 case VIRTIO_CRYPTO_MAC_CBCMAC_AES:
136 ret = RTE_CRYPTO_AUTH_AES_CBC_MAC;
138 case VIRTIO_CRYPTO_MAC_CBCMAC_KASUMI_F9:
139 ret = -VIRTIO_CRYPTO_NOTSUPP;
141 case VIRTIO_CRYPTO_MAC_XCBC_AES:
142 ret = RTE_CRYPTO_AUTH_AES_XCBC_MAC;
145 ret = -VIRTIO_CRYPTO_BADMSG;
152 static int get_iv_len(enum rte_crypto_cipher_algorithm algo)
157 case RTE_CRYPTO_CIPHER_3DES_CBC:
160 case RTE_CRYPTO_CIPHER_3DES_CTR:
163 case RTE_CRYPTO_CIPHER_3DES_ECB:
166 case RTE_CRYPTO_CIPHER_AES_CBC:
170 /* TODO: add common algos */
181 * vhost_crypto struct is used to maintain a number of virtio_cryptos and
182 * one DPDK crypto device that deals with all crypto workloads. It is declared
183 * here and defined in vhost_crypto.c
185 struct vhost_crypto {
186 /** Used to lookup DPDK Cryptodev Session based on VIRTIO crypto
189 struct rte_hash *session_map;
190 struct rte_mempool *mbuf_pool;
191 struct rte_mempool *sess_pool;
193 /** DPDK cryptodev ID */
197 uint64_t last_session_id;
199 uint64_t cache_session_id;
200 struct rte_cryptodev_sym_session *cache_session;
201 /** socket id for the device */
204 struct virtio_net *dev;
207 } __rte_cache_aligned;
209 struct vhost_crypto_data_req {
210 struct vring_desc *head;
211 struct rte_vhost_memory *mem;
212 struct virtio_crypto_inhdr *inhdr;
213 struct vhost_virtqueue *vq;
214 struct vring_desc *wb_desc;
222 transform_cipher_param(struct rte_crypto_sym_xform *xform,
223 VhostUserCryptoSessionParam *param)
227 ret = cipher_algo_transform(param->cipher_algo);
228 if (unlikely(ret < 0))
231 xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
232 xform->cipher.algo = (uint32_t)ret;
233 xform->cipher.key.length = param->cipher_key_len;
234 if (xform->cipher.key.length > 0)
235 xform->cipher.key.data = param->cipher_key_buf;
236 if (param->dir == VIRTIO_CRYPTO_OP_ENCRYPT)
237 xform->cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
238 else if (param->dir == VIRTIO_CRYPTO_OP_DECRYPT)
239 xform->cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
241 VC_LOG_DBG("Bad operation type");
242 return -VIRTIO_CRYPTO_BADMSG;
245 ret = get_iv_len(xform->cipher.algo);
246 if (unlikely(ret < 0))
248 xform->cipher.iv.length = (uint16_t)ret;
249 xform->cipher.iv.offset = IV_OFFSET;
254 transform_chain_param(struct rte_crypto_sym_xform *xforms,
255 VhostUserCryptoSessionParam *param)
257 struct rte_crypto_sym_xform *xform_cipher, *xform_auth;
260 switch (param->chaining_dir) {
261 case VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER:
263 xform_cipher = xforms->next;
264 xform_cipher->cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
265 xform_auth->auth.op = RTE_CRYPTO_AUTH_OP_VERIFY;
267 case VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH:
268 xform_cipher = xforms;
269 xform_auth = xforms->next;
270 xform_cipher->cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
271 xform_auth->auth.op = RTE_CRYPTO_AUTH_OP_GENERATE;
274 return -VIRTIO_CRYPTO_BADMSG;
278 ret = cipher_algo_transform(param->cipher_algo);
279 if (unlikely(ret < 0))
281 xform_cipher->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
282 xform_cipher->cipher.algo = (uint32_t)ret;
283 xform_cipher->cipher.key.length = param->cipher_key_len;
284 xform_cipher->cipher.key.data = param->cipher_key_buf;
285 ret = get_iv_len(xform_cipher->cipher.algo);
286 if (unlikely(ret < 0))
288 xform_cipher->cipher.iv.length = (uint16_t)ret;
289 xform_cipher->cipher.iv.offset = IV_OFFSET;
292 xform_auth->type = RTE_CRYPTO_SYM_XFORM_AUTH;
293 ret = auth_algo_transform(param->hash_algo);
294 if (unlikely(ret < 0))
296 xform_auth->auth.algo = (uint32_t)ret;
297 xform_auth->auth.digest_length = param->digest_len;
298 xform_auth->auth.key.length = param->auth_key_len;
299 xform_auth->auth.key.data = param->auth_key_buf;
305 vhost_crypto_create_sess(struct vhost_crypto *vcrypto,
306 VhostUserCryptoSessionParam *sess_param)
308 struct rte_crypto_sym_xform xform1 = {0}, xform2 = {0};
309 struct rte_cryptodev_sym_session *session;
312 switch (sess_param->op_type) {
313 case VIRTIO_CRYPTO_SYM_OP_NONE:
314 case VIRTIO_CRYPTO_SYM_OP_CIPHER:
315 ret = transform_cipher_param(&xform1, sess_param);
317 VC_LOG_ERR("Error transform session msg (%i)", ret);
318 sess_param->session_id = ret;
322 case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING:
323 if (unlikely(sess_param->hash_mode !=
324 VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH)) {
325 sess_param->session_id = -VIRTIO_CRYPTO_NOTSUPP;
326 VC_LOG_ERR("Error transform session message (%i)",
327 -VIRTIO_CRYPTO_NOTSUPP);
331 xform1.next = &xform2;
333 ret = transform_chain_param(&xform1, sess_param);
335 VC_LOG_ERR("Error transform session message (%i)", ret);
336 sess_param->session_id = ret;
342 VC_LOG_ERR("Algorithm not yet supported");
343 sess_param->session_id = -VIRTIO_CRYPTO_NOTSUPP;
347 session = rte_cryptodev_sym_session_create(vcrypto->sess_pool);
349 VC_LOG_ERR("Failed to create session");
350 sess_param->session_id = -VIRTIO_CRYPTO_ERR;
354 if (rte_cryptodev_sym_session_init(vcrypto->cid, session, &xform1,
355 vcrypto->sess_pool) < 0) {
356 VC_LOG_ERR("Failed to initialize session");
357 sess_param->session_id = -VIRTIO_CRYPTO_ERR;
361 /* insert hash to map */
362 if (rte_hash_add_key_data(vcrypto->session_map,
363 &vcrypto->last_session_id, session) < 0) {
364 VC_LOG_ERR("Failed to insert session to hash table");
366 if (rte_cryptodev_sym_session_clear(vcrypto->cid, session) < 0)
367 VC_LOG_ERR("Failed to clear session");
369 if (rte_cryptodev_sym_session_free(session) < 0)
370 VC_LOG_ERR("Failed to free session");
372 sess_param->session_id = -VIRTIO_CRYPTO_ERR;
376 VC_LOG_INFO("Session %"PRIu64" created for vdev %i.",
377 vcrypto->last_session_id, vcrypto->dev->vid);
379 sess_param->session_id = vcrypto->last_session_id;
380 vcrypto->last_session_id++;
384 vhost_crypto_close_sess(struct vhost_crypto *vcrypto, uint64_t session_id)
386 struct rte_cryptodev_sym_session *session;
387 uint64_t sess_id = session_id;
390 ret = rte_hash_lookup_data(vcrypto->session_map, &sess_id,
393 if (unlikely(ret < 0)) {
394 VC_LOG_ERR("Failed to delete session %"PRIu64".", session_id);
395 return -VIRTIO_CRYPTO_INVSESS;
398 if (rte_cryptodev_sym_session_clear(vcrypto->cid, session) < 0) {
399 VC_LOG_DBG("Failed to clear session");
400 return -VIRTIO_CRYPTO_ERR;
403 if (rte_cryptodev_sym_session_free(session) < 0) {
404 VC_LOG_DBG("Failed to free session");
405 return -VIRTIO_CRYPTO_ERR;
408 if (rte_hash_del_key(vcrypto->session_map, &sess_id) < 0) {
409 VC_LOG_DBG("Failed to delete session from hash table.");
410 return -VIRTIO_CRYPTO_ERR;
413 VC_LOG_INFO("Session %"PRIu64" deleted for vdev %i.", sess_id,
420 vhost_crypto_msg_post_handler(int vid, void *msg, uint32_t *require_reply)
422 struct virtio_net *dev = get_device(vid);
423 struct vhost_crypto *vcrypto;
424 VhostUserMsg *vmsg = msg;
427 if (dev == NULL || require_reply == NULL) {
428 VC_LOG_ERR("Invalid vid %i", vid);
432 vcrypto = dev->extern_data;
433 if (vcrypto == NULL) {
434 VC_LOG_ERR("Cannot find required data, is it initialized?");
440 if (vmsg->request.master == VHOST_USER_CRYPTO_CREATE_SESS) {
441 vhost_crypto_create_sess(vcrypto,
442 &vmsg->payload.crypto_session);
444 } else if (vmsg->request.master == VHOST_USER_CRYPTO_CLOSE_SESS)
445 ret = vhost_crypto_close_sess(vcrypto, vmsg->payload.u64);
452 static __rte_always_inline struct vring_desc *
453 find_write_desc(struct vring_desc *head, struct vring_desc *desc)
455 if (desc->flags & VRING_DESC_F_WRITE)
458 while (desc->flags & VRING_DESC_F_NEXT) {
459 desc = &head[desc->next];
460 if (desc->flags & VRING_DESC_F_WRITE)
467 static struct virtio_crypto_inhdr *
468 reach_inhdr(struct vring_desc *head, struct rte_vhost_memory *mem,
469 struct vring_desc *desc)
471 while (desc->flags & VRING_DESC_F_NEXT)
472 desc = &head[desc->next];
474 return GPA_TO_VVA(struct virtio_crypto_inhdr *, mem, desc->addr);
477 static __rte_always_inline int
478 move_desc(struct vring_desc *head, struct vring_desc **cur_desc,
481 struct vring_desc *desc = *cur_desc;
484 rte_prefetch0(&head[desc->next]);
487 while ((desc->flags & VRING_DESC_F_NEXT) && left > 0) {
488 desc = &head[desc->next];
489 rte_prefetch0(&head[desc->next]);
493 if (unlikely(left < 0)) {
494 VC_LOG_ERR("Incorrect virtio descriptor");
498 *cur_desc = &head[desc->next];
503 copy_data(void *dst_data, struct vring_desc *head, struct rte_vhost_memory *mem,
504 struct vring_desc **cur_desc, uint32_t size)
506 struct vring_desc *desc = *cur_desc;
508 uint8_t *data = dst_data;
512 rte_prefetch0(&head[desc->next]);
513 to_copy = RTE_MIN(desc->len, (uint32_t)left);
514 src = GPA_TO_VVA(uint8_t *, mem, desc->addr);
515 rte_memcpy((uint8_t *)data, src, to_copy);
518 while ((desc->flags & VRING_DESC_F_NEXT) && left > 0) {
519 desc = &head[desc->next];
520 rte_prefetch0(&head[desc->next]);
521 to_copy = RTE_MIN(desc->len, (uint32_t)left);
522 src = GPA_TO_VVA(uint8_t *, mem, desc->addr);
523 rte_memcpy(data + size - left, src, to_copy);
527 if (unlikely(left < 0)) {
528 VC_LOG_ERR("Incorrect virtio descriptor");
532 *cur_desc = &head[desc->next];
537 static __rte_always_inline void *
538 get_data_ptr(struct vring_desc *head, struct rte_vhost_memory *mem,
539 struct vring_desc **cur_desc, uint32_t size)
543 data = GPA_TO_VVA(void *, mem, (*cur_desc)->addr);
544 if (unlikely(!data)) {
545 VC_LOG_ERR("Failed to get object");
549 if (unlikely(move_desc(head, cur_desc, size) < 0))
556 write_back_data(struct rte_crypto_op *op, struct vhost_crypto_data_req *vc_req)
558 struct rte_mbuf *mbuf = op->sym->m_dst;
559 struct vring_desc *head = vc_req->head;
560 struct rte_vhost_memory *mem = vc_req->mem;
561 struct vring_desc *desc = vc_req->wb_desc;
562 int left = vc_req->wb_len;
564 uint8_t *src_data = mbuf->buf_addr, *dst;
566 rte_prefetch0(&head[desc->next]);
567 to_write = RTE_MIN(desc->len, (uint32_t)left);
568 dst = GPA_TO_VVA(uint8_t *, mem, desc->addr);
569 rte_memcpy(dst, src_data, to_write);
571 src_data += to_write;
573 while ((desc->flags & VRING_DESC_F_NEXT) && left > 0) {
574 desc = &head[desc->next];
575 rte_prefetch0(&head[desc->next]);
576 to_write = RTE_MIN(desc->len, (uint32_t)left);
577 dst = GPA_TO_VVA(uint8_t *, mem, desc->addr);
578 rte_memcpy(dst, src_data, to_write);
580 src_data += to_write;
583 if (unlikely(left < 0)) {
584 VC_LOG_ERR("Incorrect virtio descriptor");
592 prepare_sym_cipher_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op,
593 struct vhost_crypto_data_req *vc_req,
594 struct virtio_crypto_cipher_data_req *cipher,
595 struct vring_desc *cur_desc)
597 struct vring_desc *head = vc_req->head;
598 struct vring_desc *desc = cur_desc;
599 struct rte_vhost_memory *mem = vc_req->mem;
600 struct rte_mbuf *m_src = op->sym->m_src, *m_dst = op->sym->m_dst;
601 uint8_t *iv_data = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET);
606 if (unlikely(copy_data(iv_data, head, mem, &desc,
607 cipher->para.iv_len) < 0)) {
608 ret = VIRTIO_CRYPTO_BADMSG;
612 m_src->data_len = cipher->para.src_data_len;
614 switch (vcrypto->option) {
615 case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
616 m_src->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr,
617 cipher->para.src_data_len);
618 m_src->buf_addr = get_data_ptr(head, mem, &desc,
619 cipher->para.src_data_len);
620 if (unlikely(m_src->buf_iova == 0 ||
621 m_src->buf_addr == NULL)) {
622 VC_LOG_ERR("zero_copy may fail due to cross page data");
623 ret = VIRTIO_CRYPTO_ERR;
627 case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
628 if (unlikely(cipher->para.src_data_len >
629 RTE_MBUF_DEFAULT_BUF_SIZE)) {
630 VC_LOG_ERR("Not enough space to do data copy");
631 ret = VIRTIO_CRYPTO_ERR;
634 if (unlikely(copy_data(rte_pktmbuf_mtod(m_src, uint8_t *), head,
635 mem, &desc, cipher->para.src_data_len))
637 ret = VIRTIO_CRYPTO_BADMSG;
642 ret = VIRTIO_CRYPTO_BADMSG;
647 desc = find_write_desc(head, desc);
648 if (unlikely(!desc)) {
649 VC_LOG_ERR("Cannot find write location");
650 ret = VIRTIO_CRYPTO_BADMSG;
654 switch (vcrypto->option) {
655 case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
656 m_dst->buf_iova = gpa_to_hpa(vcrypto->dev,
657 desc->addr, cipher->para.dst_data_len);
658 m_dst->buf_addr = get_data_ptr(head, mem, &desc,
659 cipher->para.dst_data_len);
660 if (unlikely(m_dst->buf_iova == 0 || m_dst->buf_addr == NULL)) {
661 VC_LOG_ERR("zero_copy may fail due to cross page data");
662 ret = VIRTIO_CRYPTO_ERR;
666 m_dst->data_len = cipher->para.dst_data_len;
668 case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
669 vc_req->wb_desc = desc;
670 vc_req->wb_len = cipher->para.dst_data_len;
671 if (unlikely(move_desc(head, &desc, vc_req->wb_len) < 0)) {
672 ret = VIRTIO_CRYPTO_ERR;
677 ret = VIRTIO_CRYPTO_BADMSG;
682 op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
683 op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
685 op->sym->cipher.data.offset = 0;
686 op->sym->cipher.data.length = cipher->para.src_data_len;
688 vc_req->inhdr = get_data_ptr(head, mem, &desc, INHDR_LEN);
689 if (unlikely(vc_req->inhdr == NULL)) {
690 ret = VIRTIO_CRYPTO_BADMSG;
694 vc_req->inhdr->status = VIRTIO_CRYPTO_OK;
695 vc_req->len = cipher->para.dst_data_len + INHDR_LEN;
700 vc_req->len = INHDR_LEN;
705 prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op,
706 struct vhost_crypto_data_req *vc_req,
707 struct virtio_crypto_alg_chain_data_req *chain,
708 struct vring_desc *cur_desc)
710 struct vring_desc *head = vc_req->head;
711 struct vring_desc *desc = cur_desc;
712 struct rte_vhost_memory *mem = vc_req->mem;
713 struct rte_mbuf *m_src = op->sym->m_src, *m_dst = op->sym->m_dst;
714 uint8_t *iv_data = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET);
715 uint32_t digest_offset;
721 if (unlikely(copy_data(iv_data, head, mem, &desc,
722 chain->para.iv_len) < 0)) {
723 ret = VIRTIO_CRYPTO_BADMSG;
727 m_src->data_len = chain->para.src_data_len;
728 m_dst->data_len = chain->para.dst_data_len;
730 switch (vcrypto->option) {
731 case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
732 m_src->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr,
733 chain->para.src_data_len);
734 m_src->buf_addr = get_data_ptr(head, mem, &desc,
735 chain->para.src_data_len);
736 if (unlikely(m_src->buf_iova == 0 || m_src->buf_addr == NULL)) {
737 VC_LOG_ERR("zero_copy may fail due to cross page data");
738 ret = VIRTIO_CRYPTO_ERR;
742 case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
743 if (unlikely(chain->para.src_data_len >
744 RTE_MBUF_DEFAULT_BUF_SIZE)) {
745 VC_LOG_ERR("Not enough space to do data copy");
746 ret = VIRTIO_CRYPTO_ERR;
749 if (unlikely(copy_data(rte_pktmbuf_mtod(m_src, uint8_t *), head,
750 mem, &desc, chain->para.src_data_len)) < 0) {
751 ret = VIRTIO_CRYPTO_BADMSG;
756 ret = VIRTIO_CRYPTO_BADMSG;
761 desc = find_write_desc(head, desc);
762 if (unlikely(!desc)) {
763 VC_LOG_ERR("Cannot find write location");
764 ret = VIRTIO_CRYPTO_BADMSG;
768 switch (vcrypto->option) {
769 case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
770 m_dst->buf_iova = gpa_to_hpa(vcrypto->dev,
771 desc->addr, chain->para.dst_data_len);
772 m_dst->buf_addr = get_data_ptr(head, mem, &desc,
773 chain->para.dst_data_len);
774 if (unlikely(m_dst->buf_iova == 0 || m_dst->buf_addr == NULL)) {
775 VC_LOG_ERR("zero_copy may fail due to cross page data");
776 ret = VIRTIO_CRYPTO_ERR;
780 op->sym->auth.digest.phys_addr = gpa_to_hpa(vcrypto->dev,
781 desc->addr, chain->para.hash_result_len);
782 op->sym->auth.digest.data = get_data_ptr(head, mem, &desc,
783 chain->para.hash_result_len);
784 if (unlikely(op->sym->auth.digest.phys_addr == 0)) {
785 VC_LOG_ERR("zero_copy may fail due to cross page data");
786 ret = VIRTIO_CRYPTO_ERR;
790 case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
791 digest_offset = m_dst->data_len;
792 digest_addr = rte_pktmbuf_mtod_offset(m_dst, void *,
795 vc_req->wb_desc = desc;
796 vc_req->wb_len = m_dst->data_len + chain->para.hash_result_len;
798 if (unlikely(move_desc(head, &desc,
799 chain->para.dst_data_len) < 0)) {
800 ret = VIRTIO_CRYPTO_BADMSG;
804 if (unlikely(copy_data(digest_addr, head, mem, &desc,
805 chain->para.hash_result_len)) < 0) {
806 ret = VIRTIO_CRYPTO_BADMSG;
810 op->sym->auth.digest.data = digest_addr;
811 op->sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m_dst,
813 if (unlikely(move_desc(head, &desc,
814 chain->para.hash_result_len) < 0)) {
815 ret = VIRTIO_CRYPTO_ERR;
820 ret = VIRTIO_CRYPTO_BADMSG;
825 vc_req->inhdr = get_data_ptr(head, mem, &desc, INHDR_LEN);
826 if (unlikely(vc_req->inhdr == NULL)) {
827 ret = VIRTIO_CRYPTO_BADMSG;
831 vc_req->inhdr->status = VIRTIO_CRYPTO_OK;
833 op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
834 op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
836 op->sym->cipher.data.offset = chain->para.cipher_start_src_offset;
837 op->sym->cipher.data.length = chain->para.src_data_len -
838 chain->para.cipher_start_src_offset;
840 op->sym->auth.data.offset = chain->para.hash_start_src_offset;
841 op->sym->auth.data.length = chain->para.len_to_hash;
843 vc_req->len = chain->para.dst_data_len + chain->para.hash_result_len +
848 vc_req->len = INHDR_LEN;
853 * Process on descriptor
855 static __rte_always_inline int
856 vhost_crypto_process_one_req(struct vhost_crypto *vcrypto,
857 struct vhost_virtqueue *vq, struct rte_crypto_op *op,
858 struct vring_desc *head, uint16_t desc_idx,
859 struct rte_vhost_memory *mem)
861 struct vhost_crypto_data_req *vc_req = RTE_PTR_ADD(op->sym->m_src,
862 sizeof(struct rte_mbuf));
863 struct rte_cryptodev_sym_session *session;
864 struct virtio_crypto_op_data_req *req;
865 struct virtio_crypto_inhdr *inhdr;
866 struct vring_desc *desc = NULL;
870 vc_req->desc_idx = desc_idx;
872 if (likely(head->flags & VRING_DESC_F_INDIRECT)) {
873 head = GPA_TO_VVA(struct vring_desc *, mem, head->addr);
885 vc_req->zero_copy = vcrypto->option;
887 req = get_data_ptr(head, mem, &desc, sizeof(*req));
888 if (unlikely(req == NULL)) {
889 err = VIRTIO_CRYPTO_ERR;
890 VC_LOG_ERR("Invalid descriptor");
894 switch (req->header.opcode) {
895 case VIRTIO_CRYPTO_CIPHER_ENCRYPT:
896 case VIRTIO_CRYPTO_CIPHER_DECRYPT:
897 session_id = req->header.session_id;
899 /* one branch to avoid unnecessary table lookup */
900 if (vcrypto->cache_session_id != session_id) {
901 err = rte_hash_lookup_data(vcrypto->session_map,
902 &session_id, (void **)&session);
903 if (unlikely(err < 0)) {
904 err = VIRTIO_CRYPTO_ERR;
905 VC_LOG_ERR("Failed to find session %"PRIu64,
910 vcrypto->cache_session = session;
911 vcrypto->cache_session_id = session_id;
914 session = vcrypto->cache_session;
916 err = rte_crypto_op_attach_sym_session(op, session);
917 if (unlikely(err < 0)) {
918 err = VIRTIO_CRYPTO_ERR;
919 VC_LOG_ERR("Failed to attach session to op");
923 switch (req->u.sym_req.op_type) {
924 case VIRTIO_CRYPTO_SYM_OP_NONE:
925 err = VIRTIO_CRYPTO_NOTSUPP;
927 case VIRTIO_CRYPTO_SYM_OP_CIPHER:
928 err = prepare_sym_cipher_op(vcrypto, op, vc_req,
929 &req->u.sym_req.u.cipher, desc);
931 case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING:
932 err = prepare_sym_chain_op(vcrypto, op, vc_req,
933 &req->u.sym_req.u.chain, desc);
936 if (unlikely(err != 0)) {
937 VC_LOG_ERR("Failed to process sym request");
942 VC_LOG_ERR("Unsupported symmetric crypto request type %u",
951 inhdr = reach_inhdr(head, mem, desc);
952 if (likely(inhdr != NULL))
953 inhdr->status = (uint8_t)err;
958 static __rte_always_inline struct vhost_virtqueue *
959 vhost_crypto_finalize_one_request(struct rte_crypto_op *op,
960 struct vhost_virtqueue *old_vq)
962 struct rte_mbuf *m_src = op->sym->m_src;
963 struct rte_mbuf *m_dst = op->sym->m_dst;
964 struct vhost_crypto_data_req *vc_req = RTE_PTR_ADD(m_src,
965 sizeof(struct rte_mbuf));
969 if (unlikely(!vc_req)) {
970 VC_LOG_ERR("Failed to retrieve vc_req");
974 if (old_vq && (vc_req->vq != old_vq))
977 desc_idx = vc_req->desc_idx;
979 if (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS))
980 vc_req->inhdr->status = VIRTIO_CRYPTO_ERR;
982 if (vc_req->zero_copy == 0) {
983 ret = write_back_data(op, vc_req);
984 if (unlikely(ret != 0))
985 vc_req->inhdr->status = VIRTIO_CRYPTO_ERR;
989 vc_req->vq->used->ring[desc_idx].id = desc_idx;
990 vc_req->vq->used->ring[desc_idx].len = vc_req->len;
992 rte_mempool_put(m_dst->pool, (void *)m_dst);
993 rte_mempool_put(m_src->pool, (void *)m_src);
998 static __rte_always_inline uint16_t
999 vhost_crypto_complete_one_vm_requests(struct rte_crypto_op **ops,
1000 uint16_t nb_ops, int *callfd)
1002 uint16_t processed = 1;
1003 struct vhost_virtqueue *vq, *tmp_vq;
1005 if (unlikely(nb_ops == 0))
1008 vq = vhost_crypto_finalize_one_request(ops[0], NULL);
1009 if (unlikely(vq == NULL))
1013 while ((processed < nb_ops)) {
1014 tmp_vq = vhost_crypto_finalize_one_request(ops[processed],
1017 if (unlikely(vq != tmp_vq))
1023 *callfd = vq->callfd;
1025 *(volatile uint16_t *)&vq->used->idx += processed;