X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_vhost%2Fvhost_crypto.c;h=0edf12d528176e4b40e6bdb0f947853ffbce2fed;hb=7f9f46d6cef5b03681a3935b9a18378e08ca6f62;hp=c592b7a308eae355624a0163c24e8d5cedf952f8;hpb=a52ac8ec27fe10b67e479fdfa43f9ca7ad033b23;p=dpdk.git diff --git a/lib/librte_vhost/vhost_crypto.c b/lib/librte_vhost/vhost_crypto.c index c592b7a308..0edf12d528 100644 --- a/lib/librte_vhost/vhost_crypto.c +++ b/lib/librte_vhost/vhost_crypto.c @@ -198,6 +198,8 @@ struct vhost_crypto { struct rte_hash *session_map; struct rte_mempool *mbuf_pool; struct rte_mempool *sess_pool; + struct rte_mempool *sess_priv_pool; + struct rte_mempool *wb_pool; /** DPDK cryptodev ID */ uint8_t cid; @@ -215,13 +217,20 @@ struct vhost_crypto { uint8_t option; } __rte_cache_aligned; +struct vhost_crypto_writeback_data { + uint8_t *src; + uint8_t *dst; + uint64_t len; + struct vhost_crypto_writeback_data *next; +}; + struct vhost_crypto_data_req { struct vring_desc *head; struct virtio_net *dev; struct virtio_crypto_inhdr *inhdr; struct vhost_virtqueue *vq; - struct vring_desc *wb_desc; - uint16_t wb_len; + struct vhost_crypto_writeback_data *wb; + struct rte_mempool *wb_pool; uint16_t desc_idx; uint16_t len; uint16_t zero_copy; @@ -238,7 +247,7 @@ transform_cipher_param(struct rte_crypto_sym_xform *xform, return ret; xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER; - xform->cipher.algo = (uint32_t)ret; + xform->cipher.algo = (enum rte_crypto_cipher_algorithm)ret; xform->cipher.key.length = param->cipher_key_len; if (xform->cipher.key.length > 0) xform->cipher.key.data = param->cipher_key_buf; @@ -288,7 +297,7 @@ transform_chain_param(struct rte_crypto_sym_xform *xforms, if (unlikely(ret < 0)) return ret; xform_cipher->type = RTE_CRYPTO_SYM_XFORM_CIPHER; - xform_cipher->cipher.algo = (uint32_t)ret; + xform_cipher->cipher.algo = (enum rte_crypto_cipher_algorithm)ret; xform_cipher->cipher.key.length = param->cipher_key_len; xform_cipher->cipher.key.data = param->cipher_key_buf; ret = get_iv_len(xform_cipher->cipher.algo); @@ -302,7 +311,7 @@ transform_chain_param(struct rte_crypto_sym_xform *xforms, ret = auth_algo_transform(param->hash_algo); if (unlikely(ret < 0)) return ret; - xform_auth->auth.algo = (uint32_t)ret; + xform_auth->auth.algo = (enum rte_crypto_auth_algorithm)ret; xform_auth->auth.digest_length = param->digest_len; xform_auth->auth.key.length = param->auth_key_len; xform_auth->auth.key.data = param->auth_key_buf; @@ -361,7 +370,7 @@ vhost_crypto_create_sess(struct vhost_crypto *vcrypto, } if (rte_cryptodev_sym_session_init(vcrypto->cid, session, &xform1, - vcrypto->sess_pool) < 0) { + vcrypto->sess_priv_pool) < 0) { VC_LOG_ERR("Failed to initialize session"); sess_param->session_id = -VIRTIO_CRYPTO_ERR; return; @@ -425,44 +434,56 @@ vhost_crypto_close_sess(struct vhost_crypto *vcrypto, uint64_t session_id) return 0; } -static enum vh_result +static enum rte_vhost_msg_result vhost_crypto_msg_post_handler(int vid, void *msg) { struct virtio_net *dev = get_device(vid); struct vhost_crypto *vcrypto; VhostUserMsg *vmsg = msg; - enum vh_result ret = VH_RESULT_OK; + enum rte_vhost_msg_result ret = RTE_VHOST_MSG_RESULT_OK; if (dev == NULL) { VC_LOG_ERR("Invalid vid %i", vid); - return VH_RESULT_ERR; + return RTE_VHOST_MSG_RESULT_ERR; } vcrypto = dev->extern_data; if (vcrypto == NULL) { VC_LOG_ERR("Cannot find required data, is it initialized?"); - return VH_RESULT_ERR; + return RTE_VHOST_MSG_RESULT_ERR; } - if (vmsg->request.master == VHOST_USER_CRYPTO_CREATE_SESS) { + switch (vmsg->request.master) { + case VHOST_USER_CRYPTO_CREATE_SESS: vhost_crypto_create_sess(vcrypto, &vmsg->payload.crypto_session); - ret = VH_RESULT_REPLY; - } else if (vmsg->request.master == VHOST_USER_CRYPTO_CLOSE_SESS) { + vmsg->fd_num = 0; + ret = RTE_VHOST_MSG_RESULT_REPLY; + break; + case VHOST_USER_CRYPTO_CLOSE_SESS: if (vhost_crypto_close_sess(vcrypto, vmsg->payload.u64)) - ret = VH_RESULT_ERR; + ret = RTE_VHOST_MSG_RESULT_ERR; + break; + default: + ret = RTE_VHOST_MSG_RESULT_NOT_HANDLED; + break; } return ret; } static __rte_always_inline struct vring_desc * -find_write_desc(struct vring_desc *head, struct vring_desc *desc) +find_write_desc(struct vring_desc *head, struct vring_desc *desc, + uint32_t *nb_descs, uint32_t vq_size) { if (desc->flags & VRING_DESC_F_WRITE) return desc; while (desc->flags & VRING_DESC_F_NEXT) { + if (unlikely(*nb_descs == 0 || desc->next >= vq_size)) + return NULL; + (*nb_descs)--; + desc = &head[desc->next]; if (desc->flags & VRING_DESC_F_WRITE) return desc; @@ -472,13 +493,18 @@ find_write_desc(struct vring_desc *head, struct vring_desc *desc) } static struct virtio_crypto_inhdr * -reach_inhdr(struct vhost_crypto_data_req *vc_req, struct vring_desc *desc) +reach_inhdr(struct vhost_crypto_data_req *vc_req, struct vring_desc *desc, + uint32_t *nb_descs, uint32_t vq_size) { uint64_t dlen; struct virtio_crypto_inhdr *inhdr; - while (desc->flags & VRING_DESC_F_NEXT) + while (desc->flags & VRING_DESC_F_NEXT) { + if (unlikely(*nb_descs == 0 || desc->next >= vq_size)) + return NULL; + (*nb_descs)--; desc = &vc_req->head[desc->next]; + } dlen = desc->len; inhdr = IOVA_TO_VVA(struct virtio_crypto_inhdr *, vc_req, desc->addr, @@ -491,32 +517,55 @@ reach_inhdr(struct vhost_crypto_data_req *vc_req, struct vring_desc *desc) static __rte_always_inline int move_desc(struct vring_desc *head, struct vring_desc **cur_desc, - uint32_t size) + uint32_t size, uint32_t *nb_descs, uint32_t vq_size) { struct vring_desc *desc = *cur_desc; - int left = size; - - rte_prefetch0(&head[desc->next]); - left -= desc->len; + int left = size - desc->len; while ((desc->flags & VRING_DESC_F_NEXT) && left > 0) { + (*nb_descs)--; + if (unlikely(*nb_descs == 0 || desc->next >= vq_size)) + return -1; + desc = &head[desc->next]; rte_prefetch0(&head[desc->next]); left -= desc->len; } - if (unlikely(left > 0)) { - VC_LOG_ERR("Incorrect virtio descriptor"); + if (unlikely(left > 0)) return -1; + + if (unlikely(*nb_descs == 0)) + *cur_desc = NULL; + else { + if (unlikely(desc->next >= vq_size)) + return -1; + *cur_desc = &head[desc->next]; } - *cur_desc = &head[desc->next]; return 0; } +static __rte_always_inline void * +get_data_ptr(struct vhost_crypto_data_req *vc_req, struct vring_desc *cur_desc, + uint8_t perm) +{ + void *data; + uint64_t dlen = cur_desc->len; + + data = IOVA_TO_VVA(void *, vc_req, cur_desc->addr, &dlen, perm); + if (unlikely(!data || dlen != cur_desc->len)) { + VC_LOG_ERR("Failed to map object"); + return NULL; + } + + return data; +} + static int copy_data(void *dst_data, struct vhost_crypto_data_req *vc_req, - struct vring_desc **cur_desc, uint32_t size) + struct vring_desc **cur_desc, uint32_t size, + uint32_t *nb_descs, uint32_t vq_size) { struct vring_desc *desc = *cur_desc; uint64_t remain, addr, dlen, len; @@ -525,15 +574,12 @@ copy_data(void *dst_data, struct vhost_crypto_data_req *vc_req, uint8_t *src; int left = size; - rte_prefetch0(&vc_req->head[desc->next]); to_copy = RTE_MIN(desc->len, (uint32_t)left); dlen = to_copy; src = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen, VHOST_ACCESS_RO); - if (unlikely(!src || !dlen)) { - VC_LOG_ERR("Failed to map descriptor"); + if (unlikely(!src || !dlen)) return -1; - } rte_memcpy((uint8_t *)data, src, dlen); data += dlen; @@ -561,6 +607,12 @@ copy_data(void *dst_data, struct vhost_crypto_data_req *vc_req, left -= to_copy; while ((desc->flags & VRING_DESC_F_NEXT) && left > 0) { + if (unlikely(*nb_descs == 0 || desc->next >= vq_size)) { + VC_LOG_ERR("Invalid descriptors"); + return -1; + } + (*nb_descs)--; + desc = &vc_req->head[desc->next]; rte_prefetch0(&vc_req->head[desc->next]); to_copy = RTE_MIN(desc->len, (uint32_t)left); @@ -603,95 +655,200 @@ copy_data(void *dst_data, struct vhost_crypto_data_req *vc_req, return -1; } - *cur_desc = &vc_req->head[desc->next]; + if (unlikely(*nb_descs == 0)) + *cur_desc = NULL; + else { + if (unlikely(desc->next >= vq_size)) + return -1; + *cur_desc = &vc_req->head[desc->next]; + } return 0; } -static __rte_always_inline void * -get_data_ptr(struct vhost_crypto_data_req *vc_req, struct vring_desc **cur_desc, - uint32_t size, uint8_t perm) +static void +write_back_data(struct vhost_crypto_data_req *vc_req) { - void *data; - uint64_t dlen = (*cur_desc)->len; + struct vhost_crypto_writeback_data *wb_data = vc_req->wb, *wb_last; - data = IOVA_TO_VVA(void *, vc_req, (*cur_desc)->addr, &dlen, perm); - if (unlikely(!data || dlen != (*cur_desc)->len)) { - VC_LOG_ERR("Failed to map object"); - return NULL; + while (wb_data) { + rte_memcpy(wb_data->dst, wb_data->src, wb_data->len); + wb_last = wb_data; + wb_data = wb_data->next; + rte_mempool_put(vc_req->wb_pool, wb_last); } +} - if (unlikely(move_desc(vc_req->head, cur_desc, size) < 0)) - return NULL; +static void +free_wb_data(struct vhost_crypto_writeback_data *wb_data, + struct rte_mempool *mp) +{ + while (wb_data->next != NULL) + free_wb_data(wb_data->next, mp); - return data; + rte_mempool_put(mp, wb_data); } -static int -write_back_data(struct rte_crypto_op *op, struct vhost_crypto_data_req *vc_req) +/** + * The function will allocate a vhost_crypto_writeback_data linked list + * containing the source and destination data pointers for the write back + * operation after dequeued from Cryptodev PMD queues. + * + * @param vc_req + * The vhost crypto data request pointer + * @param cur_desc + * The pointer of the current in use descriptor pointer. The content of + * cur_desc is expected to be updated after the function execution. + * @param end_wb_data + * The last write back data element to be returned. It is used only in cipher + * and hash chain operations. + * @param src + * The source data pointer + * @param offset + * The offset to both source and destination data. For source data the offset + * is the number of bytes between src and start point of cipher operation. For + * destination data the offset is the number of bytes from *cur_desc->addr + * to the point where the src will be written to. + * @param write_back_len + * The size of the write back length. + * @return + * The pointer to the start of the write back data linked list. + */ +static struct vhost_crypto_writeback_data * +prepare_write_back_data(struct vhost_crypto_data_req *vc_req, + struct vring_desc **cur_desc, + struct vhost_crypto_writeback_data **end_wb_data, + uint8_t *src, + uint32_t offset, + uint64_t write_back_len, + uint32_t *nb_descs, uint32_t vq_size) { - struct rte_mbuf *mbuf = op->sym->m_dst; - struct vring_desc *head = vc_req->head; - struct vring_desc *desc = vc_req->wb_desc; - int left = vc_req->wb_len; - uint32_t to_write; - uint8_t *src_data = mbuf->buf_addr, *dst; + struct vhost_crypto_writeback_data *wb_data, *head; + struct vring_desc *desc = *cur_desc; uint64_t dlen; + uint8_t *dst; + int ret; - rte_prefetch0(&head[desc->next]); - to_write = RTE_MIN(desc->len, (uint32_t)left); - dlen = desc->len; - dst = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen, - VHOST_ACCESS_RW); - if (unlikely(!dst || dlen != desc->len)) { - VC_LOG_ERR("Failed to map descriptor"); - return -1; + ret = rte_mempool_get(vc_req->wb_pool, (void **)&head); + if (unlikely(ret < 0)) { + VC_LOG_ERR("no memory"); + goto error_exit; } - rte_memcpy(dst, src_data, to_write); - left -= to_write; - src_data += to_write; + wb_data = head; - while ((desc->flags & VRING_DESC_F_NEXT) && left > 0) { - desc = &head[desc->next]; - rte_prefetch0(&head[desc->next]); - to_write = RTE_MIN(desc->len, (uint32_t)left); + if (likely(desc->len > offset)) { + wb_data->src = src + offset; dlen = desc->len; - dst = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen, - VHOST_ACCESS_RW); + dst = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, + &dlen, VHOST_ACCESS_RW) + offset; if (unlikely(!dst || dlen != desc->len)) { VC_LOG_ERR("Failed to map descriptor"); - return -1; + goto error_exit; + } + + wb_data->dst = dst; + wb_data->len = desc->len - offset; + write_back_len -= wb_data->len; + src += offset + wb_data->len; + offset = 0; + + if (unlikely(write_back_len)) { + ret = rte_mempool_get(vc_req->wb_pool, + (void **)&(wb_data->next)); + if (unlikely(ret < 0)) { + VC_LOG_ERR("no memory"); + goto error_exit; + } + + wb_data = wb_data->next; + } else + wb_data->next = NULL; + } else + offset -= desc->len; + + while (write_back_len) { + if (unlikely(*nb_descs == 0 || desc->next >= vq_size)) { + VC_LOG_ERR("Invalid descriptors"); + goto error_exit; + } + (*nb_descs)--; + + desc = &vc_req->head[desc->next]; + if (unlikely(!(desc->flags & VRING_DESC_F_WRITE))) { + VC_LOG_ERR("incorrect descriptor"); + goto error_exit; + } + + if (desc->len <= offset) { + offset -= desc->len; + continue; + } + + dlen = desc->len; + dst = IOVA_TO_VVA(uint8_t *, vc_req, desc->addr, &dlen, + VHOST_ACCESS_RW) + offset; + if (unlikely(dst == NULL || dlen != desc->len)) { + VC_LOG_ERR("Failed to map descriptor"); + goto error_exit; } - rte_memcpy(dst, src_data, to_write); - left -= to_write; - src_data += to_write; + wb_data->src = src; + wb_data->dst = dst; + wb_data->len = RTE_MIN(desc->len - offset, write_back_len); + write_back_len -= wb_data->len; + src += wb_data->len; + offset = 0; + + if (write_back_len) { + ret = rte_mempool_get(vc_req->wb_pool, + (void **)&(wb_data->next)); + if (unlikely(ret < 0)) { + VC_LOG_ERR("no memory"); + goto error_exit; + } + + wb_data = wb_data->next; + } else + wb_data->next = NULL; } - if (unlikely(left < 0)) { - VC_LOG_ERR("Incorrect virtio descriptor"); - return -1; + if (unlikely(*nb_descs == 0)) + *cur_desc = NULL; + else { + if (unlikely(desc->next >= vq_size)) + goto error_exit; + *cur_desc = &vc_req->head[desc->next]; } - return 0; + *end_wb_data = wb_data; + + return head; + +error_exit: + if (head) + free_wb_data(head, vc_req->wb_pool); + + return NULL; } static uint8_t prepare_sym_cipher_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, struct vhost_crypto_data_req *vc_req, struct virtio_crypto_cipher_data_req *cipher, - struct vring_desc *cur_desc) + struct vring_desc *cur_desc, + uint32_t *nb_descs, uint32_t vq_size) { struct vring_desc *desc = cur_desc; + struct vhost_crypto_writeback_data *ewb = NULL; struct rte_mbuf *m_src = op->sym->m_src, *m_dst = op->sym->m_dst; uint8_t *iv_data = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET); uint8_t ret = 0; /* prepare */ /* iv */ - if (unlikely(copy_data(iv_data, vc_req, &desc, - cipher->para.iv_len) < 0)) { + if (unlikely(copy_data(iv_data, vc_req, &desc, cipher->para.iv_len, + nb_descs, vq_size) < 0)) { ret = VIRTIO_CRYPTO_BADMSG; goto error_exit; } @@ -702,16 +859,26 @@ prepare_sym_cipher_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE: m_src->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr, cipher->para.src_data_len); - m_src->buf_addr = get_data_ptr(vc_req, &desc, - cipher->para.src_data_len, VHOST_ACCESS_RO); + m_src->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RO); if (unlikely(m_src->buf_iova == 0 || m_src->buf_addr == NULL)) { VC_LOG_ERR("zero_copy may fail due to cross page data"); ret = VIRTIO_CRYPTO_ERR; goto error_exit; } + + if (unlikely(move_desc(vc_req->head, &desc, + cipher->para.src_data_len, nb_descs, + vq_size) < 0)) { + VC_LOG_ERR("Incorrect descriptor"); + ret = VIRTIO_CRYPTO_ERR; + goto error_exit; + } + break; case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE: + vc_req->wb_pool = vcrypto->wb_pool; + if (unlikely(cipher->para.src_data_len > RTE_MBUF_DEFAULT_BUF_SIZE)) { VC_LOG_ERR("Not enough space to do data copy"); @@ -719,8 +886,8 @@ prepare_sym_cipher_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, goto error_exit; } if (unlikely(copy_data(rte_pktmbuf_mtod(m_src, uint8_t *), - vc_req, &desc, cipher->para.src_data_len) - < 0)) { + vc_req, &desc, cipher->para.src_data_len, + nb_descs, vq_size) < 0)) { ret = VIRTIO_CRYPTO_BADMSG; goto error_exit; } @@ -731,7 +898,7 @@ prepare_sym_cipher_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, } /* dst */ - desc = find_write_desc(vc_req->head, desc); + desc = find_write_desc(vc_req->head, desc, nb_descs, vq_size); if (unlikely(!desc)) { VC_LOG_ERR("Cannot find write location"); ret = VIRTIO_CRYPTO_BADMSG; @@ -742,24 +909,32 @@ prepare_sym_cipher_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE: m_dst->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr, cipher->para.dst_data_len); - m_dst->buf_addr = get_data_ptr(vc_req, &desc, - cipher->para.dst_data_len, VHOST_ACCESS_RW); + m_dst->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RW); if (unlikely(m_dst->buf_iova == 0 || m_dst->buf_addr == NULL)) { VC_LOG_ERR("zero_copy may fail due to cross page data"); ret = VIRTIO_CRYPTO_ERR; goto error_exit; } + if (unlikely(move_desc(vc_req->head, &desc, + cipher->para.dst_data_len, + nb_descs, vq_size) < 0)) { + VC_LOG_ERR("Incorrect descriptor"); + ret = VIRTIO_CRYPTO_ERR; + goto error_exit; + } + m_dst->data_len = cipher->para.dst_data_len; break; case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE: - vc_req->wb_desc = desc; - vc_req->wb_len = cipher->para.dst_data_len; - if (unlikely(move_desc(vc_req->head, &desc, - vc_req->wb_len) < 0)) { + vc_req->wb = prepare_write_back_data(vc_req, &desc, &ewb, + rte_pktmbuf_mtod(m_src, uint8_t *), 0, + cipher->para.dst_data_len, nb_descs, vq_size); + if (unlikely(vc_req->wb == NULL)) { ret = VIRTIO_CRYPTO_ERR; goto error_exit; } + break; default: ret = VIRTIO_CRYPTO_BADMSG; @@ -773,7 +948,7 @@ prepare_sym_cipher_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, op->sym->cipher.data.offset = 0; op->sym->cipher.data.length = cipher->para.src_data_len; - vc_req->inhdr = get_data_ptr(vc_req, &desc, INHDR_LEN, VHOST_ACCESS_WO); + vc_req->inhdr = get_data_ptr(vc_req, desc, VHOST_ACCESS_WO); if (unlikely(vc_req->inhdr == NULL)) { ret = VIRTIO_CRYPTO_BADMSG; goto error_exit; @@ -785,6 +960,9 @@ prepare_sym_cipher_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, return 0; error_exit: + if (vc_req->wb) + free_wb_data(vc_req->wb, vc_req->wb_pool); + vc_req->len = INHDR_LEN; return ret; } @@ -793,9 +971,11 @@ static uint8_t prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, struct vhost_crypto_data_req *vc_req, struct virtio_crypto_alg_chain_data_req *chain, - struct vring_desc *cur_desc) + struct vring_desc *cur_desc, + uint32_t *nb_descs, uint32_t vq_size) { - struct vring_desc *desc = cur_desc; + struct vring_desc *desc = cur_desc, *digest_desc; + struct vhost_crypto_writeback_data *ewb = NULL, *ewb2 = NULL; struct rte_mbuf *m_src = op->sym->m_src, *m_dst = op->sym->m_dst; uint8_t *iv_data = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET); uint32_t digest_offset; @@ -805,27 +985,37 @@ prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, /* prepare */ /* iv */ if (unlikely(copy_data(iv_data, vc_req, &desc, - chain->para.iv_len) < 0)) { + chain->para.iv_len, nb_descs, vq_size) < 0)) { ret = VIRTIO_CRYPTO_BADMSG; goto error_exit; } m_src->data_len = chain->para.src_data_len; - m_dst->data_len = chain->para.dst_data_len; switch (vcrypto->option) { case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE: + m_dst->data_len = chain->para.dst_data_len; + m_src->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr, chain->para.src_data_len); - m_src->buf_addr = get_data_ptr(vc_req, &desc, - chain->para.src_data_len, VHOST_ACCESS_RO); + m_src->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RO); if (unlikely(m_src->buf_iova == 0 || m_src->buf_addr == NULL)) { VC_LOG_ERR("zero_copy may fail due to cross page data"); ret = VIRTIO_CRYPTO_ERR; goto error_exit; } + + if (unlikely(move_desc(vc_req->head, &desc, + chain->para.src_data_len, + nb_descs, vq_size) < 0)) { + VC_LOG_ERR("Incorrect descriptor"); + ret = VIRTIO_CRYPTO_ERR; + goto error_exit; + } break; case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE: + vc_req->wb_pool = vcrypto->wb_pool; + if (unlikely(chain->para.src_data_len > RTE_MBUF_DEFAULT_BUF_SIZE)) { VC_LOG_ERR("Not enough space to do data copy"); @@ -833,10 +1023,12 @@ prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, goto error_exit; } if (unlikely(copy_data(rte_pktmbuf_mtod(m_src, uint8_t *), - vc_req, &desc, chain->para.src_data_len)) < 0) { + vc_req, &desc, chain->para.src_data_len, + nb_descs, vq_size)) < 0) { ret = VIRTIO_CRYPTO_BADMSG; goto error_exit; } + break; default: ret = VIRTIO_CRYPTO_BADMSG; @@ -844,7 +1036,7 @@ prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, } /* dst */ - desc = find_write_desc(vc_req->head, desc); + desc = find_write_desc(vc_req->head, desc, nb_descs, vq_size); if (unlikely(!desc)) { VC_LOG_ERR("Cannot find write location"); ret = VIRTIO_CRYPTO_BADMSG; @@ -855,46 +1047,75 @@ prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE: m_dst->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr, chain->para.dst_data_len); - m_dst->buf_addr = get_data_ptr(vc_req, &desc, - chain->para.dst_data_len, VHOST_ACCESS_RW); + m_dst->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RW); if (unlikely(m_dst->buf_iova == 0 || m_dst->buf_addr == NULL)) { VC_LOG_ERR("zero_copy may fail due to cross page data"); ret = VIRTIO_CRYPTO_ERR; goto error_exit; } + if (unlikely(move_desc(vc_req->head, &desc, + chain->para.dst_data_len, + nb_descs, vq_size) < 0)) { + VC_LOG_ERR("Incorrect descriptor"); + ret = VIRTIO_CRYPTO_ERR; + goto error_exit; + } + op->sym->auth.digest.phys_addr = gpa_to_hpa(vcrypto->dev, desc->addr, chain->para.hash_result_len); - op->sym->auth.digest.data = get_data_ptr(vc_req, &desc, - chain->para.hash_result_len, VHOST_ACCESS_RW); + op->sym->auth.digest.data = get_data_ptr(vc_req, desc, + VHOST_ACCESS_RW); if (unlikely(op->sym->auth.digest.phys_addr == 0)) { VC_LOG_ERR("zero_copy may fail due to cross page data"); ret = VIRTIO_CRYPTO_ERR; goto error_exit; } + + if (unlikely(move_desc(vc_req->head, &desc, + chain->para.hash_result_len, + nb_descs, vq_size) < 0)) { + VC_LOG_ERR("Incorrect descriptor"); + ret = VIRTIO_CRYPTO_ERR; + goto error_exit; + } + break; case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE: - digest_offset = m_dst->data_len; - digest_addr = rte_pktmbuf_mtod_offset(m_dst, void *, - digest_offset); + vc_req->wb = prepare_write_back_data(vc_req, &desc, &ewb, + rte_pktmbuf_mtod(m_src, uint8_t *), + chain->para.cipher_start_src_offset, + chain->para.dst_data_len - + chain->para.cipher_start_src_offset, + nb_descs, vq_size); + if (unlikely(vc_req->wb == NULL)) { + ret = VIRTIO_CRYPTO_ERR; + goto error_exit; + } - vc_req->wb_desc = desc; - vc_req->wb_len = m_dst->data_len + chain->para.hash_result_len; + digest_offset = m_src->data_len; + digest_addr = rte_pktmbuf_mtod_offset(m_src, void *, + digest_offset); + digest_desc = desc; - if (unlikely(move_desc(vc_req->head, &desc, - chain->para.dst_data_len) < 0)) { - ret = VIRTIO_CRYPTO_BADMSG; + /** create a wb_data for digest */ + ewb->next = prepare_write_back_data(vc_req, &desc, &ewb2, + digest_addr, 0, chain->para.hash_result_len, + nb_descs, vq_size); + if (unlikely(ewb->next == NULL)) { + ret = VIRTIO_CRYPTO_ERR; goto error_exit; } - if (unlikely(copy_data(digest_addr, vc_req, &desc, - chain->para.hash_result_len)) < 0) { + if (unlikely(copy_data(digest_addr, vc_req, &digest_desc, + chain->para.hash_result_len, + nb_descs, vq_size) < 0)) { ret = VIRTIO_CRYPTO_BADMSG; goto error_exit; } op->sym->auth.digest.data = digest_addr; - op->sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m_dst, + op->sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m_src, digest_offset); break; default: @@ -903,7 +1124,7 @@ prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, } /* record inhdr */ - vc_req->inhdr = get_data_ptr(vc_req, &desc, INHDR_LEN, VHOST_ACCESS_WO); + vc_req->inhdr = get_data_ptr(vc_req, desc, VHOST_ACCESS_WO); if (unlikely(vc_req->inhdr == NULL)) { ret = VIRTIO_CRYPTO_BADMSG; goto error_exit; @@ -926,6 +1147,8 @@ prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op, return 0; error_exit: + if (vc_req->wb) + free_wb_data(vc_req->wb, vc_req->wb_pool); vc_req->len = INHDR_LEN; return ret; } @@ -945,6 +1168,7 @@ vhost_crypto_process_one_req(struct vhost_crypto *vcrypto, struct vring_desc *desc = NULL; uint64_t session_id; uint64_t dlen; + uint32_t nb_descs = vq->size; int err = 0; vc_req->desc_idx = desc_idx; @@ -953,6 +1177,10 @@ vhost_crypto_process_one_req(struct vhost_crypto *vcrypto, if (likely(head->flags & VRING_DESC_F_INDIRECT)) { dlen = head->len; + nb_descs = dlen / sizeof(struct vring_desc); + /* drop invalid descriptors */ + if (unlikely(nb_descs > vq->size)) + return -1; desc = IOVA_TO_VVA(struct vring_desc *, vc_req, head->addr, &dlen, VHOST_ACCESS_RO); if (unlikely(!desc || dlen != head->len)) @@ -966,7 +1194,7 @@ vhost_crypto_process_one_req(struct vhost_crypto *vcrypto, vc_req->head = head; vc_req->zero_copy = vcrypto->option; - req = get_data_ptr(vc_req, &desc, sizeof(*req), VHOST_ACCESS_RO); + req = get_data_ptr(vc_req, desc, VHOST_ACCESS_RO); if (unlikely(req == NULL)) { switch (vcrypto->option) { case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE: @@ -975,8 +1203,8 @@ vhost_crypto_process_one_req(struct vhost_crypto *vcrypto, goto error_exit; case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE: req = &tmp_req; - if (unlikely(copy_data(req, vc_req, &desc, sizeof(*req)) - < 0)) { + if (unlikely(copy_data(req, vc_req, &desc, sizeof(*req), + &nb_descs, vq->size) < 0)) { err = VIRTIO_CRYPTO_BADMSG; VC_LOG_ERR("Invalid descriptor"); goto error_exit; @@ -987,6 +1215,12 @@ vhost_crypto_process_one_req(struct vhost_crypto *vcrypto, VC_LOG_ERR("Invalid option"); goto error_exit; } + } else { + if (unlikely(move_desc(vc_req->head, &desc, + sizeof(*req), &nb_descs, vq->size) < 0)) { + VC_LOG_ERR("Incorrect descriptor"); + goto error_exit; + } } switch (req->header.opcode) { @@ -1024,11 +1258,13 @@ vhost_crypto_process_one_req(struct vhost_crypto *vcrypto, break; case VIRTIO_CRYPTO_SYM_OP_CIPHER: err = prepare_sym_cipher_op(vcrypto, op, vc_req, - &req->u.sym_req.u.cipher, desc); + &req->u.sym_req.u.cipher, desc, + &nb_descs, vq->size); break; case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING: err = prepare_sym_chain_op(vcrypto, op, vc_req, - &req->u.sym_req.u.chain, desc); + &req->u.sym_req.u.chain, desc, + &nb_descs, vq->size); break; } if (unlikely(err != 0)) { @@ -1046,7 +1282,7 @@ vhost_crypto_process_one_req(struct vhost_crypto *vcrypto, error_exit: - inhdr = reach_inhdr(vc_req, desc); + inhdr = reach_inhdr(vc_req, desc, &nb_descs, vq->size); if (likely(inhdr != NULL)) inhdr->status = (uint8_t)err; @@ -1061,7 +1297,6 @@ vhost_crypto_finalize_one_request(struct rte_crypto_op *op, struct rte_mbuf *m_dst = op->sym->m_dst; struct vhost_crypto_data_req *vc_req = rte_mbuf_to_priv(m_src); uint16_t desc_idx; - int ret = 0; if (unlikely(!vc_req)) { VC_LOG_ERR("Failed to retrieve vc_req"); @@ -1076,19 +1311,18 @@ vhost_crypto_finalize_one_request(struct rte_crypto_op *op, if (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS)) vc_req->inhdr->status = VIRTIO_CRYPTO_ERR; else { - if (vc_req->zero_copy == 0) { - ret = write_back_data(op, vc_req); - if (unlikely(ret != 0)) - vc_req->inhdr->status = VIRTIO_CRYPTO_ERR; - } + if (vc_req->zero_copy == 0) + write_back_data(vc_req); } vc_req->vq->used->ring[desc_idx].id = desc_idx; vc_req->vq->used->ring[desc_idx].len = vc_req->len; - rte_mempool_put(m_dst->pool, (void *)m_dst); rte_mempool_put(m_src->pool, (void *)m_src); + if (m_dst) + rte_mempool_put(m_dst->pool, (void *)m_dst); + return vc_req->vq; } @@ -1126,7 +1360,9 @@ vhost_crypto_complete_one_vm_requests(struct rte_crypto_op **ops, int __rte_experimental rte_vhost_crypto_create(int vid, uint8_t cryptodev_id, - struct rte_mempool *sess_pool, int socket_id) + struct rte_mempool *sess_pool, + struct rte_mempool *sess_priv_pool, + int socket_id) { struct virtio_net *dev = get_device(vid); struct rte_hash_parameters params = {0}; @@ -1154,6 +1390,7 @@ rte_vhost_crypto_create(int vid, uint8_t cryptodev_id, } vcrypto->sess_pool = sess_pool; + vcrypto->sess_priv_pool = sess_priv_pool; vcrypto->cid = cryptodev_id; vcrypto->cache_session_id = UINT64_MAX; vcrypto->last_session_id = 1; @@ -1185,6 +1422,18 @@ rte_vhost_crypto_create(int vid, uint8_t cryptodev_id, goto error_exit; } + snprintf(name, 127, "WB_POOL_VM_%u", (uint32_t)vid); + vcrypto->wb_pool = rte_mempool_create(name, + VHOST_CRYPTO_MBUF_POOL_SIZE, + sizeof(struct vhost_crypto_writeback_data), + 128, 0, NULL, NULL, NULL, NULL, + rte_socket_id(), 0); + if (!vcrypto->wb_pool) { + VC_LOG_ERR("Failed to creath mempool"); + ret = -ENOMEM; + goto error_exit; + } + dev->extern_data = vcrypto; dev->extern_ops.pre_msg_handle = NULL; dev->extern_ops.post_msg_handle = vhost_crypto_msg_post_handler; @@ -1221,6 +1470,7 @@ rte_vhost_crypto_free(int vid) rte_hash_free(vcrypto->session_map); rte_mempool_free(vcrypto->mbuf_pool); + rte_mempool_free(vcrypto->wb_pool); rte_free(vcrypto); dev->extern_data = NULL; @@ -1256,11 +1506,30 @@ rte_vhost_crypto_set_zero_copy(int vid, enum rte_vhost_crypto_zero_copy option) if (vcrypto->option == (uint8_t)option) return 0; - if (!(rte_mempool_full(vcrypto->mbuf_pool))) { + if (!(rte_mempool_full(vcrypto->mbuf_pool)) || + !(rte_mempool_full(vcrypto->wb_pool))) { VC_LOG_ERR("Cannot update zero copy as mempool is not full"); return -EINVAL; } + if (option == RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE) { + char name[128]; + + snprintf(name, 127, "WB_POOL_VM_%u", (uint32_t)vid); + vcrypto->wb_pool = rte_mempool_create(name, + VHOST_CRYPTO_MBUF_POOL_SIZE, + sizeof(struct vhost_crypto_writeback_data), + 128, 0, NULL, NULL, NULL, NULL, + rte_socket_id(), 0); + if (!vcrypto->wb_pool) { + VC_LOG_ERR("Failed to creath mbuf pool"); + return -ENOMEM; + } + } else { + rte_mempool_free(vcrypto->wb_pool); + vcrypto->wb_pool = NULL; + } + vcrypto->option = (uint8_t)option; return 0; @@ -1276,9 +1545,8 @@ rte_vhost_crypto_fetch_requests(int vid, uint32_t qid, struct vhost_virtqueue *vq; uint16_t avail_idx; uint16_t start_idx; - uint16_t required; uint16_t count; - uint16_t i; + uint16_t i = 0; if (unlikely(dev == NULL)) { VC_LOG_ERR("Invalid vid %i", vid); @@ -1310,27 +1578,66 @@ rte_vhost_crypto_fetch_requests(int vid, uint32_t qid, /* for zero copy, we need 2 empty mbufs for src and dst, otherwise * we need only 1 mbuf as src and dst */ - required = count * 2; - if (unlikely(rte_mempool_get_bulk(vcrypto->mbuf_pool, (void **)mbufs, - required) < 0)) { - VC_LOG_ERR("Insufficient memory"); - return -ENOMEM; - } + switch (vcrypto->option) { + case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE: + if (unlikely(rte_mempool_get_bulk(vcrypto->mbuf_pool, + (void **)mbufs, count * 2) < 0)) { + VC_LOG_ERR("Insufficient memory"); + return -ENOMEM; + } - for (i = 0; i < count; i++) { - uint16_t used_idx = (start_idx + i) & (vq->size - 1); - uint16_t desc_idx = vq->avail->ring[used_idx]; - struct vring_desc *head = &vq->desc[desc_idx]; - struct rte_crypto_op *op = ops[i]; + for (i = 0; i < count; i++) { + uint16_t used_idx = (start_idx + i) & (vq->size - 1); + uint16_t desc_idx = vq->avail->ring[used_idx]; + struct vring_desc *head = &vq->desc[desc_idx]; + struct rte_crypto_op *op = ops[i]; - op->sym->m_src = mbufs[i * 2]; - op->sym->m_dst = mbufs[i * 2 + 1]; - op->sym->m_src->data_off = 0; - op->sym->m_dst->data_off = 0; + op->sym->m_src = mbufs[i * 2]; + op->sym->m_dst = mbufs[i * 2 + 1]; + op->sym->m_src->data_off = 0; + op->sym->m_dst->data_off = 0; + + if (unlikely(vhost_crypto_process_one_req(vcrypto, vq, + op, head, desc_idx)) < 0) + break; + } + + if (unlikely(i < count)) + rte_mempool_put_bulk(vcrypto->mbuf_pool, + (void **)&mbufs[i * 2], + (count - i) * 2); + + break; + + case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE: + if (unlikely(rte_mempool_get_bulk(vcrypto->mbuf_pool, + (void **)mbufs, count) < 0)) { + VC_LOG_ERR("Insufficient memory"); + return -ENOMEM; + } + + for (i = 0; i < count; i++) { + uint16_t used_idx = (start_idx + i) & (vq->size - 1); + uint16_t desc_idx = vq->avail->ring[used_idx]; + struct vring_desc *head = &vq->desc[desc_idx]; + struct rte_crypto_op *op = ops[i]; + + op->sym->m_src = mbufs[i]; + op->sym->m_dst = NULL; + op->sym->m_src->data_off = 0; + + if (unlikely(vhost_crypto_process_one_req(vcrypto, vq, + op, head, desc_idx) < 0)) + break; + } + + if (unlikely(i < count)) + rte_mempool_put_bulk(vcrypto->mbuf_pool, + (void **)&mbufs[i], + count - i); + + break; - if (unlikely(vhost_crypto_process_one_req(vcrypto, vq, op, head, - desc_idx)) < 0) - break; } vq->last_used_idx += i;