From 82adb12a1fce1a8a45da2f8d72a56ba1f640c9d6 Mon Sep 17 00:00:00 2001 From: Jay Zhou Date: Tue, 17 Apr 2018 17:23:21 +0800 Subject: [PATCH] crypto/virtio: support burst enqueue/dequeue This patch implements the functions of virtio_crypto_pkt_tx_burst() and virtio_crypto_pkt_rx_burst(). The encryption and decryption requests are placed in the data queue and are ultimately handled by the backend crypto accelerators. Signed-off-by: Jay Zhou Reviewed-by: Fan Zhang --- drivers/crypto/virtio/virtio_cryptodev.h | 2 + drivers/crypto/virtio/virtio_rxtx.c | 441 ++++++++++++++++++++++- 2 files changed, 433 insertions(+), 10 deletions(-) diff --git a/drivers/crypto/virtio/virtio_cryptodev.h b/drivers/crypto/virtio/virtio_cryptodev.h index 5eaa6f1f31..e402c0309a 100644 --- a/drivers/crypto/virtio/virtio_cryptodev.h +++ b/drivers/crypto/virtio/virtio_cryptodev.h @@ -16,6 +16,8 @@ #define NUM_ENTRY_VIRTIO_CRYPTO_OP 7 +extern uint8_t cryptodev_virtio_driver_id; + enum virtio_crypto_cmd_id { VIRTIO_CRYPTO_CMD_CIPHER = 0, VIRTIO_CRYPTO_CMD_AUTH = 1, diff --git a/drivers/crypto/virtio/virtio_rxtx.c b/drivers/crypto/virtio/virtio_rxtx.c index 3b038a76f8..4503928438 100644 --- a/drivers/crypto/virtio/virtio_rxtx.c +++ b/drivers/crypto/virtio/virtio_rxtx.c @@ -1,8 +1,353 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD. */ +#include + #include "virtqueue.h" #include "virtio_cryptodev.h" +#include "virtio_crypto_algs.h" + +static void +vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) +{ + struct vring_desc *dp, *dp_tail; + struct vq_desc_extra *dxp; + uint16_t desc_idx_last = desc_idx; + + dp = &vq->vq_ring.desc[desc_idx]; + dxp = &vq->vq_descx[desc_idx]; + vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs); + if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { + while (dp->flags & VRING_DESC_F_NEXT) { + desc_idx_last = dp->next; + dp = &vq->vq_ring.desc[dp->next]; + } + } + dxp->ndescs = 0; + + /* + * We must append the existing free chain, if any, to the end of + * newly freed chain. If the virtqueue was completely used, then + * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). + */ + if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) { + vq->vq_desc_head_idx = desc_idx; + } else { + dp_tail = &vq->vq_ring.desc[vq->vq_desc_tail_idx]; + dp_tail->next = desc_idx; + } + + vq->vq_desc_tail_idx = desc_idx_last; + dp->next = VQ_RING_DESC_CHAIN_END; +} + +static uint16_t +virtqueue_dequeue_burst_rx(struct virtqueue *vq, + struct rte_crypto_op **rx_pkts, uint16_t num) +{ + struct vring_used_elem *uep; + struct rte_crypto_op *cop; + uint16_t used_idx, desc_idx; + uint16_t i; + struct virtio_crypto_inhdr *inhdr; + struct virtio_crypto_op_cookie *op_cookie; + + /* Caller does the check */ + for (i = 0; i < num ; i++) { + used_idx = (uint16_t)(vq->vq_used_cons_idx + & (vq->vq_nentries - 1)); + uep = &vq->vq_ring.used->ring[used_idx]; + desc_idx = (uint16_t)uep->id; + cop = (struct rte_crypto_op *) + vq->vq_descx[desc_idx].crypto_op; + if (unlikely(cop == NULL)) { + VIRTIO_CRYPTO_RX_LOG_DBG("vring descriptor with no " + "mbuf cookie at %u", + vq->vq_used_cons_idx); + break; + } + + op_cookie = (struct virtio_crypto_op_cookie *) + vq->vq_descx[desc_idx].cookie; + inhdr = &(op_cookie->inhdr); + switch (inhdr->status) { + case VIRTIO_CRYPTO_OK: + cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + break; + case VIRTIO_CRYPTO_ERR: + cop->status = RTE_CRYPTO_OP_STATUS_ERROR; + vq->packets_received_failed++; + break; + case VIRTIO_CRYPTO_BADMSG: + cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + vq->packets_received_failed++; + break; + case VIRTIO_CRYPTO_NOTSUPP: + cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + vq->packets_received_failed++; + break; + case VIRTIO_CRYPTO_INVSESS: + cop->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + vq->packets_received_failed++; + break; + default: + break; + } + + vq->packets_received_total++; + + rx_pkts[i] = cop; + rte_mempool_put(vq->mpool, op_cookie); + + vq->vq_used_cons_idx++; + vq_ring_free_chain(vq, desc_idx); + vq->vq_descx[desc_idx].crypto_op = NULL; + } + + return i; +} + +static int +virtqueue_crypto_sym_pkt_header_arrange( + struct rte_crypto_op *cop, + struct virtio_crypto_op_data_req *data, + struct virtio_crypto_session *session) +{ + struct rte_crypto_sym_op *sym_op = cop->sym; + struct virtio_crypto_op_data_req *req_data = data; + struct virtio_crypto_op_ctrl_req *ctrl = &session->ctrl; + struct virtio_crypto_sym_create_session_req *sym_sess_req = + &ctrl->u.sym_create_session; + struct virtio_crypto_alg_chain_session_para *chain_para = + &sym_sess_req->u.chain.para; + struct virtio_crypto_cipher_session_para *cipher_para; + + req_data->header.session_id = session->session_id; + + switch (sym_sess_req->op_type) { + case VIRTIO_CRYPTO_SYM_OP_CIPHER: + req_data->u.sym_req.op_type = VIRTIO_CRYPTO_SYM_OP_CIPHER; + + cipher_para = &sym_sess_req->u.cipher.para; + if (cipher_para->op == VIRTIO_CRYPTO_OP_ENCRYPT) + req_data->header.opcode = VIRTIO_CRYPTO_CIPHER_ENCRYPT; + else + req_data->header.opcode = VIRTIO_CRYPTO_CIPHER_DECRYPT; + + req_data->u.sym_req.u.cipher.para.iv_len + = session->iv.length; + + req_data->u.sym_req.u.cipher.para.src_data_len = + (sym_op->cipher.data.length + + sym_op->cipher.data.offset); + req_data->u.sym_req.u.cipher.para.dst_data_len = + req_data->u.sym_req.u.cipher.para.src_data_len; + break; + case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING: + req_data->u.sym_req.op_type = + VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING; + + cipher_para = &chain_para->cipher_param; + if (cipher_para->op == VIRTIO_CRYPTO_OP_ENCRYPT) + req_data->header.opcode = VIRTIO_CRYPTO_CIPHER_ENCRYPT; + else + req_data->header.opcode = VIRTIO_CRYPTO_CIPHER_DECRYPT; + + req_data->u.sym_req.u.chain.para.iv_len = session->iv.length; + req_data->u.sym_req.u.chain.para.aad_len = session->aad.length; + + req_data->u.sym_req.u.chain.para.src_data_len = + (sym_op->cipher.data.length + + sym_op->cipher.data.offset); + req_data->u.sym_req.u.chain.para.dst_data_len = + req_data->u.sym_req.u.chain.para.src_data_len; + req_data->u.sym_req.u.chain.para.cipher_start_src_offset = + sym_op->cipher.data.offset; + req_data->u.sym_req.u.chain.para.len_to_cipher = + sym_op->cipher.data.length; + req_data->u.sym_req.u.chain.para.hash_start_src_offset = + sym_op->auth.data.offset; + req_data->u.sym_req.u.chain.para.len_to_hash = + sym_op->auth.data.length; + req_data->u.sym_req.u.chain.para.aad_len = + chain_para->aad_len; + + if (chain_para->hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN) + req_data->u.sym_req.u.chain.para.hash_result_len = + chain_para->u.hash_param.hash_result_len; + if (chain_para->hash_mode == + VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH) + req_data->u.sym_req.u.chain.para.hash_result_len = + chain_para->u.mac_param.hash_result_len; + break; + default: + return -1; + } + + return 0; +} + +static int +virtqueue_crypto_sym_enqueue_xmit( + struct virtqueue *txvq, + struct rte_crypto_op *cop) +{ + uint16_t idx = 0; + uint16_t num_entry; + uint16_t needed = 1; + uint16_t head_idx; + struct vq_desc_extra *dxp; + struct vring_desc *start_dp; + struct vring_desc *desc; + uint64_t indirect_op_data_req_phys_addr; + uint16_t req_data_len = sizeof(struct virtio_crypto_op_data_req); + uint32_t indirect_vring_addr_offset = req_data_len + + sizeof(struct virtio_crypto_inhdr); + struct rte_crypto_sym_op *sym_op = cop->sym; + struct virtio_crypto_session *session = + (struct virtio_crypto_session *)get_session_private_data( + cop->sym->session, cryptodev_virtio_driver_id); + struct virtio_crypto_op_data_req *op_data_req; + uint32_t hash_result_len = 0; + struct virtio_crypto_op_cookie *crypto_op_cookie; + struct virtio_crypto_alg_chain_session_para *para; + + if (unlikely(sym_op->m_src->nb_segs != 1)) + return -EMSGSIZE; + if (unlikely(txvq->vq_free_cnt == 0)) + return -ENOSPC; + if (unlikely(txvq->vq_free_cnt < needed)) + return -EMSGSIZE; + head_idx = txvq->vq_desc_head_idx; + if (unlikely(head_idx >= txvq->vq_nentries)) + return -EFAULT; + if (unlikely(session == NULL)) + return -EFAULT; + + dxp = &txvq->vq_descx[head_idx]; + + if (rte_mempool_get(txvq->mpool, &dxp->cookie)) { + VIRTIO_CRYPTO_TX_LOG_ERR("can not get cookie"); + return -EFAULT; + } + crypto_op_cookie = dxp->cookie; + indirect_op_data_req_phys_addr = + rte_mempool_virt2iova(crypto_op_cookie); + op_data_req = (struct virtio_crypto_op_data_req *)crypto_op_cookie; + + if (virtqueue_crypto_sym_pkt_header_arrange(cop, op_data_req, session)) + return -EFAULT; + + /* status is initialized to VIRTIO_CRYPTO_ERR */ + ((struct virtio_crypto_inhdr *) + ((uint8_t *)op_data_req + req_data_len))->status = + VIRTIO_CRYPTO_ERR; + + /* point to indirect vring entry */ + desc = (struct vring_desc *) + ((uint8_t *)op_data_req + indirect_vring_addr_offset); + for (idx = 0; idx < (NUM_ENTRY_VIRTIO_CRYPTO_OP - 1); idx++) + desc[idx].next = idx + 1; + desc[NUM_ENTRY_VIRTIO_CRYPTO_OP - 1].next = VQ_RING_DESC_CHAIN_END; + + idx = 0; + + /* indirect vring: first part, virtio_crypto_op_data_req */ + desc[idx].addr = indirect_op_data_req_phys_addr; + desc[idx].len = req_data_len; + desc[idx++].flags = VRING_DESC_F_NEXT; + + /* indirect vring: iv of cipher */ + if (session->iv.length) { + desc[idx].addr = cop->phys_addr + session->iv.offset; + desc[idx].len = session->iv.length; + desc[idx++].flags = VRING_DESC_F_NEXT; + } + + /* indirect vring: additional auth data */ + if (session->aad.length) { + desc[idx].addr = session->aad.phys_addr; + desc[idx].len = session->aad.length; + desc[idx++].flags = VRING_DESC_F_NEXT; + } + + /* indirect vring: src data */ + desc[idx].addr = rte_pktmbuf_mtophys_offset(sym_op->m_src, 0); + desc[idx].len = (sym_op->cipher.data.offset + + sym_op->cipher.data.length); + desc[idx++].flags = VRING_DESC_F_NEXT; + + /* indirect vring: dst data */ + if (sym_op->m_dst) { + desc[idx].addr = rte_pktmbuf_mtophys_offset(sym_op->m_dst, 0); + desc[idx].len = (sym_op->cipher.data.offset + + sym_op->cipher.data.length); + } else { + desc[idx].addr = rte_pktmbuf_mtophys_offset(sym_op->m_src, 0); + desc[idx].len = (sym_op->cipher.data.offset + + sym_op->cipher.data.length); + } + desc[idx++].flags = VRING_DESC_F_WRITE | VRING_DESC_F_NEXT; + + /* indirect vring: digest result */ + para = &(session->ctrl.u.sym_create_session.u.chain.para); + if (para->hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN) + hash_result_len = para->u.hash_param.hash_result_len; + if (para->hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH) + hash_result_len = para->u.mac_param.hash_result_len; + if (hash_result_len > 0) { + desc[idx].addr = sym_op->auth.digest.phys_addr; + desc[idx].len = hash_result_len; + desc[idx++].flags = VRING_DESC_F_WRITE | VRING_DESC_F_NEXT; + } + + /* indirect vring: last part, status returned */ + desc[idx].addr = indirect_op_data_req_phys_addr + req_data_len; + desc[idx].len = sizeof(struct virtio_crypto_inhdr); + desc[idx++].flags = VRING_DESC_F_WRITE; + + num_entry = idx; + + /* save the infos to use when receiving packets */ + dxp->crypto_op = (void *)cop; + dxp->ndescs = needed; + + /* use a single buffer */ + start_dp = txvq->vq_ring.desc; + start_dp[head_idx].addr = indirect_op_data_req_phys_addr + + indirect_vring_addr_offset; + start_dp[head_idx].len = num_entry * sizeof(struct vring_desc); + start_dp[head_idx].flags = VRING_DESC_F_INDIRECT; + + idx = start_dp[head_idx].next; + txvq->vq_desc_head_idx = idx; + if (txvq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) + txvq->vq_desc_tail_idx = idx; + txvq->vq_free_cnt = (uint16_t)(txvq->vq_free_cnt - needed); + vq_update_avail_ring(txvq, head_idx); + + return 0; +} + +static int +virtqueue_crypto_enqueue_xmit(struct virtqueue *txvq, + struct rte_crypto_op *cop) +{ + int ret; + + switch (cop->type) { + case RTE_CRYPTO_OP_TYPE_SYMMETRIC: + ret = virtqueue_crypto_sym_enqueue_xmit(txvq, cop); + break; + default: + VIRTIO_CRYPTO_TX_LOG_ERR("invalid crypto op type %u", + cop->type); + ret = -EFAULT; + break; + } + + return ret; +} static int virtio_crypto_vring_start(struct virtqueue *vq) @@ -71,24 +416,100 @@ virtio_crypto_dataq_start(struct rte_cryptodev *dev) } } +/* vring size of data queue is 1024 */ +#define VIRTIO_MBUF_BURST_SZ 1024 + uint16_t -virtio_crypto_pkt_rx_burst( - void *tx_queue __rte_unused, - struct rte_crypto_op **rx_pkts __rte_unused, - uint16_t nb_pkts __rte_unused) +virtio_crypto_pkt_rx_burst(void *tx_queue, struct rte_crypto_op **rx_pkts, + uint16_t nb_pkts) { - uint16_t nb_rx = 0; + struct virtqueue *txvq = tx_queue; + uint16_t nb_used, num, nb_rx; + + nb_used = VIRTQUEUE_NUSED(txvq); + + virtio_rmb(); + + num = (uint16_t)(likely(nb_used <= nb_pkts) ? nb_used : nb_pkts); + num = (uint16_t)(likely(num <= VIRTIO_MBUF_BURST_SZ) + ? num : VIRTIO_MBUF_BURST_SZ); + + if (num == 0) + return 0; + + nb_rx = virtqueue_dequeue_burst_rx(txvq, rx_pkts, num); + VIRTIO_CRYPTO_RX_LOG_DBG("used:%d dequeue:%d", nb_used, num); return nb_rx; } uint16_t -virtio_crypto_pkt_tx_burst( - void *tx_queue __rte_unused, - struct rte_crypto_op **tx_pkts __rte_unused, - uint16_t nb_pkts __rte_unused) +virtio_crypto_pkt_tx_burst(void *tx_queue, struct rte_crypto_op **tx_pkts, + uint16_t nb_pkts) { - uint16_t nb_tx = 0; + struct virtqueue *txvq; + uint16_t nb_tx; + int error; + + if (unlikely(nb_pkts < 1)) + return nb_pkts; + if (unlikely(tx_queue == NULL)) { + VIRTIO_CRYPTO_TX_LOG_ERR("tx_queue is NULL"); + return 0; + } + txvq = tx_queue; + + VIRTIO_CRYPTO_TX_LOG_DBG("%d packets to xmit", nb_pkts); + + for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { + struct rte_mbuf *txm = tx_pkts[nb_tx]->sym->m_src; + /* nb_segs is always 1 at virtio crypto situation */ + int need = txm->nb_segs - txvq->vq_free_cnt; + + /* + * Positive value indicates it hasn't enough space in vring + * descriptors + */ + if (unlikely(need > 0)) { + /* + * try it again because the receive process may be + * free some space + */ + need = txm->nb_segs - txvq->vq_free_cnt; + if (unlikely(need > 0)) { + VIRTIO_CRYPTO_TX_LOG_DBG("No free tx " + "descriptors to transmit"); + break; + } + } + + txvq->packets_sent_total++; + + /* Enqueue Packet buffers */ + error = virtqueue_crypto_enqueue_xmit(txvq, tx_pkts[nb_tx]); + if (unlikely(error)) { + if (error == ENOSPC) + VIRTIO_CRYPTO_TX_LOG_ERR( + "virtqueue_enqueue Free count = 0"); + else if (error == EMSGSIZE) + VIRTIO_CRYPTO_TX_LOG_ERR( + "virtqueue_enqueue Free count < 1"); + else + VIRTIO_CRYPTO_TX_LOG_ERR( + "virtqueue_enqueue error: %d", error); + txvq->packets_sent_failed++; + break; + } + } + + if (likely(nb_tx)) { + vq_update_avail_idx(txvq); + + if (unlikely(virtqueue_kick_prepare(txvq))) { + virtqueue_notify(txvq); + VIRTIO_CRYPTO_TX_LOG_DBG("Notified backend after xmit"); + } + } return nb_tx; } -- 2.20.1