+ *cur_desc = desc + 1;
+
+ *end_wb_data = wb_data;
+
+ return head;
+
+error_exit:
+ if (head)
+ free_wb_data(head, vc_req->wb_pool);
+
+ return NULL;
+}
+
+static __rte_always_inline uint8_t
+vhost_crypto_check_cipher_request(struct virtio_crypto_cipher_data_req *req)
+{
+ if (likely((req->para.iv_len <= VHOST_CRYPTO_MAX_IV_LEN) &&
+ (req->para.src_data_len <= RTE_MBUF_DEFAULT_BUF_SIZE) &&
+ (req->para.dst_data_len >= req->para.src_data_len) &&
+ (req->para.dst_data_len <= RTE_MBUF_DEFAULT_BUF_SIZE)))
+ return VIRTIO_CRYPTO_OK;
+ return VIRTIO_CRYPTO_BADMSG;
+}
+
+static __rte_always_inline uint8_t
+prepare_sym_cipher_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op,
+ struct vhost_crypto_data_req *vc_req,
+ struct virtio_crypto_cipher_data_req *cipher,
+ struct vhost_crypto_desc *head,
+ uint32_t max_n_descs)
+{
+ struct vhost_crypto_desc *desc = head;
+ struct vhost_crypto_writeback_data *ewb = NULL;
+ struct rte_mbuf *m_src = op->sym->m_src, *m_dst = op->sym->m_dst;
+ uint8_t *iv_data = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET);
+ uint8_t ret = vhost_crypto_check_cipher_request(cipher);
+
+ if (unlikely(ret != VIRTIO_CRYPTO_OK))
+ goto error_exit;
+
+ /* prepare */
+ /* iv */
+ if (unlikely(copy_data(iv_data, vc_req, head, &desc,
+ cipher->para.iv_len, max_n_descs))) {
+ ret = VIRTIO_CRYPTO_BADMSG;
+ goto error_exit;
+ }
+
+ switch (vcrypto->option) {
+ case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
+ m_src->data_len = cipher->para.src_data_len;
+ m_src->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr,
+ cipher->para.src_data_len);
+ m_src->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RO);
+ if (unlikely(m_src->buf_iova == 0 ||
+ m_src->buf_addr == NULL)) {
+ VC_LOG_ERR("zero_copy may fail due to cross page data");
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
+
+ if (unlikely(move_desc(head, &desc, cipher->para.src_data_len,
+ max_n_descs) < 0)) {
+ VC_LOG_ERR("Incorrect descriptor");
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
+
+ break;
+ case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
+ vc_req->wb_pool = vcrypto->wb_pool;
+ m_src->data_len = cipher->para.src_data_len;
+ if (unlikely(copy_data(rte_pktmbuf_mtod(m_src, uint8_t *),
+ vc_req, head, &desc, cipher->para.src_data_len,
+ max_n_descs) < 0)) {
+ ret = VIRTIO_CRYPTO_BADMSG;
+ goto error_exit;
+ }
+ break;
+ default:
+ ret = VIRTIO_CRYPTO_BADMSG;
+ goto error_exit;
+ }
+
+ /* dst */
+ desc = find_write_desc(head, desc, max_n_descs);
+ if (unlikely(!desc)) {
+ VC_LOG_ERR("Cannot find write location");
+ ret = VIRTIO_CRYPTO_BADMSG;
+ goto error_exit;
+ }
+
+ switch (vcrypto->option) {
+ case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
+ m_dst->buf_iova = gpa_to_hpa(vcrypto->dev,
+ desc->addr, cipher->para.dst_data_len);
+ m_dst->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RW);
+ if (unlikely(m_dst->buf_iova == 0 || m_dst->buf_addr == NULL)) {
+ VC_LOG_ERR("zero_copy may fail due to cross page data");
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
+
+ if (unlikely(move_desc(head, &desc, cipher->para.dst_data_len,
+ max_n_descs) < 0)) {
+ VC_LOG_ERR("Incorrect descriptor");
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
+
+ m_dst->data_len = cipher->para.dst_data_len;
+ break;
+ case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
+ vc_req->wb = prepare_write_back_data(vc_req, head, &desc, &ewb,
+ rte_pktmbuf_mtod(m_src, uint8_t *), 0,
+ cipher->para.dst_data_len, max_n_descs);
+ if (unlikely(vc_req->wb == NULL)) {
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
+
+ break;
+ default:
+ ret = VIRTIO_CRYPTO_BADMSG;
+ goto error_exit;
+ }
+
+ /* src data */
+ op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
+ op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
+
+ op->sym->cipher.data.offset = 0;
+ op->sym->cipher.data.length = cipher->para.src_data_len;
+
+ vc_req->inhdr = get_data_ptr(vc_req, desc, VHOST_ACCESS_WO);
+ if (unlikely(vc_req->inhdr == NULL)) {
+ ret = VIRTIO_CRYPTO_BADMSG;
+ goto error_exit;
+ }
+
+ vc_req->inhdr->status = VIRTIO_CRYPTO_OK;
+ vc_req->len = cipher->para.dst_data_len + INHDR_LEN;
+
+ return 0;
+
+error_exit:
+ if (vc_req->wb)
+ free_wb_data(vc_req->wb, vc_req->wb_pool);
+
+ vc_req->len = INHDR_LEN;
+ return ret;
+}
+
+static __rte_always_inline uint8_t
+vhost_crypto_check_chain_request(struct virtio_crypto_alg_chain_data_req *req)
+{
+ if (likely((req->para.iv_len <= VHOST_CRYPTO_MAX_IV_LEN) &&
+ (req->para.src_data_len <= VHOST_CRYPTO_MAX_DATA_SIZE) &&
+ (req->para.dst_data_len >= req->para.src_data_len) &&
+ (req->para.dst_data_len <= VHOST_CRYPTO_MAX_DATA_SIZE) &&
+ (req->para.cipher_start_src_offset <
+ VHOST_CRYPTO_MAX_DATA_SIZE) &&
+ (req->para.len_to_cipher <= VHOST_CRYPTO_MAX_DATA_SIZE) &&
+ (req->para.hash_start_src_offset <
+ VHOST_CRYPTO_MAX_DATA_SIZE) &&
+ (req->para.len_to_hash <= VHOST_CRYPTO_MAX_DATA_SIZE) &&
+ (req->para.cipher_start_src_offset + req->para.len_to_cipher <=
+ req->para.src_data_len) &&
+ (req->para.hash_start_src_offset + req->para.len_to_hash <=
+ req->para.src_data_len) &&
+ (req->para.dst_data_len + req->para.hash_result_len <=
+ VHOST_CRYPTO_MAX_DATA_SIZE)))
+ return VIRTIO_CRYPTO_OK;
+ return VIRTIO_CRYPTO_BADMSG;
+}
+
+static __rte_always_inline uint8_t
+prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op,
+ struct vhost_crypto_data_req *vc_req,
+ struct virtio_crypto_alg_chain_data_req *chain,
+ struct vhost_crypto_desc *head,
+ uint32_t max_n_descs)
+{
+ struct vhost_crypto_desc *desc = head, *digest_desc;
+ struct vhost_crypto_writeback_data *ewb = NULL, *ewb2 = NULL;
+ struct rte_mbuf *m_src = op->sym->m_src, *m_dst = op->sym->m_dst;
+ uint8_t *iv_data = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET);
+ uint32_t digest_offset;
+ void *digest_addr;
+ uint8_t ret = vhost_crypto_check_chain_request(chain);
+
+ if (unlikely(ret != VIRTIO_CRYPTO_OK))
+ goto error_exit;
+
+ /* prepare */
+ /* iv */
+ if (unlikely(copy_data(iv_data, vc_req, head, &desc,
+ chain->para.iv_len, max_n_descs) < 0)) {
+ ret = VIRTIO_CRYPTO_BADMSG;
+ goto error_exit;
+ }
+
+ switch (vcrypto->option) {
+ case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
+ m_src->data_len = chain->para.src_data_len;
+ m_dst->data_len = chain->para.dst_data_len;
+
+ m_src->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr,
+ chain->para.src_data_len);
+ m_src->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RO);
+ if (unlikely(m_src->buf_iova == 0 || m_src->buf_addr == NULL)) {
+ VC_LOG_ERR("zero_copy may fail due to cross page data");
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
+
+ if (unlikely(move_desc(head, &desc, chain->para.src_data_len,
+ max_n_descs) < 0)) {
+ VC_LOG_ERR("Incorrect descriptor");
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
+ break;
+ case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
+ vc_req->wb_pool = vcrypto->wb_pool;
+ m_src->data_len = chain->para.src_data_len;
+ if (unlikely(copy_data(rte_pktmbuf_mtod(m_src, uint8_t *),
+ vc_req, head, &desc, chain->para.src_data_len,
+ max_n_descs) < 0)) {
+ ret = VIRTIO_CRYPTO_BADMSG;
+ goto error_exit;
+ }
+
+ break;
+ default:
+ ret = VIRTIO_CRYPTO_BADMSG;
+ goto error_exit;
+ }
+
+ /* dst */
+ desc = find_write_desc(head, desc, max_n_descs);
+ if (unlikely(!desc)) {
+ VC_LOG_ERR("Cannot find write location");
+ ret = VIRTIO_CRYPTO_BADMSG;
+ goto error_exit;
+ }
+
+ switch (vcrypto->option) {
+ case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
+ m_dst->buf_iova = gpa_to_hpa(vcrypto->dev,
+ desc->addr, chain->para.dst_data_len);
+ m_dst->buf_addr = get_data_ptr(vc_req, desc, VHOST_ACCESS_RW);
+ if (unlikely(m_dst->buf_iova == 0 || m_dst->buf_addr == NULL)) {
+ VC_LOG_ERR("zero_copy may fail due to cross page data");
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
+
+ if (unlikely(move_desc(vc_req->head, &desc,
+ chain->para.dst_data_len, max_n_descs) < 0)) {
+ VC_LOG_ERR("Incorrect descriptor");
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
+
+ op->sym->auth.digest.phys_addr = gpa_to_hpa(vcrypto->dev,
+ desc->addr, chain->para.hash_result_len);
+ op->sym->auth.digest.data = get_data_ptr(vc_req, desc,
+ VHOST_ACCESS_RW);
+ if (unlikely(op->sym->auth.digest.phys_addr == 0)) {
+ VC_LOG_ERR("zero_copy may fail due to cross page data");
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
+
+ if (unlikely(move_desc(head, &desc,
+ chain->para.hash_result_len,
+ max_n_descs) < 0)) {
+ VC_LOG_ERR("Incorrect descriptor");
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
+
+ break;
+ case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
+ vc_req->wb = prepare_write_back_data(vc_req, head, &desc, &ewb,
+ rte_pktmbuf_mtod(m_src, uint8_t *),
+ chain->para.cipher_start_src_offset,
+ chain->para.dst_data_len -
+ chain->para.cipher_start_src_offset,
+ max_n_descs);
+ if (unlikely(vc_req->wb == NULL)) {
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
+
+ digest_desc = desc;
+ digest_offset = m_src->data_len;
+ digest_addr = rte_pktmbuf_mtod_offset(m_src, void *,
+ digest_offset);
+
+ /** create a wb_data for digest */
+ ewb->next = prepare_write_back_data(vc_req, head, &desc,
+ &ewb2, digest_addr, 0,
+ chain->para.hash_result_len, max_n_descs);
+ if (unlikely(ewb->next == NULL)) {
+ ret = VIRTIO_CRYPTO_ERR;
+ goto error_exit;
+ }
+
+ if (unlikely(copy_data(digest_addr, vc_req, head, &digest_desc,
+ chain->para.hash_result_len,
+ max_n_descs) < 0)) {
+ ret = VIRTIO_CRYPTO_BADMSG;
+ goto error_exit;
+ }
+
+ op->sym->auth.digest.data = digest_addr;
+ op->sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m_src,
+ digest_offset);
+ break;
+ default:
+ ret = VIRTIO_CRYPTO_BADMSG;
+ goto error_exit;
+ }
+
+ /* record inhdr */
+ vc_req->inhdr = get_data_ptr(vc_req, desc, VHOST_ACCESS_WO);
+ if (unlikely(vc_req->inhdr == NULL)) {
+ ret = VIRTIO_CRYPTO_BADMSG;
+ goto error_exit;
+ }
+
+ vc_req->inhdr->status = VIRTIO_CRYPTO_OK;
+
+ op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
+ op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
+
+ op->sym->cipher.data.offset = chain->para.cipher_start_src_offset;
+ op->sym->cipher.data.length = chain->para.src_data_len -
+ chain->para.cipher_start_src_offset;
+
+ op->sym->auth.data.offset = chain->para.hash_start_src_offset;
+ op->sym->auth.data.length = chain->para.len_to_hash;
+
+ vc_req->len = chain->para.dst_data_len + chain->para.hash_result_len +
+ INHDR_LEN;
+ return 0;
+
+error_exit:
+ if (vc_req->wb)
+ free_wb_data(vc_req->wb, vc_req->wb_pool);
+ vc_req->len = INHDR_LEN;
+ return ret;
+}
+
+/**
+ * Process on descriptor
+ */
+static __rte_always_inline int
+vhost_crypto_process_one_req(struct vhost_crypto *vcrypto,
+ struct vhost_virtqueue *vq, struct rte_crypto_op *op,
+ struct vring_desc *head, struct vhost_crypto_desc *descs,
+ uint16_t desc_idx)
+{
+ struct vhost_crypto_data_req *vc_req = rte_mbuf_to_priv(op->sym->m_src);
+ struct rte_cryptodev_sym_session *session;
+ struct virtio_crypto_op_data_req req;
+ struct virtio_crypto_inhdr *inhdr;
+ struct vhost_crypto_desc *desc = descs;
+ struct vring_desc *src_desc;
+ uint64_t session_id;
+ uint64_t dlen;
+ uint32_t nb_descs = 0, max_n_descs, i;
+ int err;
+
+ vc_req->desc_idx = desc_idx;
+ vc_req->dev = vcrypto->dev;
+ vc_req->vq = vq;
+
+ if (unlikely((head->flags & VRING_DESC_F_INDIRECT) == 0)) {
+ VC_LOG_ERR("Invalid descriptor");
+ return -1;
+ }
+
+ dlen = head->len;
+ src_desc = IOVA_TO_VVA(struct vring_desc *, vc_req, head->addr,
+ &dlen, VHOST_ACCESS_RO);
+ if (unlikely(!src_desc || dlen != head->len)) {
+ VC_LOG_ERR("Invalid descriptor");
+ return -1;
+ }
+ head = src_desc;
+
+ nb_descs = max_n_descs = dlen / sizeof(struct vring_desc);
+ if (unlikely(nb_descs > VHOST_CRYPTO_MAX_N_DESC || nb_descs == 0)) {
+ err = VIRTIO_CRYPTO_ERR;
+ VC_LOG_ERR("Cannot process num of descriptors %u", nb_descs);
+ if (nb_descs > 0) {
+ struct vring_desc *inhdr_desc = head;
+ while (inhdr_desc->flags & VRING_DESC_F_NEXT) {
+ if (inhdr_desc->next >= max_n_descs)
+ return -1;
+ inhdr_desc = &head[inhdr_desc->next];
+ }
+ if (inhdr_desc->len != sizeof(*inhdr))
+ return -1;
+ inhdr = IOVA_TO_VVA(struct virtio_crypto_inhdr *,
+ vc_req, inhdr_desc->addr, &dlen,
+ VHOST_ACCESS_WO);
+ if (unlikely(!inhdr || dlen != inhdr_desc->len))
+ return -1;
+ inhdr->status = VIRTIO_CRYPTO_ERR;
+ return -1;
+ }
+ }
+
+ /* copy descriptors to local variable */
+ for (i = 0; i < max_n_descs; i++) {
+ desc->addr = src_desc->addr;
+ desc->len = src_desc->len;
+ desc->flags = src_desc->flags;
+ desc++;
+ if (unlikely((src_desc->flags & VRING_DESC_F_NEXT) == 0))
+ break;
+ if (unlikely(src_desc->next >= max_n_descs)) {
+ err = VIRTIO_CRYPTO_BADMSG;
+ VC_LOG_ERR("Invalid descriptor");
+ goto error_exit;
+ }
+ src_desc = &head[src_desc->next];
+ }
+
+ vc_req->head = head;
+ vc_req->zero_copy = vcrypto->option;
+
+ nb_descs = desc - descs;
+ desc = descs;
+
+ if (unlikely(desc->len < sizeof(req))) {
+ err = VIRTIO_CRYPTO_BADMSG;
+ VC_LOG_ERR("Invalid descriptor");
+ goto error_exit;
+ }
+
+ if (unlikely(copy_data(&req, vc_req, descs, &desc, sizeof(req),
+ max_n_descs) < 0)) {
+ err = VIRTIO_CRYPTO_BADMSG;
+ VC_LOG_ERR("Invalid descriptor");
+ goto error_exit;
+ }
+
+ /* desc is advanced by 1 now */
+ max_n_descs -= 1;
+
+ switch (req.header.opcode) {
+ case VIRTIO_CRYPTO_CIPHER_ENCRYPT:
+ case VIRTIO_CRYPTO_CIPHER_DECRYPT:
+ session_id = req.header.session_id;
+
+ /* one branch to avoid unnecessary table lookup */
+ if (vcrypto->cache_session_id != session_id) {
+ err = rte_hash_lookup_data(vcrypto->session_map,
+ &session_id, (void **)&session);
+ if (unlikely(err < 0)) {
+ err = VIRTIO_CRYPTO_ERR;
+ VC_LOG_ERR("Failed to find session %"PRIu64,
+ session_id);
+ goto error_exit;
+ }
+
+ vcrypto->cache_session = session;
+ vcrypto->cache_session_id = session_id;
+ }
+
+ session = vcrypto->cache_session;
+
+ err = rte_crypto_op_attach_sym_session(op, session);
+ if (unlikely(err < 0)) {
+ err = VIRTIO_CRYPTO_ERR;
+ VC_LOG_ERR("Failed to attach session to op");
+ goto error_exit;
+ }
+
+ switch (req.u.sym_req.op_type) {
+ case VIRTIO_CRYPTO_SYM_OP_NONE:
+ err = VIRTIO_CRYPTO_NOTSUPP;
+ break;
+ case VIRTIO_CRYPTO_SYM_OP_CIPHER:
+ err = prepare_sym_cipher_op(vcrypto, op, vc_req,
+ &req.u.sym_req.u.cipher, desc,
+ max_n_descs);
+ break;
+ case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING:
+ err = prepare_sym_chain_op(vcrypto, op, vc_req,
+ &req.u.sym_req.u.chain, desc,
+ max_n_descs);
+ break;
+ }
+ if (unlikely(err != 0)) {
+ VC_LOG_ERR("Failed to process sym request");
+ goto error_exit;
+ }
+ break;
+ default:
+ err = VIRTIO_CRYPTO_ERR;
+ VC_LOG_ERR("Unsupported symmetric crypto request type %u",
+ req.header.opcode);
+ goto error_exit;
+ }
+
+ return 0;
+
+error_exit:
+
+ inhdr = reach_inhdr(vc_req, descs, max_n_descs);
+ if (likely(inhdr != NULL))
+ inhdr->status = (uint8_t)err;
+
+ return -1;
+}
+
+static __rte_always_inline struct vhost_virtqueue *
+vhost_crypto_finalize_one_request(struct rte_crypto_op *op,
+ struct vhost_virtqueue *old_vq)
+{
+ struct rte_mbuf *m_src = op->sym->m_src;
+ struct rte_mbuf *m_dst = op->sym->m_dst;
+ struct vhost_crypto_data_req *vc_req = rte_mbuf_to_priv(m_src);
+ struct vhost_virtqueue *vq = vc_req->vq;
+ uint16_t used_idx = vc_req->desc_idx, desc_idx;
+
+ if (unlikely(!vc_req)) {
+ VC_LOG_ERR("Failed to retrieve vc_req");
+ return NULL;
+ }
+
+ if (old_vq && (vq != old_vq))
+ return vq;
+
+ if (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS))
+ vc_req->inhdr->status = VIRTIO_CRYPTO_ERR;
+ else {
+ if (vc_req->zero_copy == 0)
+ write_back_data(vc_req);
+ }
+
+ desc_idx = vq->avail->ring[used_idx];
+ vq->used->ring[desc_idx].id = vq->avail->ring[desc_idx];
+ vq->used->ring[desc_idx].len = vc_req->len;
+
+ rte_mempool_put(m_src->pool, (void *)m_src);
+
+ if (m_dst)
+ rte_mempool_put(m_dst->pool, (void *)m_dst);
+
+ return vc_req->vq;
+}
+
+static __rte_always_inline uint16_t
+vhost_crypto_complete_one_vm_requests(struct rte_crypto_op **ops,
+ uint16_t nb_ops, int *callfd)
+{
+ uint16_t processed = 1;
+ struct vhost_virtqueue *vq, *tmp_vq;
+
+ if (unlikely(nb_ops == 0))
+ return 0;
+
+ vq = vhost_crypto_finalize_one_request(ops[0], NULL);
+ if (unlikely(vq == NULL))
+ return 0;
+ tmp_vq = vq;
+
+ while ((processed < nb_ops)) {
+ tmp_vq = vhost_crypto_finalize_one_request(ops[processed],
+ tmp_vq);
+
+ if (unlikely(vq != tmp_vq))
+ break;
+
+ processed++;
+ }
+
+ *callfd = vq->callfd;
+
+ *(volatile uint16_t *)&vq->used->idx += processed;
+
+ return processed;
+}
+
+int
+rte_vhost_crypto_driver_start(const char *path)
+{
+ uint64_t protocol_features;
+ int ret;
+
+ ret = rte_vhost_driver_set_features(path, VIRTIO_CRYPTO_FEATURES);
+ if (ret)
+ return -1;
+
+ ret = rte_vhost_driver_get_protocol_features(path, &protocol_features);
+ if (ret)
+ return -1;
+ protocol_features |= (1ULL << VHOST_USER_PROTOCOL_F_CONFIG);
+ ret = rte_vhost_driver_set_protocol_features(path, protocol_features);
+ if (ret)
+ return -1;
+
+ return rte_vhost_driver_start(path);
+}
+
+int
+rte_vhost_crypto_create(int vid, uint8_t cryptodev_id,
+ struct rte_mempool *sess_pool,
+ struct rte_mempool *sess_priv_pool,
+ int socket_id)
+{
+ struct virtio_net *dev = get_device(vid);
+ struct rte_hash_parameters params = {0};
+ struct vhost_crypto *vcrypto;
+ char name[128];
+ int ret;
+
+ if (!dev) {
+ VC_LOG_ERR("Invalid vid %i", vid);
+ return -EINVAL;
+ }
+
+ vcrypto = rte_zmalloc_socket(NULL, sizeof(*vcrypto),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (!vcrypto) {
+ VC_LOG_ERR("Insufficient memory");
+ return -ENOMEM;
+ }
+
+ vcrypto->sess_pool = sess_pool;
+ vcrypto->sess_priv_pool = sess_priv_pool;
+ vcrypto->cid = cryptodev_id;
+ vcrypto->cache_session_id = UINT64_MAX;
+ vcrypto->last_session_id = 1;
+ vcrypto->dev = dev;
+ vcrypto->option = RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE;
+
+ snprintf(name, 127, "HASH_VHOST_CRYPT_%u", (uint32_t)vid);
+ params.name = name;
+ params.entries = VHOST_CRYPTO_SESSION_MAP_ENTRIES;
+ params.hash_func = rte_jhash;
+ params.key_len = sizeof(uint64_t);
+ params.socket_id = socket_id;
+ vcrypto->session_map = rte_hash_create(¶ms);
+ if (!vcrypto->session_map) {
+ VC_LOG_ERR("Failed to creath session map");
+ ret = -ENOMEM;
+ goto error_exit;
+ }
+
+ snprintf(name, 127, "MBUF_POOL_VM_%u", (uint32_t)vid);
+ vcrypto->mbuf_pool = rte_pktmbuf_pool_create(name,
+ VHOST_CRYPTO_MBUF_POOL_SIZE, 512,
+ sizeof(struct vhost_crypto_data_req),
+ VHOST_CRYPTO_MAX_DATA_SIZE + RTE_PKTMBUF_HEADROOM,
+ rte_socket_id());
+ if (!vcrypto->mbuf_pool) {
+ VC_LOG_ERR("Failed to creath mbuf pool");
+ ret = -ENOMEM;
+ goto error_exit;
+ }
+
+ snprintf(name, 127, "WB_POOL_VM_%u", (uint32_t)vid);
+ vcrypto->wb_pool = rte_mempool_create(name,
+ VHOST_CRYPTO_MBUF_POOL_SIZE,
+ sizeof(struct vhost_crypto_writeback_data),
+ 128, 0, NULL, NULL, NULL, NULL,
+ rte_socket_id(), 0);
+ if (!vcrypto->wb_pool) {
+ VC_LOG_ERR("Failed to creath mempool");
+ ret = -ENOMEM;
+ goto error_exit;
+ }
+
+ dev->extern_data = vcrypto;
+ dev->extern_ops.pre_msg_handle = NULL;
+ dev->extern_ops.post_msg_handle = vhost_crypto_msg_post_handler;
+
+ return 0;
+
+error_exit:
+ if (vcrypto->session_map)
+ rte_hash_free(vcrypto->session_map);
+ if (vcrypto->mbuf_pool)
+ rte_mempool_free(vcrypto->mbuf_pool);
+
+ rte_free(vcrypto);