lib/librte_vhost/vhost_crypto.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017-2018 Intel Corporation
   3  */
   4 #include <rte_malloc.h>
   5 #include <rte_hash.h>
   6 #include <rte_mbuf.h>
   7 #include <rte_cryptodev.h>
   8
   9 #include "rte_vhost_crypto.h"
  10 #include "vhost.h"
  11 #include "vhost_user.h"
  12 #include "virtio_crypto.h"
  13
  14 #define INHDR_LEN               (sizeof(struct virtio_crypto_inhdr))
  15 #define IV_OFFSET               (sizeof(struct rte_crypto_op) + \
  16                                 sizeof(struct rte_crypto_sym_op))
  17
  18 #ifdef RTE_LIBRTE_VHOST_DEBUG
  19 #define VC_LOG_ERR(fmt, args...)                                \
  20         RTE_LOG(ERR, USER1, "[%s] %s() line %u: " fmt "\n",     \
  21                 "Vhost-Crypto", __func__, __LINE__, ## args)
  22 #define VC_LOG_INFO(fmt, args...)                               \
  23         RTE_LOG(INFO, USER1, "[%s] %s() line %u: " fmt "\n",    \
  24                 "Vhost-Crypto", __func__, __LINE__, ## args)
  25
  26 #define VC_LOG_DBG(fmt, args...)                                \
  27         RTE_LOG(DEBUG, USER1, "[%s] %s() line %u: " fmt "\n",   \
  28                 "Vhost-Crypto", __func__, __LINE__, ## args)
  29 #else
  30 #define VC_LOG_ERR(fmt, args...)                                \
  31         RTE_LOG(ERR, USER1, "[VHOST-Crypto]: " fmt "\n", ## args)
  32 #define VC_LOG_INFO(fmt, args...)                               \
  33         RTE_LOG(INFO, USER1, "[VHOST-Crypto]: " fmt "\n", ## args)
  34 #define VC_LOG_DBG(fmt, args...)
  35 #endif
  36
  37 #define GPA_TO_VVA(t, m, a)     ((t)(uintptr_t)rte_vhost_gpa_to_vva(m, a))
  38
  39 static int
  40 cipher_algo_transform(uint32_t virtio_cipher_algo)
  41 {
  42         int ret;
  43
  44         switch (virtio_cipher_algo) {
  45         case VIRTIO_CRYPTO_CIPHER_AES_CBC:
  46                 ret = RTE_CRYPTO_CIPHER_AES_CBC;
  47                 break;
  48         case VIRTIO_CRYPTO_CIPHER_AES_CTR:
  49                 ret = RTE_CRYPTO_CIPHER_AES_CTR;
  50                 break;
  51         case VIRTIO_CRYPTO_CIPHER_DES_ECB:
  52                 ret = -VIRTIO_CRYPTO_NOTSUPP;
  53                 break;
  54         case VIRTIO_CRYPTO_CIPHER_DES_CBC:
  55                 ret = RTE_CRYPTO_CIPHER_DES_CBC;
  56                 break;
  57         case VIRTIO_CRYPTO_CIPHER_3DES_ECB:
  58                 ret = RTE_CRYPTO_CIPHER_3DES_ECB;
  59                 break;
  60         case VIRTIO_CRYPTO_CIPHER_3DES_CBC:
  61                 ret = RTE_CRYPTO_CIPHER_3DES_CBC;
  62                 break;
  63         case VIRTIO_CRYPTO_CIPHER_3DES_CTR:
  64                 ret = RTE_CRYPTO_CIPHER_3DES_CTR;
  65                 break;
  66         case VIRTIO_CRYPTO_CIPHER_KASUMI_F8:
  67                 ret = RTE_CRYPTO_CIPHER_KASUMI_F8;
  68                 break;
  69         case VIRTIO_CRYPTO_CIPHER_SNOW3G_UEA2:
  70                 ret = RTE_CRYPTO_CIPHER_SNOW3G_UEA2;
  71                 break;
  72         case VIRTIO_CRYPTO_CIPHER_AES_F8:
  73                 ret = RTE_CRYPTO_CIPHER_AES_F8;
  74                 break;
  75         case VIRTIO_CRYPTO_CIPHER_AES_XTS:
  76                 ret = RTE_CRYPTO_CIPHER_AES_XTS;
  77                 break;
  78         case VIRTIO_CRYPTO_CIPHER_ZUC_EEA3:
  79                 ret = RTE_CRYPTO_CIPHER_ZUC_EEA3;
  80                 break;
  81         default:
  82                 ret = -VIRTIO_CRYPTO_BADMSG;
  83                 break;
  84         }
  85
  86         return ret;
  87 }
  88
  89 static int
  90 auth_algo_transform(uint32_t virtio_auth_algo)
  91 {
  92         int ret;
  93
  94         switch (virtio_auth_algo) {
  95
  96         case VIRTIO_CRYPTO_NO_MAC:
  97                 ret = RTE_CRYPTO_AUTH_NULL;
  98                 break;
  99         case VIRTIO_CRYPTO_MAC_HMAC_MD5:
 100                 ret = RTE_CRYPTO_AUTH_MD5_HMAC;
 101                 break;
 102         case VIRTIO_CRYPTO_MAC_HMAC_SHA1:
 103                 ret = RTE_CRYPTO_AUTH_SHA1_HMAC;
 104                 break;
 105         case VIRTIO_CRYPTO_MAC_HMAC_SHA_224:
 106                 ret = RTE_CRYPTO_AUTH_SHA224_HMAC;
 107                 break;
 108         case VIRTIO_CRYPTO_MAC_HMAC_SHA_256:
 109                 ret = RTE_CRYPTO_AUTH_SHA256_HMAC;
 110                 break;
 111         case VIRTIO_CRYPTO_MAC_HMAC_SHA_384:
 112                 ret = RTE_CRYPTO_AUTH_SHA384_HMAC;
 113                 break;
 114         case VIRTIO_CRYPTO_MAC_HMAC_SHA_512:
 115                 ret = RTE_CRYPTO_AUTH_SHA512_HMAC;
 116                 break;
 117         case VIRTIO_CRYPTO_MAC_CMAC_3DES:
 118                 ret = -VIRTIO_CRYPTO_NOTSUPP;
 119                 break;
 120         case VIRTIO_CRYPTO_MAC_CMAC_AES:
 121                 ret = RTE_CRYPTO_AUTH_AES_CMAC;
 122                 break;
 123         case VIRTIO_CRYPTO_MAC_KASUMI_F9:
 124                 ret = RTE_CRYPTO_AUTH_KASUMI_F9;
 125                 break;
 126         case VIRTIO_CRYPTO_MAC_SNOW3G_UIA2:
 127                 ret = RTE_CRYPTO_AUTH_SNOW3G_UIA2;
 128                 break;
 129         case VIRTIO_CRYPTO_MAC_GMAC_AES:
 130                 ret = RTE_CRYPTO_AUTH_AES_GMAC;
 131                 break;
 132         case VIRTIO_CRYPTO_MAC_GMAC_TWOFISH:
 133                 ret = -VIRTIO_CRYPTO_NOTSUPP;
 134                 break;
 135         case VIRTIO_CRYPTO_MAC_CBCMAC_AES:
 136                 ret = RTE_CRYPTO_AUTH_AES_CBC_MAC;
 137                 break;
 138         case VIRTIO_CRYPTO_MAC_CBCMAC_KASUMI_F9:
 139                 ret = -VIRTIO_CRYPTO_NOTSUPP;
 140                 break;
 141         case VIRTIO_CRYPTO_MAC_XCBC_AES:
 142                 ret = RTE_CRYPTO_AUTH_AES_XCBC_MAC;
 143                 break;
 144         default:
 145                 ret = -VIRTIO_CRYPTO_BADMSG;
 146                 break;
 147         }
 148
 149         return ret;
 150 }
 151
 152 static int get_iv_len(enum rte_crypto_cipher_algorithm algo)
 153 {
 154         int len;
 155
 156         switch (algo) {
 157         case RTE_CRYPTO_CIPHER_3DES_CBC:
 158                 len = 8;
 159                 break;
 160         case RTE_CRYPTO_CIPHER_3DES_CTR:
 161                 len = 8;
 162                 break;
 163         case RTE_CRYPTO_CIPHER_3DES_ECB:
 164                 len = 8;
 165                 break;
 166         case RTE_CRYPTO_CIPHER_AES_CBC:
 167                 len = 16;
 168                 break;
 169
 170         /* TODO: add common algos */
 171
 172         default:
 173                 len = -1;
 174                 break;
 175         }
 176
 177         return len;
 178 }
 179
 180 /**
 181  * vhost_crypto struct is used to maintain a number of virtio_cryptos and
 182  * one DPDK crypto device that deals with all crypto workloads. It is declared
 183  * here and defined in vhost_crypto.c
 184  */
 185 struct vhost_crypto {
 186         /** Used to lookup DPDK Cryptodev Session based on VIRTIO crypto
 187          *  session ID.
 188          */
 189         struct rte_hash *session_map;
 190         struct rte_mempool *mbuf_pool;
 191         struct rte_mempool *sess_pool;
 192
 193         /** DPDK cryptodev ID */
 194         uint8_t cid;
 195         uint16_t nb_qps;
 196
 197         uint64_t last_session_id;
 198
 199         uint64_t cache_session_id;
 200         struct rte_cryptodev_sym_session *cache_session;
 201         /** socket id for the device */
 202         int socket_id;
 203
 204         struct virtio_net *dev;
 205
 206         uint8_t option;
 207 } __rte_cache_aligned;
 208
 209 struct vhost_crypto_data_req {
 210         struct vring_desc *head;
 211         struct rte_vhost_memory *mem;
 212         struct virtio_crypto_inhdr *inhdr;
 213         struct vhost_virtqueue *vq;
 214         struct vring_desc *wb_desc;
 215         uint16_t wb_len;
 216         uint16_t desc_idx;
 217         uint16_t len;
 218         uint16_t zero_copy;
 219 };
 220
 221 static int
 222 transform_cipher_param(struct rte_crypto_sym_xform *xform,
 223                 VhostUserCryptoSessionParam *param)
 224 {
 225         int ret;
 226
 227         ret = cipher_algo_transform(param->cipher_algo);
 228         if (unlikely(ret < 0))
 229                 return ret;
 230
 231         xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
 232         xform->cipher.algo = (uint32_t)ret;
 233         xform->cipher.key.length = param->cipher_key_len;
 234         if (xform->cipher.key.length > 0)
 235                 xform->cipher.key.data = param->cipher_key_buf;
 236         if (param->dir == VIRTIO_CRYPTO_OP_ENCRYPT)
 237                 xform->cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
 238         else if (param->dir == VIRTIO_CRYPTO_OP_DECRYPT)
 239                 xform->cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
 240         else {
 241                 VC_LOG_DBG("Bad operation type");
 242                 return -VIRTIO_CRYPTO_BADMSG;
 243         }
 244
 245         ret = get_iv_len(xform->cipher.algo);
 246         if (unlikely(ret < 0))
 247                 return ret;
 248         xform->cipher.iv.length = (uint16_t)ret;
 249         xform->cipher.iv.offset = IV_OFFSET;
 250         return 0;
 251 }
 252
 253 static int
 254 transform_chain_param(struct rte_crypto_sym_xform *xforms,
 255                 VhostUserCryptoSessionParam *param)
 256 {
 257         struct rte_crypto_sym_xform *xform_cipher, *xform_auth;
 258         int ret;
 259
 260         switch (param->chaining_dir) {
 261         case VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER:
 262                 xform_auth = xforms;
 263                 xform_cipher = xforms->next;
 264                 xform_cipher->cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
 265                 xform_auth->auth.op = RTE_CRYPTO_AUTH_OP_VERIFY;
 266                 break;
 267         case VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH:
 268                 xform_cipher = xforms;
 269                 xform_auth = xforms->next;
 270                 xform_cipher->cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
 271                 xform_auth->auth.op = RTE_CRYPTO_AUTH_OP_GENERATE;
 272                 break;
 273         default:
 274                 return -VIRTIO_CRYPTO_BADMSG;
 275         }
 276
 277         /* cipher */
 278         ret = cipher_algo_transform(param->cipher_algo);
 279         if (unlikely(ret < 0))
 280                 return ret;
 281         xform_cipher->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
 282         xform_cipher->cipher.algo = (uint32_t)ret;
 283         xform_cipher->cipher.key.length = param->cipher_key_len;
 284         xform_cipher->cipher.key.data = param->cipher_key_buf;
 285         ret = get_iv_len(xform_cipher->cipher.algo);
 286         if (unlikely(ret < 0))
 287                 return ret;
 288         xform_cipher->cipher.iv.length = (uint16_t)ret;
 289         xform_cipher->cipher.iv.offset = IV_OFFSET;
 290
 291         /* auth */
 292         xform_auth->type = RTE_CRYPTO_SYM_XFORM_AUTH;
 293         ret = auth_algo_transform(param->hash_algo);
 294         if (unlikely(ret < 0))
 295                 return ret;
 296         xform_auth->auth.algo = (uint32_t)ret;
 297         xform_auth->auth.digest_length = param->digest_len;
 298         xform_auth->auth.key.length = param->auth_key_len;
 299         xform_auth->auth.key.data = param->auth_key_buf;
 300
 301         return 0;
 302 }
 303
 304 static void
 305 vhost_crypto_create_sess(struct vhost_crypto *vcrypto,
 306                 VhostUserCryptoSessionParam *sess_param)
 307 {
 308         struct rte_crypto_sym_xform xform1 = {0}, xform2 = {0};
 309         struct rte_cryptodev_sym_session *session;
 310         int ret;
 311
 312         switch (sess_param->op_type) {
 313         case VIRTIO_CRYPTO_SYM_OP_NONE:
 314         case VIRTIO_CRYPTO_SYM_OP_CIPHER:
 315                 ret = transform_cipher_param(&xform1, sess_param);
 316                 if (unlikely(ret)) {
 317                         VC_LOG_ERR("Error transform session msg (%i)", ret);
 318                         sess_param->session_id = ret;
 319                         return;
 320                 }
 321                 break;
 322         case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING:
 323                 if (unlikely(sess_param->hash_mode !=
 324                                 VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH)) {
 325                         sess_param->session_id = -VIRTIO_CRYPTO_NOTSUPP;
 326                         VC_LOG_ERR("Error transform session message (%i)",
 327                                         -VIRTIO_CRYPTO_NOTSUPP);
 328                         return;
 329                 }
 330
 331                 xform1.next = &xform2;
 332
 333                 ret = transform_chain_param(&xform1, sess_param);
 334                 if (unlikely(ret)) {
 335                         VC_LOG_ERR("Error transform session message (%i)", ret);
 336                         sess_param->session_id = ret;
 337                         return;
 338                 }
 339
 340                 break;
 341         default:
 342                 VC_LOG_ERR("Algorithm not yet supported");
 343                 sess_param->session_id = -VIRTIO_CRYPTO_NOTSUPP;
 344                 return;
 345         }
 346
 347         session = rte_cryptodev_sym_session_create(vcrypto->sess_pool);
 348         if (!session) {
 349                 VC_LOG_ERR("Failed to create session");
 350                 sess_param->session_id = -VIRTIO_CRYPTO_ERR;
 351                 return;
 352         }
 353
 354         if (rte_cryptodev_sym_session_init(vcrypto->cid, session, &xform1,
 355                         vcrypto->sess_pool) < 0) {
 356                 VC_LOG_ERR("Failed to initialize session");
 357                 sess_param->session_id = -VIRTIO_CRYPTO_ERR;
 358                 return;
 359         }
 360
 361         /* insert hash to map */
 362         if (rte_hash_add_key_data(vcrypto->session_map,
 363                         &vcrypto->last_session_id, session) < 0) {
 364                 VC_LOG_ERR("Failed to insert session to hash table");
 365
 366                 if (rte_cryptodev_sym_session_clear(vcrypto->cid, session) < 0)
 367                         VC_LOG_ERR("Failed to clear session");
 368                 else {
 369                         if (rte_cryptodev_sym_session_free(session) < 0)
 370                                 VC_LOG_ERR("Failed to free session");
 371                 }
 372                 sess_param->session_id = -VIRTIO_CRYPTO_ERR;
 373                 return;
 374         }
 375
 376         VC_LOG_INFO("Session %"PRIu64" created for vdev %i.",
 377                         vcrypto->last_session_id, vcrypto->dev->vid);
 378
 379         sess_param->session_id = vcrypto->last_session_id;
 380         vcrypto->last_session_id++;
 381 }
 382
 383 static int
 384 vhost_crypto_close_sess(struct vhost_crypto *vcrypto, uint64_t session_id)
 385 {
 386         struct rte_cryptodev_sym_session *session;
 387         uint64_t sess_id = session_id;
 388         int ret;
 389
 390         ret = rte_hash_lookup_data(vcrypto->session_map, &sess_id,
 391                         (void **)&session);
 392
 393         if (unlikely(ret < 0)) {
 394                 VC_LOG_ERR("Failed to delete session %"PRIu64".", session_id);
 395                 return -VIRTIO_CRYPTO_INVSESS;
 396         }
 397
 398         if (rte_cryptodev_sym_session_clear(vcrypto->cid, session) < 0) {
 399                 VC_LOG_DBG("Failed to clear session");
 400                 return -VIRTIO_CRYPTO_ERR;
 401         }
 402
 403         if (rte_cryptodev_sym_session_free(session) < 0) {
 404                 VC_LOG_DBG("Failed to free session");
 405                 return -VIRTIO_CRYPTO_ERR;
 406         }
 407
 408         if (rte_hash_del_key(vcrypto->session_map, &sess_id) < 0) {
 409                 VC_LOG_DBG("Failed to delete session from hash table.");
 410                 return -VIRTIO_CRYPTO_ERR;
 411         }
 412
 413         VC_LOG_INFO("Session %"PRIu64" deleted for vdev %i.", sess_id,
 414                         vcrypto->dev->vid);
 415
 416         return 0;
 417 }
 418
 419 static int
 420 vhost_crypto_msg_post_handler(int vid, void *msg, uint32_t *require_reply)
 421 {
 422         struct virtio_net *dev = get_device(vid);
 423         struct vhost_crypto *vcrypto;
 424         VhostUserMsg *vmsg = msg;
 425         int ret = 0;
 426
 427         if (dev == NULL || require_reply == NULL) {
 428                 VC_LOG_ERR("Invalid vid %i", vid);
 429                 return -EINVAL;
 430         }
 431
 432         vcrypto = dev->extern_data;
 433         if (vcrypto == NULL) {
 434                 VC_LOG_ERR("Cannot find required data, is it initialized?");
 435                 return -ENOENT;
 436         }
 437
 438         *require_reply = 0;
 439
 440         if (vmsg->request.master == VHOST_USER_CRYPTO_CREATE_SESS) {
 441                 vhost_crypto_create_sess(vcrypto,
 442                                 &vmsg->payload.crypto_session);
 443                 *require_reply = 1;
 444         } else if (vmsg->request.master == VHOST_USER_CRYPTO_CLOSE_SESS)
 445                 ret = vhost_crypto_close_sess(vcrypto, vmsg->payload.u64);
 446         else
 447                 ret = -EINVAL;
 448
 449         return ret;
 450 }
 451
 452 static __rte_always_inline struct vring_desc *
 453 find_write_desc(struct vring_desc *head, struct vring_desc *desc)
 454 {
 455         if (desc->flags & VRING_DESC_F_WRITE)
 456                 return desc;
 457
 458         while (desc->flags & VRING_DESC_F_NEXT) {
 459                 desc = &head[desc->next];
 460                 if (desc->flags & VRING_DESC_F_WRITE)
 461                         return desc;
 462         }
 463
 464         return NULL;
 465 }
 466
 467 static struct virtio_crypto_inhdr *
 468 reach_inhdr(struct vring_desc *head, struct rte_vhost_memory *mem,
 469                 struct vring_desc *desc)
 470 {
 471         while (desc->flags & VRING_DESC_F_NEXT)
 472                 desc = &head[desc->next];
 473
 474         return GPA_TO_VVA(struct virtio_crypto_inhdr *, mem, desc->addr);
 475 }
 476
 477 static __rte_always_inline int
 478 move_desc(struct vring_desc *head, struct vring_desc **cur_desc,
 479                 uint32_t size)
 480 {
 481         struct vring_desc *desc = *cur_desc;
 482         int left = size;
 483
 484         rte_prefetch0(&head[desc->next]);
 485         left -= desc->len;
 486
 487         while ((desc->flags & VRING_DESC_F_NEXT) && left > 0) {
 488                 desc = &head[desc->next];
 489                 rte_prefetch0(&head[desc->next]);
 490                 left -= desc->len;
 491         }
 492
 493         if (unlikely(left < 0)) {
 494                 VC_LOG_ERR("Incorrect virtio descriptor");
 495                 return -1;
 496         }
 497
 498         *cur_desc = &head[desc->next];
 499         return 0;
 500 }
 501
 502 static int
 503 copy_data(void *dst_data, struct vring_desc *head, struct rte_vhost_memory *mem,
 504                 struct vring_desc **cur_desc, uint32_t size)
 505 {
 506         struct vring_desc *desc = *cur_desc;
 507         uint32_t to_copy;
 508         uint8_t *data = dst_data;
 509         uint8_t *src;
 510         int left = size;
 511
 512         rte_prefetch0(&head[desc->next]);
 513         to_copy = RTE_MIN(desc->len, (uint32_t)left);
 514         src = GPA_TO_VVA(uint8_t *, mem, desc->addr);
 515         rte_memcpy((uint8_t *)data, src, to_copy);
 516         left -= to_copy;
 517
 518         while ((desc->flags & VRING_DESC_F_NEXT) && left > 0) {
 519                 desc = &head[desc->next];
 520                 rte_prefetch0(&head[desc->next]);
 521                 to_copy = RTE_MIN(desc->len, (uint32_t)left);
 522                 src = GPA_TO_VVA(uint8_t *, mem, desc->addr);
 523                 rte_memcpy(data + size - left, src, to_copy);
 524                 left -= to_copy;
 525         }
 526
 527         if (unlikely(left < 0)) {
 528                 VC_LOG_ERR("Incorrect virtio descriptor");
 529                 return -1;
 530         }
 531
 532         *cur_desc = &head[desc->next];
 533
 534         return 0;
 535 }
 536
 537 static __rte_always_inline void *
 538 get_data_ptr(struct vring_desc *head, struct rte_vhost_memory *mem,
 539                 struct vring_desc **cur_desc, uint32_t size)
 540 {
 541         void *data;
 542
 543         data = GPA_TO_VVA(void *, mem, (*cur_desc)->addr);
 544         if (unlikely(!data)) {
 545                 VC_LOG_ERR("Failed to get object");
 546                 return NULL;
 547         }
 548
 549         if (unlikely(move_desc(head, cur_desc, size) < 0))
 550                 return NULL;
 551
 552         return data;
 553 }
 554
 555 static int
 556 write_back_data(struct rte_crypto_op *op, struct vhost_crypto_data_req *vc_req)
 557 {
 558         struct rte_mbuf *mbuf = op->sym->m_dst;
 559         struct vring_desc *head = vc_req->head;
 560         struct rte_vhost_memory *mem = vc_req->mem;
 561         struct vring_desc *desc = vc_req->wb_desc;
 562         int left = vc_req->wb_len;
 563         uint32_t to_write;
 564         uint8_t *src_data = mbuf->buf_addr, *dst;
 565
 566         rte_prefetch0(&head[desc->next]);
 567         to_write = RTE_MIN(desc->len, (uint32_t)left);
 568         dst = GPA_TO_VVA(uint8_t *, mem, desc->addr);
 569         rte_memcpy(dst, src_data, to_write);
 570         left -= to_write;
 571         src_data += to_write;
 572
 573         while ((desc->flags & VRING_DESC_F_NEXT) && left > 0) {
 574                 desc = &head[desc->next];
 575                 rte_prefetch0(&head[desc->next]);
 576                 to_write = RTE_MIN(desc->len, (uint32_t)left);
 577                 dst = GPA_TO_VVA(uint8_t *, mem, desc->addr);
 578                 rte_memcpy(dst, src_data, to_write);
 579                 left -= to_write;
 580                 src_data += to_write;
 581         }
 582
 583         if (unlikely(left < 0)) {
 584                 VC_LOG_ERR("Incorrect virtio descriptor");
 585                 return -1;
 586         }
 587
 588         return 0;
 589 }
 590
 591 static uint8_t
 592 prepare_sym_cipher_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op,
 593                 struct vhost_crypto_data_req *vc_req,
 594                 struct virtio_crypto_cipher_data_req *cipher,
 595                 struct vring_desc *cur_desc)
 596 {
 597         struct vring_desc *head = vc_req->head;
 598         struct vring_desc *desc = cur_desc;
 599         struct rte_vhost_memory *mem = vc_req->mem;
 600         struct rte_mbuf *m_src = op->sym->m_src, *m_dst = op->sym->m_dst;
 601         uint8_t *iv_data = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET);
 602         uint8_t ret = 0;
 603
 604         /* prepare */
 605         /* iv */
 606         if (unlikely(copy_data(iv_data, head, mem, &desc,
 607                         cipher->para.iv_len) < 0)) {
 608                 ret = VIRTIO_CRYPTO_BADMSG;
 609                 goto error_exit;
 610         }
 611
 612         m_src->data_len = cipher->para.src_data_len;
 613
 614         switch (vcrypto->option) {
 615         case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
 616                 m_src->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr,
 617                                 cipher->para.src_data_len);
 618                 m_src->buf_addr = get_data_ptr(head, mem, &desc,
 619                                 cipher->para.src_data_len);
 620                 if (unlikely(m_src->buf_iova == 0 ||
 621                                 m_src->buf_addr == NULL)) {
 622                         VC_LOG_ERR("zero_copy may fail due to cross page data");
 623                         ret = VIRTIO_CRYPTO_ERR;
 624                         goto error_exit;
 625                 }
 626                 break;
 627         case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
 628                 if (unlikely(cipher->para.src_data_len >
 629                                 RTE_MBUF_DEFAULT_BUF_SIZE)) {
 630                         VC_LOG_ERR("Not enough space to do data copy");
 631                         ret = VIRTIO_CRYPTO_ERR;
 632                         goto error_exit;
 633                 }
 634                 if (unlikely(copy_data(rte_pktmbuf_mtod(m_src, uint8_t *), head,
 635                                 mem, &desc, cipher->para.src_data_len))
 636                                 < 0) {
 637                         ret = VIRTIO_CRYPTO_BADMSG;
 638                         goto error_exit;
 639                 }
 640                 break;
 641         default:
 642                 ret = VIRTIO_CRYPTO_BADMSG;
 643                 goto error_exit;
 644         }
 645
 646         /* dst */
 647         desc = find_write_desc(head, desc);
 648         if (unlikely(!desc)) {
 649                 VC_LOG_ERR("Cannot find write location");
 650                 ret = VIRTIO_CRYPTO_BADMSG;
 651                 goto error_exit;
 652         }
 653
 654         switch (vcrypto->option) {
 655         case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
 656                 m_dst->buf_iova = gpa_to_hpa(vcrypto->dev,
 657                                 desc->addr, cipher->para.dst_data_len);
 658                 m_dst->buf_addr = get_data_ptr(head, mem, &desc,
 659                                 cipher->para.dst_data_len);
 660                 if (unlikely(m_dst->buf_iova == 0 || m_dst->buf_addr == NULL)) {
 661                         VC_LOG_ERR("zero_copy may fail due to cross page data");
 662                         ret = VIRTIO_CRYPTO_ERR;
 663                         goto error_exit;
 664                 }
 665
 666                 m_dst->data_len = cipher->para.dst_data_len;
 667                 break;
 668         case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
 669                 vc_req->wb_desc = desc;
 670                 vc_req->wb_len = cipher->para.dst_data_len;
 671                 if (unlikely(move_desc(head, &desc, vc_req->wb_len) < 0)) {
 672                         ret = VIRTIO_CRYPTO_ERR;
 673                         goto error_exit;
 674                 }
 675                 break;
 676         default:
 677                 ret = VIRTIO_CRYPTO_BADMSG;
 678                 goto error_exit;
 679         }
 680
 681         /* src data */
 682         op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 683         op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
 684
 685         op->sym->cipher.data.offset = 0;
 686         op->sym->cipher.data.length = cipher->para.src_data_len;
 687
 688         vc_req->inhdr = get_data_ptr(head, mem, &desc, INHDR_LEN);
 689         if (unlikely(vc_req->inhdr == NULL)) {
 690                 ret = VIRTIO_CRYPTO_BADMSG;
 691                 goto error_exit;
 692         }
 693
 694         vc_req->inhdr->status = VIRTIO_CRYPTO_OK;
 695         vc_req->len = cipher->para.dst_data_len + INHDR_LEN;
 696
 697         return 0;
 698
 699 error_exit:
 700         vc_req->len = INHDR_LEN;
 701         return ret;
 702 }
 703
 704 static uint8_t
 705 prepare_sym_chain_op(struct vhost_crypto *vcrypto, struct rte_crypto_op *op,
 706                 struct vhost_crypto_data_req *vc_req,
 707                 struct virtio_crypto_alg_chain_data_req *chain,
 708                 struct vring_desc *cur_desc)
 709 {
 710         struct vring_desc *head = vc_req->head;
 711         struct vring_desc *desc = cur_desc;
 712         struct rte_vhost_memory *mem = vc_req->mem;
 713         struct rte_mbuf *m_src = op->sym->m_src, *m_dst = op->sym->m_dst;
 714         uint8_t *iv_data = rte_crypto_op_ctod_offset(op, uint8_t *, IV_OFFSET);
 715         uint32_t digest_offset;
 716         void *digest_addr;
 717         uint8_t ret = 0;
 718
 719         /* prepare */
 720         /* iv */
 721         if (unlikely(copy_data(iv_data, head, mem, &desc,
 722                         chain->para.iv_len) < 0)) {
 723                 ret = VIRTIO_CRYPTO_BADMSG;
 724                 goto error_exit;
 725         }
 726
 727         m_src->data_len = chain->para.src_data_len;
 728         m_dst->data_len = chain->para.dst_data_len;
 729
 730         switch (vcrypto->option) {
 731         case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
 732                 m_src->buf_iova = gpa_to_hpa(vcrypto->dev, desc->addr,
 733                                 chain->para.src_data_len);
 734                 m_src->buf_addr = get_data_ptr(head, mem, &desc,
 735                                 chain->para.src_data_len);
 736                 if (unlikely(m_src->buf_iova == 0 || m_src->buf_addr == NULL)) {
 737                         VC_LOG_ERR("zero_copy may fail due to cross page data");
 738                         ret = VIRTIO_CRYPTO_ERR;
 739                         goto error_exit;
 740                 }
 741                 break;
 742         case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
 743                 if (unlikely(chain->para.src_data_len >
 744                                 RTE_MBUF_DEFAULT_BUF_SIZE)) {
 745                         VC_LOG_ERR("Not enough space to do data copy");
 746                         ret = VIRTIO_CRYPTO_ERR;
 747                         goto error_exit;
 748                 }
 749                 if (unlikely(copy_data(rte_pktmbuf_mtod(m_src, uint8_t *), head,
 750                                 mem, &desc, chain->para.src_data_len)) < 0) {
 751                         ret = VIRTIO_CRYPTO_BADMSG;
 752                         goto error_exit;
 753                 }
 754                 break;
 755         default:
 756                 ret = VIRTIO_CRYPTO_BADMSG;
 757                 goto error_exit;
 758         }
 759
 760         /* dst */
 761         desc = find_write_desc(head, desc);
 762         if (unlikely(!desc)) {
 763                 VC_LOG_ERR("Cannot find write location");
 764                 ret = VIRTIO_CRYPTO_BADMSG;
 765                 goto error_exit;
 766         }
 767
 768         switch (vcrypto->option) {
 769         case RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE:
 770                 m_dst->buf_iova = gpa_to_hpa(vcrypto->dev,
 771                                 desc->addr, chain->para.dst_data_len);
 772                 m_dst->buf_addr = get_data_ptr(head, mem, &desc,
 773                                 chain->para.dst_data_len);
 774                 if (unlikely(m_dst->buf_iova == 0 || m_dst->buf_addr == NULL)) {
 775                         VC_LOG_ERR("zero_copy may fail due to cross page data");
 776                         ret = VIRTIO_CRYPTO_ERR;
 777                         goto error_exit;
 778                 }
 779
 780                 op->sym->auth.digest.phys_addr = gpa_to_hpa(vcrypto->dev,
 781                                 desc->addr, chain->para.hash_result_len);
 782                 op->sym->auth.digest.data = get_data_ptr(head, mem, &desc,
 783                                 chain->para.hash_result_len);
 784                 if (unlikely(op->sym->auth.digest.phys_addr == 0)) {
 785                         VC_LOG_ERR("zero_copy may fail due to cross page data");
 786                         ret = VIRTIO_CRYPTO_ERR;
 787                         goto error_exit;
 788                 }
 789                 break;
 790         case RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE:
 791                 digest_offset = m_dst->data_len;
 792                 digest_addr = rte_pktmbuf_mtod_offset(m_dst, void *,
 793                                 digest_offset);
 794
 795                 vc_req->wb_desc = desc;
 796                 vc_req->wb_len = m_dst->data_len + chain->para.hash_result_len;
 797
 798                 if (unlikely(move_desc(head, &desc,
 799                                 chain->para.dst_data_len) < 0)) {
 800                         ret = VIRTIO_CRYPTO_BADMSG;
 801                         goto error_exit;
 802                 }
 803
 804                 if (unlikely(copy_data(digest_addr, head, mem, &desc,
 805                                 chain->para.hash_result_len)) < 0) {
 806                         ret = VIRTIO_CRYPTO_BADMSG;
 807                         goto error_exit;
 808                 }
 809
 810                 op->sym->auth.digest.data = digest_addr;
 811                 op->sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m_dst,
 812                                 digest_offset);
 813                 if (unlikely(move_desc(head, &desc,
 814                                 chain->para.hash_result_len) < 0)) {
 815                         ret = VIRTIO_CRYPTO_ERR;
 816                         goto error_exit;
 817                 }
 818                 break;
 819         default:
 820                 ret = VIRTIO_CRYPTO_BADMSG;
 821                 goto error_exit;
 822         }
 823
 824         /* record inhdr */
 825         vc_req->inhdr = get_data_ptr(head, mem, &desc, INHDR_LEN);
 826         if (unlikely(vc_req->inhdr == NULL)) {
 827                 ret = VIRTIO_CRYPTO_BADMSG;
 828                 goto error_exit;
 829         }
 830
 831         vc_req->inhdr->status = VIRTIO_CRYPTO_OK;
 832
 833         op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 834         op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
 835
 836         op->sym->cipher.data.offset = chain->para.cipher_start_src_offset;
 837         op->sym->cipher.data.length = chain->para.src_data_len -
 838                         chain->para.cipher_start_src_offset;
 839
 840         op->sym->auth.data.offset = chain->para.hash_start_src_offset;
 841         op->sym->auth.data.length = chain->para.len_to_hash;
 842
 843         vc_req->len = chain->para.dst_data_len + chain->para.hash_result_len +
 844                         INHDR_LEN;
 845         return 0;
 846
 847 error_exit:
 848         vc_req->len = INHDR_LEN;
 849         return ret;
 850 }
 851
 852 /**
 853  * Process on descriptor
 854  */
 855 static __rte_always_inline int
 856 vhost_crypto_process_one_req(struct vhost_crypto *vcrypto,
 857                 struct vhost_virtqueue *vq, struct rte_crypto_op *op,
 858                 struct vring_desc *head, uint16_t desc_idx,
 859                 struct rte_vhost_memory *mem)
 860 {
 861         struct vhost_crypto_data_req *vc_req = RTE_PTR_ADD(op->sym->m_src,
 862                         sizeof(struct rte_mbuf));
 863         struct rte_cryptodev_sym_session *session;
 864         struct virtio_crypto_op_data_req *req;
 865         struct virtio_crypto_inhdr *inhdr;
 866         struct vring_desc *desc = NULL;
 867         uint64_t session_id;
 868         int err = 0;
 869
 870         vc_req->desc_idx = desc_idx;
 871
 872         if (likely(head->flags & VRING_DESC_F_INDIRECT)) {
 873                 head = GPA_TO_VVA(struct vring_desc *, mem, head->addr);
 874                 if (unlikely(!head))
 875                         return 0;
 876                 desc_idx = 0;
 877         }
 878
 879         desc = head;
 880
 881         vc_req->mem = mem;
 882         vc_req->head = head;
 883         vc_req->vq = vq;
 884
 885         vc_req->zero_copy = vcrypto->option;
 886
 887         req = get_data_ptr(head, mem, &desc, sizeof(*req));
 888         if (unlikely(req == NULL)) {
 889                 err = VIRTIO_CRYPTO_ERR;
 890                 VC_LOG_ERR("Invalid descriptor");
 891                 goto error_exit;
 892         }
 893
 894         switch (req->header.opcode) {
 895         case VIRTIO_CRYPTO_CIPHER_ENCRYPT:
 896         case VIRTIO_CRYPTO_CIPHER_DECRYPT:
 897                 session_id = req->header.session_id;
 898
 899                 /* one branch to avoid unnecessary table lookup */
 900                 if (vcrypto->cache_session_id != session_id) {
 901                         err = rte_hash_lookup_data(vcrypto->session_map,
 902                                         &session_id, (void **)&session);
 903                         if (unlikely(err < 0)) {
 904                                 err = VIRTIO_CRYPTO_ERR;
 905                                 VC_LOG_ERR("Failed to find session %"PRIu64,
 906                                                 session_id);
 907                                 goto error_exit;
 908                         }
 909
 910                         vcrypto->cache_session = session;
 911                         vcrypto->cache_session_id = session_id;
 912                 }
 913
 914                 session = vcrypto->cache_session;
 915
 916                 err = rte_crypto_op_attach_sym_session(op, session);
 917                 if (unlikely(err < 0)) {
 918                         err = VIRTIO_CRYPTO_ERR;
 919                         VC_LOG_ERR("Failed to attach session to op");
 920                         goto error_exit;
 921                 }
 922
 923                 switch (req->u.sym_req.op_type) {
 924                 case VIRTIO_CRYPTO_SYM_OP_NONE:
 925                         err = VIRTIO_CRYPTO_NOTSUPP;
 926                         break;
 927                 case VIRTIO_CRYPTO_SYM_OP_CIPHER:
 928                         err = prepare_sym_cipher_op(vcrypto, op, vc_req,
 929                                         &req->u.sym_req.u.cipher, desc);
 930                         break;
 931                 case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING:
 932                         err = prepare_sym_chain_op(vcrypto, op, vc_req,
 933                                         &req->u.sym_req.u.chain, desc);
 934                         break;
 935                 }
 936                 if (unlikely(err != 0)) {
 937                         VC_LOG_ERR("Failed to process sym request");
 938                         goto error_exit;
 939                 }
 940                 break;
 941         default:
 942                 VC_LOG_ERR("Unsupported symmetric crypto request type %u",
 943                                 req->header.opcode);
 944                 goto error_exit;
 945         }
 946
 947         return 0;
 948
 949 error_exit:
 950
 951         inhdr = reach_inhdr(head, mem, desc);
 952         if (likely(inhdr != NULL))
 953                 inhdr->status = (uint8_t)err;
 954
 955         return -1;
 956 }
 957
 958 static __rte_always_inline struct vhost_virtqueue *
 959 vhost_crypto_finalize_one_request(struct rte_crypto_op *op,
 960                 struct vhost_virtqueue *old_vq)
 961 {
 962         struct rte_mbuf *m_src = op->sym->m_src;
 963         struct rte_mbuf *m_dst = op->sym->m_dst;
 964         struct vhost_crypto_data_req *vc_req = RTE_PTR_ADD(m_src,
 965                         sizeof(struct rte_mbuf));
 966         uint16_t desc_idx;
 967         int ret = 0;
 968
 969         if (unlikely(!vc_req)) {
 970                 VC_LOG_ERR("Failed to retrieve vc_req");
 971                 return NULL;
 972         }
 973
 974         if (old_vq && (vc_req->vq != old_vq))
 975                 return vc_req->vq;
 976
 977         desc_idx = vc_req->desc_idx;
 978
 979         if (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS))
 980                 vc_req->inhdr->status = VIRTIO_CRYPTO_ERR;
 981         else {
 982                 if (vc_req->zero_copy == 0) {
 983                         ret = write_back_data(op, vc_req);
 984                         if (unlikely(ret != 0))
 985                                 vc_req->inhdr->status = VIRTIO_CRYPTO_ERR;
 986                 }
 987         }
 988
 989         vc_req->vq->used->ring[desc_idx].id = desc_idx;
 990         vc_req->vq->used->ring[desc_idx].len = vc_req->len;
 991
 992         rte_mempool_put(m_dst->pool, (void *)m_dst);
 993         rte_mempool_put(m_src->pool, (void *)m_src);
 994
 995         return vc_req->vq;
 996 }
 997
 998 static __rte_always_inline uint16_t
 999 vhost_crypto_complete_one_vm_requests(struct rte_crypto_op **ops,
1000                 uint16_t nb_ops, int *callfd)
1001 {
1002         uint16_t processed = 1;
1003         struct vhost_virtqueue *vq, *tmp_vq;
1004
1005         if (unlikely(nb_ops == 0))
1006                 return 0;
1007
1008         vq = vhost_crypto_finalize_one_request(ops[0], NULL);
1009         if (unlikely(vq == NULL))
1010                 return 0;
1011         tmp_vq = vq;
1012
1013         while ((processed < nb_ops)) {
1014                 tmp_vq = vhost_crypto_finalize_one_request(ops[processed],
1015                                 tmp_vq);
1016
1017                 if (unlikely(vq != tmp_vq))
1018                         break;
1019
1020                 processed++;
1021         }
1022
1023         *callfd = vq->callfd;
1024
1025         *(volatile uint16_t *)&vq->used->idx += processed;
1026
1027         return processed;
1028 }