examples/vhost_crypto/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017-2018 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <string.h>
   8 #include <unistd.h>
   9 #include <stdbool.h>
  10 #include <assert.h>
  11 #include <getopt.h>
  12
  13 #include <rte_malloc.h>
  14 #include <rte_cycles.h>
  15 #include <rte_vhost.h>
  16 #include <rte_cryptodev.h>
  17 #include <rte_vhost_crypto.h>
  18 #include <rte_string_fns.h>
  19
  20 #include <cmdline_rdline.h>
  21 #include <cmdline_parse.h>
  22 #include <cmdline_parse_string.h>
  23 #include <cmdline.h>
  24
  25 #define NB_VIRTIO_QUEUES                (1)
  26 #define MAX_PKT_BURST                   (64)
  27 #define MAX_IV_LEN                      (32)
  28 #define NB_MEMPOOL_OBJS                 (8192)
  29 #define NB_CRYPTO_DESCRIPTORS           (4096)
  30 #define NB_CACHE_OBJS                   (128)
  31 #define SESSION_MAP_ENTRIES             (1024)
  32 #define REFRESH_TIME_SEC                (3)
  33
  34 #define MAX_NB_SOCKETS                  (4)
  35 #define MAX_NB_WORKER_CORES             (16)
  36
  37 struct lcore_option {
  38         uint32_t lcore_id;
  39         char *socket_files[MAX_NB_SOCKETS];
  40         uint32_t nb_sockets;
  41         uint8_t cid;
  42         uint16_t qid;
  43 };
  44
  45 struct vhost_crypto_info {
  46         int vids[MAX_NB_SOCKETS];
  47         uint32_t nb_vids;
  48         struct rte_mempool *sess_pool;
  49         struct rte_mempool *sess_priv_pool;
  50         struct rte_mempool *cop_pool;
  51         uint8_t cid;
  52         uint32_t qid;
  53         uint32_t nb_inflight_ops;
  54         volatile uint32_t initialized[MAX_NB_SOCKETS];
  55 } __rte_cache_aligned;
  56
  57 struct vhost_crypto_options {
  58         struct lcore_option los[MAX_NB_WORKER_CORES];
  59         struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES];
  60         uint32_t nb_los;
  61         uint32_t zero_copy;
  62         uint32_t guest_polling;
  63 } options;
  64
  65 enum {
  66 #define OPT_CONFIG          "config"
  67         OPT_CONFIG_NUM = 256,
  68 #define OPT_SOCKET_FILE     "socket-file"
  69         OPT_SOCKET_FILE_NUM,
  70 #define OPT_ZERO_COPY       "zero-copy"
  71         OPT_ZERO_COPY_NUM,
  72 #define OPT_POLLING         "guest-polling"
  73         OPT_POLLING_NUM,
  74 };
  75
  76 #define NB_SOCKET_FIELDS        (2)
  77
  78 static uint32_t
  79 find_lo(uint32_t lcore_id)
  80 {
  81         uint32_t i;
  82
  83         for (i = 0; i < options.nb_los; i++)
  84                 if (options.los[i].lcore_id == lcore_id)
  85                         return i;
  86
  87         return UINT32_MAX;
  88 }
  89
  90 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */
  91 static int
  92 parse_socket_arg(char *arg)
  93 {
  94         uint32_t nb_sockets;
  95         uint32_t lcore_id;
  96         char *str_fld[NB_SOCKET_FIELDS];
  97         struct lcore_option *lo;
  98         uint32_t idx;
  99         char *end;
 100
 101         if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') !=
 102                                 NB_SOCKET_FIELDS) {
 103                 RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg);
 104                 return -EINVAL;
 105         }
 106
 107         errno = 0;
 108         lcore_id = strtoul(str_fld[0], &end, 0);
 109         if (errno != 0 || end == str_fld[0] || lcore_id > 255)
 110                 return -EINVAL;
 111
 112         idx = find_lo(lcore_id);
 113         if (idx == UINT32_MAX) {
 114                 if (options.nb_los == MAX_NB_WORKER_CORES)
 115                         return -ENOMEM;
 116                 lo = &options.los[options.nb_los];
 117                 lo->lcore_id = lcore_id;
 118                 options.nb_los++;
 119         } else
 120                 lo = &options.los[idx];
 121
 122         nb_sockets = lo->nb_sockets;
 123
 124         if (nb_sockets >= MAX_NB_SOCKETS) {
 125                 RTE_LOG(ERR, USER1, "Too many socket files!\n");
 126                 return -ENOMEM;
 127         }
 128
 129         lo->socket_files[nb_sockets] = strdup(str_fld[1]);
 130         if (!lo->socket_files[nb_sockets]) {
 131                 RTE_LOG(ERR, USER1, "Insufficient memory\n");
 132                 return -ENOMEM;
 133         }
 134
 135         lo->nb_sockets++;
 136
 137         return 0;
 138 }
 139
 140 static int
 141 parse_config(char *q_arg)
 142 {
 143         struct lcore_option *lo;
 144         char s[256];
 145         const char *p, *p0 = q_arg;
 146         char *end;
 147         enum fieldnames {
 148                 FLD_LCORE = 0,
 149                 FLD_CID,
 150                 FLD_QID,
 151                 _NUM_FLD
 152         };
 153         uint32_t flds[_NUM_FLD];
 154         char *str_fld[_NUM_FLD];
 155         uint32_t i;
 156         uint32_t size;
 157
 158         while ((p = strchr(p0, '(')) != NULL) {
 159                 ++p;
 160                 p0 = strchr(p, ')');
 161                 if (p0 == NULL)
 162                         return -1;
 163
 164                 size = p0 - p;
 165                 if (size >= sizeof(s))
 166                         return -1;
 167
 168                 snprintf(s, sizeof(s), "%.*s", size, p);
 169                 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
 170                                 _NUM_FLD)
 171                         return -1;
 172                 for (i = 0; i < _NUM_FLD; i++) {
 173                         errno = 0;
 174                         flds[i] = strtoul(str_fld[i], &end, 0);
 175                         if (errno != 0 || end == str_fld[i] || flds[i] > 255)
 176                                 return -EINVAL;
 177                 }
 178
 179                 if (flds[FLD_LCORE] > RTE_MAX_LCORE)
 180                         return -EINVAL;
 181
 182                 i = find_lo(flds[FLD_LCORE]);
 183                 if (i == UINT32_MAX) {
 184                         if (options.nb_los == MAX_NB_WORKER_CORES)
 185                                 return -ENOMEM;
 186                         lo = &options.los[options.nb_los];
 187                         options.nb_los++;
 188                 } else
 189                         lo = &options.los[i];
 190
 191                 lo->lcore_id = flds[FLD_LCORE];
 192                 lo->cid = flds[FLD_CID];
 193                 lo->qid = flds[FLD_QID];
 194         }
 195
 196         return 0;
 197 }
 198
 199 static void
 200 vhost_crypto_usage(const char *prgname)
 201 {
 202         printf("%s [EAL options] --\n"
 203                 "  --%s <lcore>,SOCKET-FILE-PATH\n"
 204                 "  --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]\n"
 205                 "  --%s: zero copy\n"
 206                 "  --%s: guest polling\n",
 207                 prgname, OPT_SOCKET_FILE, OPT_CONFIG,
 208                 OPT_ZERO_COPY, OPT_POLLING);
 209 }
 210
 211 static int
 212 vhost_crypto_parse_args(int argc, char **argv)
 213 {
 214         int opt, ret;
 215         char *prgname = argv[0];
 216         char **argvopt;
 217         int option_index;
 218         struct option lgopts[] = {
 219                 {OPT_SOCKET_FILE, required_argument,
 220                                 NULL, OPT_SOCKET_FILE_NUM},
 221                 {OPT_CONFIG, required_argument,
 222                                 NULL, OPT_CONFIG_NUM},
 223                 {OPT_ZERO_COPY, no_argument,
 224                                 NULL, OPT_ZERO_COPY_NUM},
 225                 {OPT_POLLING, no_argument,
 226                                 NULL, OPT_POLLING_NUM},
 227                 {NULL, 0, 0, 0}
 228         };
 229
 230         argvopt = argv;
 231
 232         while ((opt = getopt_long(argc, argvopt, "",
 233                                   lgopts, &option_index)) != EOF) {
 234
 235                 if (opt == '?') {
 236                         vhost_crypto_usage(prgname);
 237                         return -1;
 238                 }
 239
 240                 switch (opt) {
 241                 case OPT_SOCKET_FILE_NUM:
 242                         ret = parse_socket_arg(optarg);
 243                         if (ret < 0) {
 244                                 vhost_crypto_usage(prgname);
 245                                 return ret;
 246                         }
 247                         break;
 248
 249                 case OPT_CONFIG_NUM:
 250                         ret = parse_config(optarg);
 251                         if (ret < 0) {
 252                                 vhost_crypto_usage(prgname);
 253                                 return ret;
 254                         }
 255                         break;
 256
 257                 case OPT_ZERO_COPY_NUM:
 258                         options.zero_copy =
 259                                 RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE;
 260                         break;
 261
 262                 case OPT_POLLING_NUM:
 263                         options.guest_polling = 1;
 264                         break;
 265
 266                 default:
 267                         vhost_crypto_usage(prgname);
 268                         return -EINVAL;
 269                 }
 270         }
 271
 272         return 0;
 273 }
 274
 275 static int
 276 new_device(int vid)
 277 {
 278         struct vhost_crypto_info *info = NULL;
 279         char path[PATH_MAX];
 280         uint32_t i, j;
 281         int ret;
 282
 283         ret = rte_vhost_get_ifname(vid, path, PATH_MAX);
 284         if (ret) {
 285                 RTE_LOG(ERR, USER1, "Cannot find matched socket\n");
 286                 return ret;
 287         }
 288
 289         for (i = 0; i < options.nb_los; i++) {
 290                 for (j = 0; j < options.los[i].nb_sockets; j++) {
 291                         if (strcmp(path, options.los[i].socket_files[j]) == 0) {
 292                                 info = options.infos[i];
 293                                 break;
 294                         }
 295                 }
 296
 297                 if (info)
 298                         break;
 299         }
 300
 301         if (!info) {
 302                 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n");
 303                 return -ENOENT;
 304         }
 305
 306         ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool,
 307                         info->sess_priv_pool,
 308                         rte_lcore_to_socket_id(options.los[i].lcore_id));
 309         if (ret) {
 310                 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n");
 311                 return ret;
 312         }
 313
 314         ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy);
 315         if (ret) {
 316                 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n",
 317                                 options.zero_copy == 1 ? "enable" : "disable");
 318                 return ret;
 319         }
 320
 321         info->vids[j] = vid;
 322         info->initialized[j] = 1;
 323
 324         rte_wmb();
 325
 326         RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path,
 327                         vid);
 328         return 0;
 329 }
 330
 331 static void
 332 destroy_device(int vid)
 333 {
 334         struct vhost_crypto_info *info = NULL;
 335         uint32_t i, j;
 336
 337         for (i = 0; i < options.nb_los; i++) {
 338                 for (j = 0; j < options.los[i].nb_sockets; j++) {
 339                         if (options.infos[i]->vids[j] == vid) {
 340                                 info = options.infos[i];
 341                                 break;
 342                         }
 343                 }
 344                 if (info)
 345                         break;
 346         }
 347
 348         if (!info) {
 349                 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n");
 350                 return;
 351         }
 352
 353         do {
 354
 355         } while (info->nb_inflight_ops);
 356
 357         info->initialized[j] = 0;
 358
 359         rte_wmb();
 360
 361         rte_vhost_crypto_free(vid);
 362
 363         RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid);
 364 }
 365
 366 static const struct vhost_device_ops virtio_crypto_device_ops = {
 367         .new_device =  new_device,
 368         .destroy_device = destroy_device,
 369 };
 370
 371 static int
 372 vhost_crypto_worker(void *arg)
 373 {
 374         struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
 375         struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
 376         struct vhost_crypto_info *info = arg;
 377         uint16_t nb_callfds;
 378         int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS];
 379         uint32_t lcore_id = rte_lcore_id();
 380         uint32_t burst_size = MAX_PKT_BURST;
 381         uint32_t i, j, k;
 382         uint32_t to_fetch, fetched;
 383
 384         int ret = 0;
 385
 386         RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id);
 387
 388         for (i = 0; i < NB_VIRTIO_QUEUES; i++) {
 389                 if (rte_crypto_op_bulk_alloc(info->cop_pool,
 390                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i],
 391                                 burst_size) < burst_size) {
 392                         RTE_LOG(ERR, USER1, "Failed to alloc cops\n");
 393                         ret = -1;
 394                         goto exit;
 395                 }
 396         }
 397
 398         while (1) {
 399                 for (i = 0; i < info->nb_vids; i++) {
 400                         if (unlikely(info->initialized[i] == 0))
 401                                 continue;
 402
 403                         for (j = 0; j < NB_VIRTIO_QUEUES; j++) {
 404                                 to_fetch = RTE_MIN(burst_size,
 405                                                 (NB_CRYPTO_DESCRIPTORS -
 406                                                 info->nb_inflight_ops));
 407                                 fetched = rte_vhost_crypto_fetch_requests(
 408                                                 info->vids[i], j, ops[j],
 409                                                 to_fetch);
 410                                 info->nb_inflight_ops +=
 411                                                 rte_cryptodev_enqueue_burst(
 412                                                 info->cid, info->qid, ops[j],
 413                                                 fetched);
 414                                 if (unlikely(rte_crypto_op_bulk_alloc(
 415                                                 info->cop_pool,
 416                                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 417                                                 ops[j], fetched) < fetched)) {
 418                                         RTE_LOG(ERR, USER1, "Failed realloc\n");
 419                                         return -1;
 420                                 }
 421
 422                                 fetched = rte_cryptodev_dequeue_burst(
 423                                                 info->cid, info->qid,
 424                                                 ops_deq[j], RTE_MIN(burst_size,
 425                                                 info->nb_inflight_ops));
 426                                 fetched = rte_vhost_crypto_finalize_requests(
 427                                                 ops_deq[j], fetched, callfds,
 428                                                 &nb_callfds);
 429
 430                                 info->nb_inflight_ops -= fetched;
 431
 432                                 if (!options.guest_polling) {
 433                                         for (k = 0; k < nb_callfds; k++)
 434                                                 eventfd_write(callfds[k],
 435                                                                 (eventfd_t)1);
 436                                 }
 437
 438                                 rte_mempool_put_bulk(info->cop_pool,
 439                                                 (void **)ops_deq[j], fetched);
 440                         }
 441                 }
 442         }
 443 exit:
 444         return ret;
 445 }
 446
 447 static void
 448 free_resource(void)
 449 {
 450         uint32_t i, j;
 451
 452         for (i = 0; i < options.nb_los; i++) {
 453                 struct lcore_option *lo = &options.los[i];
 454                 struct vhost_crypto_info *info = options.infos[i];
 455
 456                 if (!info)
 457                         continue;
 458
 459                 rte_mempool_free(info->cop_pool);
 460                 rte_mempool_free(info->sess_pool);
 461                 rte_mempool_free(info->sess_priv_pool);
 462
 463                 for (j = 0; j < lo->nb_sockets; j++) {
 464                         rte_vhost_driver_unregister(lo->socket_files[i]);
 465                         free(lo->socket_files[i]);
 466                 }
 467
 468                 rte_free(info);
 469         }
 470
 471         memset(&options, 0, sizeof(options));
 472 }
 473
 474 int
 475 main(int argc, char *argv[])
 476 {
 477         struct rte_cryptodev_qp_conf qp_conf;
 478         struct rte_cryptodev_config config;
 479         struct rte_cryptodev_info dev_info;
 480         char name[128];
 481         uint32_t i, j, lcore;
 482         int ret;
 483
 484         ret = rte_eal_init(argc, argv);
 485         if (ret < 0)
 486                 return -1;
 487         argc -= ret;
 488         argv += ret;
 489
 490         ret = vhost_crypto_parse_args(argc, argv);
 491         if (ret < 0)
 492                 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n");
 493
 494         for (i = 0; i < options.nb_los; i++) {
 495                 struct lcore_option *lo = &options.los[i];
 496                 struct vhost_crypto_info *info;
 497
 498                 info = rte_zmalloc_socket(NULL, sizeof(*info),
 499                                 RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id(
 500                                                 lo->lcore_id));
 501                 if (!info) {
 502                         ret = -ENOMEM;
 503                         goto error_exit;
 504                 }
 505
 506                 info->cid = lo->cid;
 507                 info->qid = lo->qid;
 508                 info->nb_vids = lo->nb_sockets;
 509
 510                 rte_cryptodev_info_get(info->cid, &dev_info);
 511                 if (options.zero_copy == RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE) {
 512 #define VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD     crypto_aesni_mb
 513 #define VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD    crypto_aesni_gcm
 514                         if (strstr(dev_info.driver_name,
 515                                 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD)) ||
 516                                 strstr(dev_info.driver_name,
 517                                 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD))) {
 518                                 RTE_LOG(ERR, USER1, "Cannot enable zero-copy in %s\n",
 519                                         dev_info.driver_name);
 520                                 ret = -EPERM;
 521                                 goto error_exit;
 522                         }
 523                 }
 524
 525                 if (dev_info.max_nb_queue_pairs < info->qid + 1) {
 526                         RTE_LOG(ERR, USER1, "Number of queues cannot over %u",
 527                                         dev_info.max_nb_queue_pairs);
 528                         goto error_exit;
 529                 }
 530
 531                 config.nb_queue_pairs = dev_info.max_nb_queue_pairs;
 532                 config.socket_id = rte_lcore_to_socket_id(lo->lcore_id);
 533                 config.ff_disable = RTE_CRYPTODEV_FF_SECURITY;
 534
 535                 ret = rte_cryptodev_configure(info->cid, &config);
 536                 if (ret < 0) {
 537                         RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u",
 538                                         info->cid);
 539                         goto error_exit;
 540                 }
 541
 542                 snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id);
 543                 info->sess_pool = rte_cryptodev_sym_session_pool_create(name,
 544                                 SESSION_MAP_ENTRIES, 0, 0, 0,
 545                                 rte_lcore_to_socket_id(lo->lcore_id));
 546
 547                 snprintf(name, 127, "SESS_POOL_PRIV_%u", lo->lcore_id);
 548                 info->sess_priv_pool = rte_mempool_create(name,
 549                                 SESSION_MAP_ENTRIES,
 550                                 rte_cryptodev_sym_get_private_session_size(
 551                                 info->cid), 64, 0, NULL, NULL, NULL, NULL,
 552                                 rte_lcore_to_socket_id(lo->lcore_id), 0);
 553                 if (!info->sess_priv_pool || !info->sess_pool) {
 554                         RTE_LOG(ERR, USER1, "Failed to create mempool");
 555                         goto error_exit;
 556                 }
 557
 558                 snprintf(name, 127, "COPPOOL_%u", lo->lcore_id);
 559                 info->cop_pool = rte_crypto_op_pool_create(name,
 560                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS,
 561                                 NB_CACHE_OBJS, VHOST_CRYPTO_MAX_IV_LEN,
 562                                 rte_lcore_to_socket_id(lo->lcore_id));
 563
 564                 if (!info->cop_pool) {
 565                         RTE_LOG(ERR, USER1, "Failed to create crypto pool");
 566                         ret = -ENOMEM;
 567                         goto error_exit;
 568                 }
 569
 570                 options.infos[i] = info;
 571
 572                 qp_conf.nb_descriptors = NB_CRYPTO_DESCRIPTORS;
 573                 qp_conf.mp_session = info->sess_pool;
 574                 qp_conf.mp_session_private = info->sess_priv_pool;
 575
 576                 for (j = 0; j < dev_info.max_nb_queue_pairs; j++) {
 577                         ret = rte_cryptodev_queue_pair_setup(info->cid, j,
 578                                         &qp_conf, rte_lcore_to_socket_id(
 579                                                         lo->lcore_id));
 580                         if (ret < 0) {
 581                                 RTE_LOG(ERR, USER1, "Failed to configure qp\n");
 582                                 goto error_exit;
 583                         }
 584                 }
 585         }
 586
 587         for (i = 0; i < options.nb_los; i++) {
 588                 struct lcore_option *lo = &options.los[i];
 589                 struct vhost_crypto_info *info = options.infos[i];
 590
 591                 ret = rte_cryptodev_start(lo->cid);
 592                 if (ret < 0) {
 593                         RTE_LOG(ERR, USER1, "Failed to start cryptodev\n");
 594                         goto error_exit;
 595                 }
 596
 597                 if (rte_eal_remote_launch(vhost_crypto_worker, info,
 598                                 lo->lcore_id) < 0) {
 599                         RTE_LOG(ERR, USER1, "Failed to start worker lcore");
 600                         goto error_exit;
 601                 }
 602
 603                 for (j = 0; j < lo->nb_sockets; j++) {
 604                         ret = rte_vhost_driver_register(lo->socket_files[j],
 605                                 RTE_VHOST_USER_ASYNC_COPY);
 606                         if (ret < 0) {
 607                                 RTE_LOG(ERR, USER1, "socket %s already exists\n",
 608                                         lo->socket_files[j]);
 609                                 goto error_exit;
 610                         }
 611
 612                         rte_vhost_driver_callback_register(lo->socket_files[j],
 613                                 &virtio_crypto_device_ops);
 614
 615                         ret = rte_vhost_crypto_driver_start(
 616                                         lo->socket_files[j]);
 617                         if (ret < 0)  {
 618                                 RTE_LOG(ERR, USER1, "failed to start vhost.\n");
 619                                 goto error_exit;
 620                         }
 621                 }
 622         }
 623
 624         RTE_LCORE_FOREACH(lcore)
 625                 rte_eal_wait_lcore(lcore);
 626
 627         free_resource();
 628
 629         return 0;
 630
 631 error_exit:
 632
 633         free_resource();
 634
 635         return -1;
 636 }