1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2018 Intel Corporation
12 #include <rte_malloc.h>
13 #include <rte_cycles.h>
14 #include <rte_vhost.h>
15 #include <rte_cryptodev.h>
16 #include <rte_vhost_crypto.h>
17 #include <rte_string_fns.h>
19 #include <cmdline_rdline.h>
20 #include <cmdline_parse.h>
21 #include <cmdline_parse_string.h>
24 #define NB_VIRTIO_QUEUES (1)
25 #define MAX_PKT_BURST (64)
26 #define MAX_IV_LEN (32)
27 #define NB_MEMPOOL_OBJS (8192)
28 #define NB_CRYPTO_DESCRIPTORS (4096)
29 #define NB_CACHE_OBJS (128)
30 #define SESSION_MAP_ENTRIES (1024)
31 #define REFRESH_TIME_SEC (3)
33 #define MAX_NB_SOCKETS (4)
34 #define MAX_NB_WORKER_CORES (16)
38 char *socket_files[MAX_NB_SOCKETS];
44 struct vhost_crypto_info {
45 int vids[MAX_NB_SOCKETS];
47 struct rte_mempool *sess_pool;
48 struct rte_mempool *cop_pool;
51 uint32_t nb_inflight_ops;
52 volatile uint32_t initialized[MAX_NB_SOCKETS];
53 } __rte_cache_aligned;
55 struct vhost_crypto_options {
56 struct lcore_option los[MAX_NB_WORKER_CORES];
57 struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES];
60 uint32_t guest_polling;
63 #define CONFIG_KEYWORD "config"
64 #define SOCKET_FILE_KEYWORD "socket-file"
65 #define ZERO_COPY_KEYWORD "zero-copy"
66 #define POLLING_KEYWORD "guest-polling"
68 #define NB_SOCKET_FIELDS (2)
71 find_lo(uint32_t lcore_id)
75 for (i = 0; i < options.nb_los; i++)
76 if (options.los[i].lcore_id == lcore_id)
82 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */
84 parse_socket_arg(char *arg)
88 char *str_fld[NB_SOCKET_FIELDS];
89 struct lcore_option *lo;
93 if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') !=
95 RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg);
100 lcore_id = strtoul(str_fld[0], &end, 0);
101 if (errno != 0 || end == str_fld[0] || lcore_id > 255)
104 idx = find_lo(lcore_id);
105 if (idx == UINT32_MAX) {
106 if (options.nb_los == MAX_NB_WORKER_CORES)
108 lo = &options.los[options.nb_los];
109 lo->lcore_id = lcore_id;
112 lo = &options.los[idx];
114 nb_sockets = lo->nb_sockets;
116 if (nb_sockets >= MAX_NB_SOCKETS) {
117 RTE_LOG(ERR, USER1, "Too many socket files!\n");
121 lo->socket_files[nb_sockets] = strdup(str_fld[1]);
122 if (!lo->socket_files[nb_sockets]) {
123 RTE_LOG(ERR, USER1, "Insufficient memory\n");
133 parse_config(char *q_arg)
135 struct lcore_option *lo;
137 const char *p, *p0 = q_arg;
145 uint32_t flds[_NUM_FLD];
146 char *str_fld[_NUM_FLD];
150 while ((p = strchr(p0, '(')) != NULL) {
157 if (size >= sizeof(s))
160 snprintf(s, sizeof(s), "%.*s", size, p);
161 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
164 for (i = 0; i < _NUM_FLD; i++) {
166 flds[i] = strtoul(str_fld[i], &end, 0);
167 if (errno != 0 || end == str_fld[i] || flds[i] > 255)
171 if (flds[FLD_LCORE] > RTE_MAX_LCORE)
174 i = find_lo(flds[FLD_LCORE]);
175 if (i == UINT32_MAX) {
176 if (options.nb_los == MAX_NB_WORKER_CORES)
178 lo = &options.los[options.nb_los];
181 lo = &options.los[i];
183 lo->lcore_id = flds[FLD_LCORE];
184 lo->cid = flds[FLD_CID];
185 lo->qid = flds[FLD_QID];
192 vhost_crypto_usage(const char *prgname)
194 printf("%s [EAL options] --\n"
195 " --%s <lcore>,SOCKET-FILE-PATH\n"
196 " --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]"
198 " --%s: guest polling\n",
199 prgname, SOCKET_FILE_KEYWORD, CONFIG_KEYWORD,
200 ZERO_COPY_KEYWORD, POLLING_KEYWORD);
204 vhost_crypto_parse_args(int argc, char **argv)
207 char *prgname = argv[0];
210 struct option lgopts[] = {
211 {SOCKET_FILE_KEYWORD, required_argument, 0, 0},
212 {CONFIG_KEYWORD, required_argument, 0, 0},
213 {ZERO_COPY_KEYWORD, no_argument, 0, 0},
214 {POLLING_KEYWORD, no_argument, 0, 0},
220 while ((opt = getopt_long(argc, argvopt, "s:",
221 lgopts, &option_index)) != EOF) {
225 if (strcmp(lgopts[option_index].name,
226 SOCKET_FILE_KEYWORD) == 0) {
227 ret = parse_socket_arg(optarg);
229 vhost_crypto_usage(prgname);
232 } else if (strcmp(lgopts[option_index].name,
233 CONFIG_KEYWORD) == 0) {
234 ret = parse_config(optarg);
236 vhost_crypto_usage(prgname);
239 } else if (strcmp(lgopts[option_index].name,
240 ZERO_COPY_KEYWORD) == 0) {
242 RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE;
243 } else if (strcmp(lgopts[option_index].name,
244 POLLING_KEYWORD) == 0) {
245 options.guest_polling = 1;
247 vhost_crypto_usage(prgname);
262 struct vhost_crypto_info *info = NULL;
267 ret = rte_vhost_get_ifname(vid, path, PATH_MAX);
269 RTE_LOG(ERR, USER1, "Cannot find matched socket\n");
273 for (i = 0; i < options.nb_los; i++) {
274 for (j = 0; j < options.los[i].nb_sockets; j++) {
275 if (strcmp(path, options.los[i].socket_files[j]) == 0) {
276 info = options.infos[i];
286 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n");
290 ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool,
291 rte_lcore_to_socket_id(options.los[i].lcore_id));
293 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n");
297 ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy);
299 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n",
300 options.zero_copy == 1 ? "enable" : "disable");
305 info->initialized[j] = 1;
309 RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path,
315 destroy_device(int vid)
317 struct vhost_crypto_info *info = NULL;
320 for (i = 0; i < options.nb_los; i++) {
321 for (j = 0; j < options.los[i].nb_sockets; j++) {
322 if (options.infos[i]->vids[j] == vid) {
323 info = options.infos[i];
332 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n");
338 } while (info->nb_inflight_ops);
340 info->initialized[j] = 0;
344 rte_vhost_crypto_free(vid);
346 RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid);
349 static const struct vhost_device_ops virtio_crypto_device_ops = {
350 .new_device = new_device,
351 .destroy_device = destroy_device,
354 __attribute__((unused))
355 static void clrscr(void)
357 system("@cls||clear");
361 vhost_crypto_worker(void *arg)
363 struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
364 struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
365 struct vhost_crypto_info *info = arg;
367 int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS];
368 uint32_t lcore_id = rte_lcore_id();
369 uint32_t burst_size = MAX_PKT_BURST;
371 uint32_t to_fetch, fetched;
375 RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id);
377 for (i = 0; i < NB_VIRTIO_QUEUES; i++) {
378 if (rte_crypto_op_bulk_alloc(info->cop_pool,
379 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i],
380 burst_size) < burst_size) {
381 RTE_LOG(ERR, USER1, "Failed to alloc cops\n");
388 for (i = 0; i < info->nb_vids; i++) {
389 if (unlikely(info->initialized[i] == 0))
392 for (j = 0; j < NB_VIRTIO_QUEUES; j++) {
393 to_fetch = RTE_MIN(burst_size,
394 (NB_CRYPTO_DESCRIPTORS -
395 info->nb_inflight_ops));
396 fetched = rte_vhost_crypto_fetch_requests(
397 info->vids[i], j, ops[j],
399 info->nb_inflight_ops +=
400 rte_cryptodev_enqueue_burst(
401 info->cid, info->qid, ops[j],
403 if (unlikely(rte_crypto_op_bulk_alloc(
405 RTE_CRYPTO_OP_TYPE_SYMMETRIC,
406 ops[j], fetched) < fetched)) {
407 RTE_LOG(ERR, USER1, "Failed realloc\n");
411 fetched = rte_cryptodev_dequeue_burst(
412 info->cid, info->qid,
413 ops_deq[j], RTE_MIN(burst_size,
414 info->nb_inflight_ops));
415 fetched = rte_vhost_crypto_finalize_requests(
416 ops_deq[j], fetched, callfds,
419 info->nb_inflight_ops -= fetched;
421 if (!options.guest_polling) {
422 for (k = 0; k < nb_callfds; k++)
423 eventfd_write(callfds[k],
427 rte_mempool_put_bulk(info->cop_pool,
428 (void **)ops_deq[j], fetched);
441 for (i = 0; i < options.nb_los; i++) {
442 struct lcore_option *lo = &options.los[i];
443 struct vhost_crypto_info *info = options.infos[i];
445 rte_mempool_free(info->cop_pool);
446 rte_mempool_free(info->sess_pool);
448 for (j = 0; j < lo->nb_sockets; j++) {
449 rte_vhost_driver_unregister(lo->socket_files[i]);
450 free(lo->socket_files[i]);
456 memset(&options, 0, sizeof(options));
460 main(int argc, char *argv[])
462 struct rte_cryptodev_qp_conf qp_conf = {NB_CRYPTO_DESCRIPTORS};
463 struct rte_cryptodev_config config;
464 struct rte_cryptodev_info dev_info;
466 uint32_t i, j, lcore;
469 ret = rte_eal_init(argc, argv);
475 ret = vhost_crypto_parse_args(argc, argv);
477 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n");
479 for (i = 0; i < options.nb_los; i++) {
480 struct lcore_option *lo = &options.los[i];
481 struct vhost_crypto_info *info;
483 info = rte_zmalloc_socket(NULL, sizeof(*info),
484 RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id(
493 info->nb_vids = lo->nb_sockets;
495 rte_cryptodev_info_get(info->cid, &dev_info);
496 if (dev_info.max_nb_queue_pairs < info->qid + 1) {
497 RTE_LOG(ERR, USER1, "Number of queues cannot over %u",
498 dev_info.max_nb_queue_pairs);
502 config.nb_queue_pairs = dev_info.max_nb_queue_pairs;
503 config.socket_id = rte_lcore_to_socket_id(lo->lcore_id);
505 ret = rte_cryptodev_configure(info->cid, &config);
507 RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u",
512 snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id);
513 info->sess_pool = rte_mempool_create(name, SESSION_MAP_ENTRIES,
514 rte_cryptodev_sym_get_private_session_size(
515 info->cid), 64, 0, NULL, NULL, NULL, NULL,
516 rte_lcore_to_socket_id(lo->lcore_id), 0);
517 if (!info->sess_pool) {
518 RTE_LOG(ERR, USER1, "Failed to create mempool");
522 snprintf(name, 127, "COPPOOL_%u", lo->lcore_id);
523 info->cop_pool = rte_crypto_op_pool_create(name,
524 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS,
526 rte_lcore_to_socket_id(lo->lcore_id));
528 if (!info->cop_pool) {
529 RTE_LOG(ERR, USER1, "Failed to create crypto pool");
534 options.infos[i] = info;
536 for (j = 0; j < dev_info.max_nb_queue_pairs; j++) {
537 ret = rte_cryptodev_queue_pair_setup(info->cid, j,
538 &qp_conf, rte_lcore_to_socket_id(
542 RTE_LOG(ERR, USER1, "Failed to configure qp\n");
548 for (i = 0; i < options.nb_los; i++) {
549 struct lcore_option *lo = &options.los[i];
550 struct vhost_crypto_info *info = options.infos[i];
552 ret = rte_cryptodev_start(lo->cid);
554 RTE_LOG(ERR, USER1, "Failed to start cryptodev\n");
558 if (rte_eal_remote_launch(vhost_crypto_worker, info,
560 RTE_LOG(ERR, USER1, "Failed to start worker lcore");
564 for (j = 0; j < lo->nb_sockets; j++) {
565 ret = rte_vhost_driver_register(lo->socket_files[j],
566 RTE_VHOST_USER_DEQUEUE_ZERO_COPY);
568 RTE_LOG(ERR, USER1, "socket %s already exists\n",
569 lo->socket_files[j]);
573 rte_vhost_driver_callback_register(lo->socket_files[j],
574 &virtio_crypto_device_ops);
576 ret = rte_vhost_driver_start(lo->socket_files[j]);
578 RTE_LOG(ERR, USER1, "failed to start vhost.\n");
584 RTE_LCORE_FOREACH(lcore)
585 rte_eal_wait_lcore(lcore);