common/cnxk: allow building for generic arm64
[dpdk.git] / examples / vhost_crypto / main.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2017-2018 Intel Corporation
3  */
4
5 #include <stdio.h>
6 #include <stdlib.h>
7 #include <string.h>
8 #include <unistd.h>
9 #include <stdbool.h>
10 #include <assert.h>
11 #include <getopt.h>
12
13 #include <rte_malloc.h>
14 #include <rte_cycles.h>
15 #include <rte_vhost.h>
16 #include <rte_cryptodev.h>
17 #include <rte_vhost_crypto.h>
18 #include <rte_string_fns.h>
19
20 #include <cmdline_rdline.h>
21 #include <cmdline_parse.h>
22 #include <cmdline_parse_string.h>
23 #include <cmdline.h>
24
25 #define NB_VIRTIO_QUEUES                (1)
26 #define MAX_PKT_BURST                   (64)
27 #define MAX_IV_LEN                      (32)
28 #define NB_MEMPOOL_OBJS                 (8192)
29 #define NB_CRYPTO_DESCRIPTORS           (4096)
30 #define NB_CACHE_OBJS                   (128)
31 #define SESSION_MAP_ENTRIES             (1024)
32 #define REFRESH_TIME_SEC                (3)
33
34 #define MAX_NB_SOCKETS                  (4)
35 #define MAX_NB_WORKER_CORES             (16)
36
37 struct lcore_option {
38         uint32_t lcore_id;
39         char *socket_files[MAX_NB_SOCKETS];
40         uint32_t nb_sockets;
41         uint8_t cid;
42         uint16_t qid;
43 };
44
45 struct vhost_crypto_info {
46         int vids[MAX_NB_SOCKETS];
47         uint32_t nb_vids;
48         struct rte_mempool *sess_pool;
49         struct rte_mempool *sess_priv_pool;
50         struct rte_mempool *cop_pool;
51         uint8_t cid;
52         uint32_t qid;
53         uint32_t nb_inflight_ops;
54         volatile uint32_t initialized[MAX_NB_SOCKETS];
55 } __rte_cache_aligned;
56
57 struct vhost_crypto_options {
58         struct lcore_option los[MAX_NB_WORKER_CORES];
59         struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES];
60         uint32_t nb_los;
61         uint32_t zero_copy;
62         uint32_t guest_polling;
63 } options;
64
65 enum {
66 #define OPT_CONFIG          "config"
67         OPT_CONFIG_NUM = 256,
68 #define OPT_SOCKET_FILE     "socket-file"
69         OPT_SOCKET_FILE_NUM,
70 #define OPT_ZERO_COPY       "zero-copy"
71         OPT_ZERO_COPY_NUM,
72 #define OPT_POLLING         "guest-polling"
73         OPT_POLLING_NUM,
74 };
75
76 #define NB_SOCKET_FIELDS        (2)
77
78 static uint32_t
79 find_lo(uint32_t lcore_id)
80 {
81         uint32_t i;
82
83         for (i = 0; i < options.nb_los; i++)
84                 if (options.los[i].lcore_id == lcore_id)
85                         return i;
86
87         return UINT32_MAX;
88 }
89
90 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */
91 static int
92 parse_socket_arg(char *arg)
93 {
94         uint32_t nb_sockets;
95         uint32_t lcore_id;
96         char *str_fld[NB_SOCKET_FIELDS];
97         struct lcore_option *lo;
98         uint32_t idx;
99         char *end;
100
101         if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') !=
102                                 NB_SOCKET_FIELDS) {
103                 RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg);
104                 return -EINVAL;
105         }
106
107         errno = 0;
108         lcore_id = strtoul(str_fld[0], &end, 0);
109         if (errno != 0 || end == str_fld[0] || lcore_id > 255)
110                 return -EINVAL;
111
112         idx = find_lo(lcore_id);
113         if (idx == UINT32_MAX) {
114                 if (options.nb_los == MAX_NB_WORKER_CORES)
115                         return -ENOMEM;
116                 lo = &options.los[options.nb_los];
117                 lo->lcore_id = lcore_id;
118                 options.nb_los++;
119         } else
120                 lo = &options.los[idx];
121
122         nb_sockets = lo->nb_sockets;
123
124         if (nb_sockets >= MAX_NB_SOCKETS) {
125                 RTE_LOG(ERR, USER1, "Too many socket files!\n");
126                 return -ENOMEM;
127         }
128
129         lo->socket_files[nb_sockets] = strdup(str_fld[1]);
130         if (!lo->socket_files[nb_sockets]) {
131                 RTE_LOG(ERR, USER1, "Insufficient memory\n");
132                 return -ENOMEM;
133         }
134
135         lo->nb_sockets++;
136
137         return 0;
138 }
139
140 static int
141 parse_config(char *q_arg)
142 {
143         struct lcore_option *lo;
144         char s[256];
145         const char *p, *p0 = q_arg;
146         char *end;
147         enum fieldnames {
148                 FLD_LCORE = 0,
149                 FLD_CID,
150                 FLD_QID,
151                 _NUM_FLD
152         };
153         uint32_t flds[_NUM_FLD];
154         char *str_fld[_NUM_FLD];
155         uint32_t i;
156         uint32_t size;
157
158         while ((p = strchr(p0, '(')) != NULL) {
159                 ++p;
160                 p0 = strchr(p, ')');
161                 if (p0 == NULL)
162                         return -1;
163
164                 size = p0 - p;
165                 if (size >= sizeof(s))
166                         return -1;
167
168                 snprintf(s, sizeof(s), "%.*s", size, p);
169                 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
170                                 _NUM_FLD)
171                         return -1;
172                 for (i = 0; i < _NUM_FLD; i++) {
173                         errno = 0;
174                         flds[i] = strtoul(str_fld[i], &end, 0);
175                         if (errno != 0 || end == str_fld[i] || flds[i] > 255)
176                                 return -EINVAL;
177                 }
178
179                 if (flds[FLD_LCORE] > RTE_MAX_LCORE)
180                         return -EINVAL;
181
182                 i = find_lo(flds[FLD_LCORE]);
183                 if (i == UINT32_MAX) {
184                         if (options.nb_los == MAX_NB_WORKER_CORES)
185                                 return -ENOMEM;
186                         lo = &options.los[options.nb_los];
187                         options.nb_los++;
188                 } else
189                         lo = &options.los[i];
190
191                 lo->lcore_id = flds[FLD_LCORE];
192                 lo->cid = flds[FLD_CID];
193                 lo->qid = flds[FLD_QID];
194         }
195
196         return 0;
197 }
198
199 static void
200 vhost_crypto_usage(const char *prgname)
201 {
202         printf("%s [EAL options] --\n"
203                 "  --%s <lcore>,SOCKET-FILE-PATH\n"
204                 "  --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]\n"
205                 "  --%s: zero copy\n"
206                 "  --%s: guest polling\n",
207                 prgname, OPT_SOCKET_FILE, OPT_CONFIG,
208                 OPT_ZERO_COPY, OPT_POLLING);
209 }
210
211 static int
212 vhost_crypto_parse_args(int argc, char **argv)
213 {
214         int opt, ret;
215         char *prgname = argv[0];
216         char **argvopt;
217         int option_index;
218         struct option lgopts[] = {
219                 {OPT_SOCKET_FILE, required_argument,
220                                 NULL, OPT_SOCKET_FILE_NUM},
221                 {OPT_CONFIG, required_argument,
222                                 NULL, OPT_CONFIG_NUM},
223                 {OPT_ZERO_COPY, no_argument,
224                                 NULL, OPT_ZERO_COPY_NUM},
225                 {OPT_POLLING, no_argument,
226                                 NULL, OPT_POLLING_NUM},
227                 {NULL, 0, 0, 0}
228         };
229
230         argvopt = argv;
231
232         while ((opt = getopt_long(argc, argvopt, "",
233                                   lgopts, &option_index)) != EOF) {
234
235                 if (opt == '?') {
236                         vhost_crypto_usage(prgname);
237                         return -1;
238                 }
239
240                 switch (opt) {
241                 case OPT_SOCKET_FILE_NUM:
242                         ret = parse_socket_arg(optarg);
243                         if (ret < 0) {
244                                 vhost_crypto_usage(prgname);
245                                 return ret;
246                         }
247                         break;
248
249                 case OPT_CONFIG_NUM:
250                         ret = parse_config(optarg);
251                         if (ret < 0) {
252                                 vhost_crypto_usage(prgname);
253                                 return ret;
254                         }
255                         break;
256
257                 case OPT_ZERO_COPY_NUM:
258                         options.zero_copy =
259                                 RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE;
260                         break;
261
262                 case OPT_POLLING_NUM:
263                         options.guest_polling = 1;
264                         break;
265
266                 default:
267                         vhost_crypto_usage(prgname);
268                         return -EINVAL;
269                 }
270         }
271
272         return 0;
273 }
274
275 static int
276 new_device(int vid)
277 {
278         struct vhost_crypto_info *info = NULL;
279         char path[PATH_MAX];
280         uint32_t i, j;
281         int ret;
282
283         ret = rte_vhost_get_ifname(vid, path, PATH_MAX);
284         if (ret) {
285                 RTE_LOG(ERR, USER1, "Cannot find matched socket\n");
286                 return ret;
287         }
288
289         for (i = 0; i < options.nb_los; i++) {
290                 for (j = 0; j < options.los[i].nb_sockets; j++) {
291                         if (strcmp(path, options.los[i].socket_files[j]) == 0) {
292                                 info = options.infos[i];
293                                 break;
294                         }
295                 }
296
297                 if (info)
298                         break;
299         }
300
301         if (!info) {
302                 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n");
303                 return -ENOENT;
304         }
305
306         ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool,
307                         info->sess_priv_pool,
308                         rte_lcore_to_socket_id(options.los[i].lcore_id));
309         if (ret) {
310                 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n");
311                 return ret;
312         }
313
314         ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy);
315         if (ret) {
316                 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n",
317                                 options.zero_copy == 1 ? "enable" : "disable");
318                 return ret;
319         }
320
321         info->vids[j] = vid;
322         info->initialized[j] = 1;
323
324         rte_wmb();
325
326         RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path,
327                         vid);
328         return 0;
329 }
330
331 static void
332 destroy_device(int vid)
333 {
334         struct vhost_crypto_info *info = NULL;
335         uint32_t i, j;
336
337         for (i = 0; i < options.nb_los; i++) {
338                 for (j = 0; j < options.los[i].nb_sockets; j++) {
339                         if (options.infos[i]->vids[j] == vid) {
340                                 info = options.infos[i];
341                                 break;
342                         }
343                 }
344                 if (info)
345                         break;
346         }
347
348         if (!info) {
349                 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n");
350                 return;
351         }
352
353         do {
354
355         } while (info->nb_inflight_ops);
356
357         info->initialized[j] = 0;
358
359         rte_wmb();
360
361         rte_vhost_crypto_free(vid);
362
363         RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid);
364 }
365
366 static const struct rte_vhost_device_ops virtio_crypto_device_ops = {
367         .new_device =  new_device,
368         .destroy_device = destroy_device,
369 };
370
371 static int
372 vhost_crypto_worker(void *arg)
373 {
374         struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
375         struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
376         struct vhost_crypto_info *info = arg;
377         uint16_t nb_callfds;
378         int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS];
379         uint32_t lcore_id = rte_lcore_id();
380         uint32_t burst_size = MAX_PKT_BURST;
381         uint32_t i, j, k;
382         uint32_t to_fetch, fetched;
383
384         int ret = 0;
385
386         RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id);
387
388         for (i = 0; i < NB_VIRTIO_QUEUES; i++) {
389                 if (rte_crypto_op_bulk_alloc(info->cop_pool,
390                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i],
391                                 burst_size) < burst_size) {
392                         RTE_LOG(ERR, USER1, "Failed to alloc cops\n");
393                         ret = -1;
394                         goto exit;
395                 }
396         }
397
398         while (1) {
399                 for (i = 0; i < info->nb_vids; i++) {
400                         if (unlikely(info->initialized[i] == 0))
401                                 continue;
402
403                         for (j = 0; j < NB_VIRTIO_QUEUES; j++) {
404                                 to_fetch = RTE_MIN(burst_size,
405                                                 (NB_CRYPTO_DESCRIPTORS -
406                                                 info->nb_inflight_ops));
407                                 fetched = rte_vhost_crypto_fetch_requests(
408                                                 info->vids[i], j, ops[j],
409                                                 to_fetch);
410                                 info->nb_inflight_ops +=
411                                                 rte_cryptodev_enqueue_burst(
412                                                 info->cid, info->qid, ops[j],
413                                                 fetched);
414                                 if (unlikely(rte_crypto_op_bulk_alloc(
415                                                 info->cop_pool,
416                                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC,
417                                                 ops[j], fetched) < fetched)) {
418                                         RTE_LOG(ERR, USER1, "Failed realloc\n");
419                                         return -1;
420                                 }
421
422                                 fetched = rte_cryptodev_dequeue_burst(
423                                                 info->cid, info->qid,
424                                                 ops_deq[j], RTE_MIN(burst_size,
425                                                 info->nb_inflight_ops));
426                                 fetched = rte_vhost_crypto_finalize_requests(
427                                                 ops_deq[j], fetched, callfds,
428                                                 &nb_callfds);
429
430                                 info->nb_inflight_ops -= fetched;
431
432                                 if (!options.guest_polling) {
433                                         for (k = 0; k < nb_callfds; k++)
434                                                 eventfd_write(callfds[k],
435                                                                 (eventfd_t)1);
436                                 }
437
438                                 rte_mempool_put_bulk(info->cop_pool,
439                                                 (void **)ops_deq[j], fetched);
440                         }
441                 }
442         }
443 exit:
444         return ret;
445 }
446
447 static void
448 free_resource(void)
449 {
450         uint32_t i, j;
451
452         for (i = 0; i < options.nb_los; i++) {
453                 struct lcore_option *lo = &options.los[i];
454                 struct vhost_crypto_info *info = options.infos[i];
455
456                 if (!info)
457                         continue;
458
459                 rte_mempool_free(info->cop_pool);
460                 rte_mempool_free(info->sess_pool);
461                 rte_mempool_free(info->sess_priv_pool);
462
463                 for (j = 0; j < lo->nb_sockets; j++) {
464                         rte_vhost_driver_unregister(lo->socket_files[i]);
465                         free(lo->socket_files[i]);
466                 }
467
468                 rte_free(info);
469         }
470
471         memset(&options, 0, sizeof(options));
472
473         /* clean up the EAL */
474         rte_eal_cleanup();
475 }
476
477 int
478 main(int argc, char *argv[])
479 {
480         struct rte_cryptodev_qp_conf qp_conf;
481         struct rte_cryptodev_config config;
482         struct rte_cryptodev_info dev_info;
483         char name[128];
484         uint32_t i, j, lcore;
485         int ret;
486
487         ret = rte_eal_init(argc, argv);
488         if (ret < 0)
489                 return -1;
490         argc -= ret;
491         argv += ret;
492
493         ret = vhost_crypto_parse_args(argc, argv);
494         if (ret < 0)
495                 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n");
496
497         for (i = 0; i < options.nb_los; i++) {
498                 struct lcore_option *lo = &options.los[i];
499                 struct vhost_crypto_info *info;
500
501                 info = rte_zmalloc_socket(NULL, sizeof(*info),
502                                 RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id(
503                                                 lo->lcore_id));
504                 if (!info) {
505                         ret = -ENOMEM;
506                         goto error_exit;
507                 }
508
509                 info->cid = lo->cid;
510                 info->qid = lo->qid;
511                 info->nb_vids = lo->nb_sockets;
512
513                 rte_cryptodev_info_get(info->cid, &dev_info);
514                 if (options.zero_copy == RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE) {
515 #define VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD     crypto_aesni_mb
516 #define VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD    crypto_aesni_gcm
517                         if (strstr(dev_info.driver_name,
518                                 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD)) ||
519                                 strstr(dev_info.driver_name,
520                                 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD))) {
521                                 RTE_LOG(ERR, USER1, "Cannot enable zero-copy in %s\n",
522                                         dev_info.driver_name);
523                                 ret = -EPERM;
524                                 goto error_exit;
525                         }
526                 }
527
528                 if (dev_info.max_nb_queue_pairs < info->qid + 1) {
529                         RTE_LOG(ERR, USER1, "Number of queues cannot over %u",
530                                         dev_info.max_nb_queue_pairs);
531                         goto error_exit;
532                 }
533
534                 config.nb_queue_pairs = dev_info.max_nb_queue_pairs;
535                 config.socket_id = rte_lcore_to_socket_id(lo->lcore_id);
536                 config.ff_disable = RTE_CRYPTODEV_FF_SECURITY;
537
538                 ret = rte_cryptodev_configure(info->cid, &config);
539                 if (ret < 0) {
540                         RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u",
541                                         info->cid);
542                         goto error_exit;
543                 }
544
545                 snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id);
546                 info->sess_pool = rte_cryptodev_sym_session_pool_create(name,
547                                 SESSION_MAP_ENTRIES, 0, 0, 0,
548                                 rte_lcore_to_socket_id(lo->lcore_id));
549
550                 snprintf(name, 127, "SESS_POOL_PRIV_%u", lo->lcore_id);
551                 info->sess_priv_pool = rte_mempool_create(name,
552                                 SESSION_MAP_ENTRIES,
553                                 rte_cryptodev_sym_get_private_session_size(
554                                 info->cid), 64, 0, NULL, NULL, NULL, NULL,
555                                 rte_lcore_to_socket_id(lo->lcore_id), 0);
556                 if (!info->sess_priv_pool || !info->sess_pool) {
557                         RTE_LOG(ERR, USER1, "Failed to create mempool");
558                         goto error_exit;
559                 }
560
561                 snprintf(name, 127, "COPPOOL_%u", lo->lcore_id);
562                 info->cop_pool = rte_crypto_op_pool_create(name,
563                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS,
564                                 NB_CACHE_OBJS, VHOST_CRYPTO_MAX_IV_LEN,
565                                 rte_lcore_to_socket_id(lo->lcore_id));
566
567                 if (!info->cop_pool) {
568                         RTE_LOG(ERR, USER1, "Failed to create crypto pool");
569                         ret = -ENOMEM;
570                         goto error_exit;
571                 }
572
573                 options.infos[i] = info;
574
575                 qp_conf.nb_descriptors = NB_CRYPTO_DESCRIPTORS;
576                 qp_conf.mp_session = info->sess_pool;
577                 qp_conf.mp_session_private = info->sess_priv_pool;
578
579                 for (j = 0; j < dev_info.max_nb_queue_pairs; j++) {
580                         ret = rte_cryptodev_queue_pair_setup(info->cid, j,
581                                         &qp_conf, rte_lcore_to_socket_id(
582                                                         lo->lcore_id));
583                         if (ret < 0) {
584                                 RTE_LOG(ERR, USER1, "Failed to configure qp\n");
585                                 goto error_exit;
586                         }
587                 }
588         }
589
590         for (i = 0; i < options.nb_los; i++) {
591                 struct lcore_option *lo = &options.los[i];
592                 struct vhost_crypto_info *info = options.infos[i];
593
594                 ret = rte_cryptodev_start(lo->cid);
595                 if (ret < 0) {
596                         RTE_LOG(ERR, USER1, "Failed to start cryptodev\n");
597                         goto error_exit;
598                 }
599
600                 if (rte_eal_remote_launch(vhost_crypto_worker, info,
601                                 lo->lcore_id) < 0) {
602                         RTE_LOG(ERR, USER1, "Failed to start worker lcore");
603                         goto error_exit;
604                 }
605
606                 for (j = 0; j < lo->nb_sockets; j++) {
607                         ret = rte_vhost_driver_register(lo->socket_files[j],
608                                 RTE_VHOST_USER_ASYNC_COPY);
609                         if (ret < 0) {
610                                 RTE_LOG(ERR, USER1, "socket %s already exists\n",
611                                         lo->socket_files[j]);
612                                 goto error_exit;
613                         }
614
615                         rte_vhost_driver_callback_register(lo->socket_files[j],
616                                 &virtio_crypto_device_ops);
617
618                         ret = rte_vhost_crypto_driver_start(
619                                         lo->socket_files[j]);
620                         if (ret < 0)  {
621                                 RTE_LOG(ERR, USER1, "failed to start vhost.\n");
622                                 goto error_exit;
623                         }
624                 }
625         }
626
627         RTE_LCORE_FOREACH(lcore)
628                 rte_eal_wait_lcore(lcore);
629
630         free_resource();
631
632         return 0;
633
634 error_exit:
635
636         free_resource();
637
638         return -1;
639 }