mempool/octeontx2: add fast path mempool ops
[dpdk.git] / examples / vhost_crypto / main.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2017-2018 Intel Corporation
3  */
4
5 #include <stdio.h>
6 #include <stdlib.h>
7 #include <string.h>
8 #include <unistd.h>
9 #include <stdbool.h>
10 #include <assert.h>
11 #include <getopt.h>
12
13 #include <rte_malloc.h>
14 #include <rte_cycles.h>
15 #include <rte_vhost.h>
16 #include <rte_cryptodev.h>
17 #include <rte_vhost_crypto.h>
18 #include <rte_string_fns.h>
19
20 #include <cmdline_rdline.h>
21 #include <cmdline_parse.h>
22 #include <cmdline_parse_string.h>
23 #include <cmdline.h>
24
25 #define NB_VIRTIO_QUEUES                (1)
26 #define MAX_PKT_BURST                   (64)
27 #define MAX_IV_LEN                      (32)
28 #define NB_MEMPOOL_OBJS                 (8192)
29 #define NB_CRYPTO_DESCRIPTORS           (4096)
30 #define NB_CACHE_OBJS                   (128)
31 #define SESSION_MAP_ENTRIES             (1024)
32 #define REFRESH_TIME_SEC                (3)
33
34 #define MAX_NB_SOCKETS                  (4)
35 #define MAX_NB_WORKER_CORES             (16)
36
37 struct lcore_option {
38         uint32_t lcore_id;
39         char *socket_files[MAX_NB_SOCKETS];
40         uint32_t nb_sockets;
41         uint8_t cid;
42         uint16_t qid;
43 };
44
45 struct vhost_crypto_info {
46         int vids[MAX_NB_SOCKETS];
47         uint32_t nb_vids;
48         struct rte_mempool *sess_pool;
49         struct rte_mempool *sess_priv_pool;
50         struct rte_mempool *cop_pool;
51         uint8_t cid;
52         uint32_t qid;
53         uint32_t nb_inflight_ops;
54         volatile uint32_t initialized[MAX_NB_SOCKETS];
55 } __rte_cache_aligned;
56
57 struct vhost_crypto_options {
58         struct lcore_option los[MAX_NB_WORKER_CORES];
59         struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES];
60         uint32_t nb_los;
61         uint32_t zero_copy;
62         uint32_t guest_polling;
63 } options;
64
65 #define CONFIG_KEYWORD          "config"
66 #define SOCKET_FILE_KEYWORD     "socket-file"
67 #define ZERO_COPY_KEYWORD       "zero-copy"
68 #define POLLING_KEYWORD         "guest-polling"
69
70 #define NB_SOCKET_FIELDS        (2)
71
72 static uint32_t
73 find_lo(uint32_t lcore_id)
74 {
75         uint32_t i;
76
77         for (i = 0; i < options.nb_los; i++)
78                 if (options.los[i].lcore_id == lcore_id)
79                         return i;
80
81         return UINT32_MAX;
82 }
83
84 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */
85 static int
86 parse_socket_arg(char *arg)
87 {
88         uint32_t nb_sockets;
89         uint32_t lcore_id;
90         char *str_fld[NB_SOCKET_FIELDS];
91         struct lcore_option *lo;
92         uint32_t idx;
93         char *end;
94
95         if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') !=
96                                 NB_SOCKET_FIELDS) {
97                 RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg);
98                 return -EINVAL;
99         }
100
101         errno = 0;
102         lcore_id = strtoul(str_fld[0], &end, 0);
103         if (errno != 0 || end == str_fld[0] || lcore_id > 255)
104                 return -EINVAL;
105
106         idx = find_lo(lcore_id);
107         if (idx == UINT32_MAX) {
108                 if (options.nb_los == MAX_NB_WORKER_CORES)
109                         return -ENOMEM;
110                 lo = &options.los[options.nb_los];
111                 lo->lcore_id = lcore_id;
112                 options.nb_los++;
113         } else
114                 lo = &options.los[idx];
115
116         nb_sockets = lo->nb_sockets;
117
118         if (nb_sockets >= MAX_NB_SOCKETS) {
119                 RTE_LOG(ERR, USER1, "Too many socket files!\n");
120                 return -ENOMEM;
121         }
122
123         lo->socket_files[nb_sockets] = strdup(str_fld[1]);
124         if (!lo->socket_files[nb_sockets]) {
125                 RTE_LOG(ERR, USER1, "Insufficient memory\n");
126                 return -ENOMEM;
127         }
128
129         lo->nb_sockets++;
130
131         return 0;
132 }
133
134 static int
135 parse_config(char *q_arg)
136 {
137         struct lcore_option *lo;
138         char s[256];
139         const char *p, *p0 = q_arg;
140         char *end;
141         enum fieldnames {
142                 FLD_LCORE = 0,
143                 FLD_CID,
144                 FLD_QID,
145                 _NUM_FLD
146         };
147         uint32_t flds[_NUM_FLD];
148         char *str_fld[_NUM_FLD];
149         uint32_t i;
150         uint32_t size;
151
152         while ((p = strchr(p0, '(')) != NULL) {
153                 ++p;
154                 p0 = strchr(p, ')');
155                 if (p0 == NULL)
156                         return -1;
157
158                 size = p0 - p;
159                 if (size >= sizeof(s))
160                         return -1;
161
162                 snprintf(s, sizeof(s), "%.*s", size, p);
163                 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
164                                 _NUM_FLD)
165                         return -1;
166                 for (i = 0; i < _NUM_FLD; i++) {
167                         errno = 0;
168                         flds[i] = strtoul(str_fld[i], &end, 0);
169                         if (errno != 0 || end == str_fld[i] || flds[i] > 255)
170                                 return -EINVAL;
171                 }
172
173                 if (flds[FLD_LCORE] > RTE_MAX_LCORE)
174                         return -EINVAL;
175
176                 i = find_lo(flds[FLD_LCORE]);
177                 if (i == UINT32_MAX) {
178                         if (options.nb_los == MAX_NB_WORKER_CORES)
179                                 return -ENOMEM;
180                         lo = &options.los[options.nb_los];
181                         options.nb_los++;
182                 } else
183                         lo = &options.los[i];
184
185                 lo->lcore_id = flds[FLD_LCORE];
186                 lo->cid = flds[FLD_CID];
187                 lo->qid = flds[FLD_QID];
188         }
189
190         return 0;
191 }
192
193 static void
194 vhost_crypto_usage(const char *prgname)
195 {
196         printf("%s [EAL options] --\n"
197                 "  --%s <lcore>,SOCKET-FILE-PATH\n"
198                 "  --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]"
199                 "  --%s: zero copy\n"
200                 "  --%s: guest polling\n",
201                 prgname, SOCKET_FILE_KEYWORD, CONFIG_KEYWORD,
202                 ZERO_COPY_KEYWORD, POLLING_KEYWORD);
203 }
204
205 static int
206 vhost_crypto_parse_args(int argc, char **argv)
207 {
208         int opt, ret;
209         char *prgname = argv[0];
210         char **argvopt;
211         int option_index;
212         struct option lgopts[] = {
213                         {SOCKET_FILE_KEYWORD, required_argument, 0, 0},
214                         {CONFIG_KEYWORD, required_argument, 0, 0},
215                         {ZERO_COPY_KEYWORD, no_argument, 0, 0},
216                         {POLLING_KEYWORD, no_argument, 0, 0},
217                         {NULL, 0, 0, 0}
218         };
219
220         argvopt = argv;
221
222         while ((opt = getopt_long(argc, argvopt, "s:",
223                                   lgopts, &option_index)) != EOF) {
224
225                 switch (opt) {
226                 case 0:
227                         if (strcmp(lgopts[option_index].name,
228                                         SOCKET_FILE_KEYWORD) == 0) {
229                                 ret = parse_socket_arg(optarg);
230                                 if (ret < 0) {
231                                         vhost_crypto_usage(prgname);
232                                         return ret;
233                                 }
234                         } else if (strcmp(lgopts[option_index].name,
235                                         CONFIG_KEYWORD) == 0) {
236                                 ret = parse_config(optarg);
237                                 if (ret < 0) {
238                                         vhost_crypto_usage(prgname);
239                                         return ret;
240                                 }
241                         } else if (strcmp(lgopts[option_index].name,
242                                         ZERO_COPY_KEYWORD) == 0) {
243                                 options.zero_copy =
244                                         RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE;
245                         } else if (strcmp(lgopts[option_index].name,
246                                         POLLING_KEYWORD) == 0) {
247                                 options.guest_polling = 1;
248                         } else {
249                                 vhost_crypto_usage(prgname);
250                                 return -EINVAL;
251                         }
252                         break;
253                 default:
254                         return -1;
255                 }
256         }
257
258         return 0;
259 }
260
261 static int
262 new_device(int vid)
263 {
264         struct vhost_crypto_info *info = NULL;
265         char path[PATH_MAX];
266         uint32_t i, j;
267         int ret;
268
269         ret = rte_vhost_get_ifname(vid, path, PATH_MAX);
270         if (ret) {
271                 RTE_LOG(ERR, USER1, "Cannot find matched socket\n");
272                 return ret;
273         }
274
275         for (i = 0; i < options.nb_los; i++) {
276                 for (j = 0; j < options.los[i].nb_sockets; j++) {
277                         if (strcmp(path, options.los[i].socket_files[j]) == 0) {
278                                 info = options.infos[i];
279                                 break;
280                         }
281                 }
282
283                 if (info)
284                         break;
285         }
286
287         if (!info) {
288                 RTE_LOG(ERR, USER1, "Cannot find recorded socket\n");
289                 return -ENOENT;
290         }
291
292         ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool,
293                         info->sess_priv_pool,
294                         rte_lcore_to_socket_id(options.los[i].lcore_id));
295         if (ret) {
296                 RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n");
297                 return ret;
298         }
299
300         ret = rte_vhost_crypto_set_zero_copy(vid, options.zero_copy);
301         if (ret) {
302                 RTE_LOG(ERR, USER1, "Cannot %s zero copy feature\n",
303                                 options.zero_copy == 1 ? "enable" : "disable");
304                 return ret;
305         }
306
307         info->vids[j] = vid;
308         info->initialized[j] = 1;
309
310         rte_wmb();
311
312         RTE_LOG(INFO, USER1, "New Vhost-crypto Device %s, Device ID %d\n", path,
313                         vid);
314         return 0;
315 }
316
317 static void
318 destroy_device(int vid)
319 {
320         struct vhost_crypto_info *info = NULL;
321         uint32_t i, j;
322
323         for (i = 0; i < options.nb_los; i++) {
324                 for (j = 0; j < options.los[i].nb_sockets; j++) {
325                         if (options.infos[i]->vids[j] == vid) {
326                                 info = options.infos[i];
327                                 break;
328                         }
329                 }
330                 if (info)
331                         break;
332         }
333
334         if (!info) {
335                 RTE_LOG(ERR, USER1, "Cannot find socket file from list\n");
336                 return;
337         }
338
339         do {
340
341         } while (info->nb_inflight_ops);
342
343         info->initialized[j] = 0;
344
345         rte_wmb();
346
347         rte_vhost_crypto_free(vid);
348
349         RTE_LOG(INFO, USER1, "Vhost Crypto Device %i Removed\n", vid);
350 }
351
352 static const struct vhost_device_ops virtio_crypto_device_ops = {
353         .new_device =  new_device,
354         .destroy_device = destroy_device,
355 };
356
357 static int
358 vhost_crypto_worker(void *arg)
359 {
360         struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
361         struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
362         struct vhost_crypto_info *info = arg;
363         uint16_t nb_callfds;
364         int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS];
365         uint32_t lcore_id = rte_lcore_id();
366         uint32_t burst_size = MAX_PKT_BURST;
367         uint32_t i, j, k;
368         uint32_t to_fetch, fetched;
369
370         int ret = 0;
371
372         RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id);
373
374         for (i = 0; i < NB_VIRTIO_QUEUES; i++) {
375                 if (rte_crypto_op_bulk_alloc(info->cop_pool,
376                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i],
377                                 burst_size) < burst_size) {
378                         RTE_LOG(ERR, USER1, "Failed to alloc cops\n");
379                         ret = -1;
380                         goto exit;
381                 }
382         }
383
384         while (1) {
385                 for (i = 0; i < info->nb_vids; i++) {
386                         if (unlikely(info->initialized[i] == 0))
387                                 continue;
388
389                         for (j = 0; j < NB_VIRTIO_QUEUES; j++) {
390                                 to_fetch = RTE_MIN(burst_size,
391                                                 (NB_CRYPTO_DESCRIPTORS -
392                                                 info->nb_inflight_ops));
393                                 fetched = rte_vhost_crypto_fetch_requests(
394                                                 info->vids[i], j, ops[j],
395                                                 to_fetch);
396                                 info->nb_inflight_ops +=
397                                                 rte_cryptodev_enqueue_burst(
398                                                 info->cid, info->qid, ops[j],
399                                                 fetched);
400                                 if (unlikely(rte_crypto_op_bulk_alloc(
401                                                 info->cop_pool,
402                                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC,
403                                                 ops[j], fetched) < fetched)) {
404                                         RTE_LOG(ERR, USER1, "Failed realloc\n");
405                                         return -1;
406                                 }
407
408                                 fetched = rte_cryptodev_dequeue_burst(
409                                                 info->cid, info->qid,
410                                                 ops_deq[j], RTE_MIN(burst_size,
411                                                 info->nb_inflight_ops));
412                                 fetched = rte_vhost_crypto_finalize_requests(
413                                                 ops_deq[j], fetched, callfds,
414                                                 &nb_callfds);
415
416                                 info->nb_inflight_ops -= fetched;
417
418                                 if (!options.guest_polling) {
419                                         for (k = 0; k < nb_callfds; k++)
420                                                 eventfd_write(callfds[k],
421                                                                 (eventfd_t)1);
422                                 }
423
424                                 rte_mempool_put_bulk(info->cop_pool,
425                                                 (void **)ops_deq[j], fetched);
426                         }
427                 }
428         }
429 exit:
430         return ret;
431 }
432
433 static void
434 free_resource(void)
435 {
436         uint32_t i, j;
437
438         for (i = 0; i < options.nb_los; i++) {
439                 struct lcore_option *lo = &options.los[i];
440                 struct vhost_crypto_info *info = options.infos[i];
441
442                 if (!info)
443                         continue;
444
445                 rte_mempool_free(info->cop_pool);
446                 rte_mempool_free(info->sess_pool);
447                 rte_mempool_free(info->sess_priv_pool);
448
449                 for (j = 0; j < lo->nb_sockets; j++) {
450                         rte_vhost_driver_unregister(lo->socket_files[i]);
451                         free(lo->socket_files[i]);
452                 }
453
454                 rte_free(info);
455         }
456
457         memset(&options, 0, sizeof(options));
458 }
459
460 int
461 main(int argc, char *argv[])
462 {
463         struct rte_cryptodev_qp_conf qp_conf;
464         struct rte_cryptodev_config config;
465         struct rte_cryptodev_info dev_info;
466         char name[128];
467         uint32_t i, j, lcore;
468         int ret;
469
470         ret = rte_eal_init(argc, argv);
471         if (ret < 0)
472                 return -1;
473         argc -= ret;
474         argv += ret;
475
476         ret = vhost_crypto_parse_args(argc, argv);
477         if (ret < 0)
478                 rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n");
479
480         for (i = 0; i < options.nb_los; i++) {
481                 struct lcore_option *lo = &options.los[i];
482                 struct vhost_crypto_info *info;
483
484                 info = rte_zmalloc_socket(NULL, sizeof(*info),
485                                 RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id(
486                                                 lo->lcore_id));
487                 if (!info) {
488                         ret = -ENOMEM;
489                         goto error_exit;
490                 }
491
492                 info->cid = lo->cid;
493                 info->qid = lo->qid;
494                 info->nb_vids = lo->nb_sockets;
495
496                 rte_cryptodev_info_get(info->cid, &dev_info);
497                 if (options.zero_copy == RTE_VHOST_CRYPTO_ZERO_COPY_ENABLE) {
498 #define VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD     crypto_aesni_mb
499 #define VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD    crypto_aesni_gcm
500                         if (strstr(dev_info.driver_name,
501                                 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_MB_PMD)) ||
502                                 strstr(dev_info.driver_name,
503                                 RTE_STR(VHOST_CRYPTO_CDEV_NAME_AESNI_GCM_PMD))) {
504                                 RTE_LOG(ERR, USER1, "Cannot enable zero-copy in %s\n",
505                                         dev_info.driver_name);
506                                 ret = -EPERM;
507                                 goto error_exit;
508                         }
509                 }
510
511                 if (dev_info.max_nb_queue_pairs < info->qid + 1) {
512                         RTE_LOG(ERR, USER1, "Number of queues cannot over %u",
513                                         dev_info.max_nb_queue_pairs);
514                         goto error_exit;
515                 }
516
517                 config.nb_queue_pairs = dev_info.max_nb_queue_pairs;
518                 config.socket_id = rte_lcore_to_socket_id(lo->lcore_id);
519
520                 ret = rte_cryptodev_configure(info->cid, &config);
521                 if (ret < 0) {
522                         RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u",
523                                         info->cid);
524                         goto error_exit;
525                 }
526
527                 snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id);
528                 info->sess_pool = rte_cryptodev_sym_session_pool_create(name,
529                                 SESSION_MAP_ENTRIES, 0, 0, 0,
530                                 rte_lcore_to_socket_id(lo->lcore_id));
531
532                 snprintf(name, 127, "SESS_POOL_PRIV_%u", lo->lcore_id);
533                 info->sess_priv_pool = rte_mempool_create(name,
534                                 SESSION_MAP_ENTRIES,
535                                 rte_cryptodev_sym_get_private_session_size(
536                                 info->cid), 64, 0, NULL, NULL, NULL, NULL,
537                                 rte_lcore_to_socket_id(lo->lcore_id), 0);
538                 if (!info->sess_priv_pool || !info->sess_pool) {
539                         RTE_LOG(ERR, USER1, "Failed to create mempool");
540                         goto error_exit;
541                 }
542
543                 snprintf(name, 127, "COPPOOL_%u", lo->lcore_id);
544                 info->cop_pool = rte_crypto_op_pool_create(name,
545                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS,
546                                 NB_CACHE_OBJS, 0,
547                                 rte_lcore_to_socket_id(lo->lcore_id));
548
549                 if (!info->cop_pool) {
550                         RTE_LOG(ERR, USER1, "Failed to create crypto pool");
551                         ret = -ENOMEM;
552                         goto error_exit;
553                 }
554
555                 options.infos[i] = info;
556
557                 qp_conf.nb_descriptors = NB_CRYPTO_DESCRIPTORS;
558                 qp_conf.mp_session = info->sess_pool;
559                 qp_conf.mp_session_private = info->sess_priv_pool;
560
561                 for (j = 0; j < dev_info.max_nb_queue_pairs; j++) {
562                         ret = rte_cryptodev_queue_pair_setup(info->cid, j,
563                                         &qp_conf, rte_lcore_to_socket_id(
564                                                         lo->lcore_id));
565                         if (ret < 0) {
566                                 RTE_LOG(ERR, USER1, "Failed to configure qp\n");
567                                 goto error_exit;
568                         }
569                 }
570         }
571
572         for (i = 0; i < options.nb_los; i++) {
573                 struct lcore_option *lo = &options.los[i];
574                 struct vhost_crypto_info *info = options.infos[i];
575
576                 ret = rte_cryptodev_start(lo->cid);
577                 if (ret < 0) {
578                         RTE_LOG(ERR, USER1, "Failed to start cryptodev\n");
579                         goto error_exit;
580                 }
581
582                 if (rte_eal_remote_launch(vhost_crypto_worker, info,
583                                 lo->lcore_id) < 0) {
584                         RTE_LOG(ERR, USER1, "Failed to start worker lcore");
585                         goto error_exit;
586                 }
587
588                 for (j = 0; j < lo->nb_sockets; j++) {
589                         ret = rte_vhost_driver_register(lo->socket_files[j],
590                                 RTE_VHOST_USER_DEQUEUE_ZERO_COPY);
591                         if (ret < 0) {
592                                 RTE_LOG(ERR, USER1, "socket %s already exists\n",
593                                         lo->socket_files[j]);
594                                 goto error_exit;
595                         }
596
597                         rte_vhost_driver_callback_register(lo->socket_files[j],
598                                 &virtio_crypto_device_ops);
599
600                         ret = rte_vhost_driver_start(lo->socket_files[j]);
601                         if (ret < 0)  {
602                                 RTE_LOG(ERR, USER1, "failed to start vhost.\n");
603                                 goto error_exit;
604                         }
605                 }
606         }
607
608         RTE_LCORE_FOREACH(lcore)
609                 rte_eal_wait_lcore(lcore);
610
611         free_resource();
612
613         return 0;
614
615 error_exit:
616
617         free_resource();
618
619         return -1;
620 }