build: check AVX512 rather than binutils version
[dpdk.git] / drivers / crypto / scheduler / scheduler_pmd.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2017 Intel Corporation
3  */
4 #include <rte_common.h>
5 #include <rte_hexdump.h>
6 #include <rte_cryptodev.h>
7 #include <rte_cryptodev_pmd.h>
8 #include <rte_bus_vdev.h>
9 #include <rte_malloc.h>
10 #include <rte_cpuflags.h>
11 #include <rte_reorder.h>
12 #include <rte_string_fns.h>
13
14 #include "rte_cryptodev_scheduler.h"
15 #include "scheduler_pmd_private.h"
16
17 uint8_t cryptodev_scheduler_driver_id;
18
19 struct scheduler_init_params {
20         struct rte_cryptodev_pmd_init_params def_p;
21         uint32_t nb_slaves;
22         enum rte_cryptodev_scheduler_mode mode;
23         char mode_param_str[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
24         uint32_t enable_ordering;
25         uint16_t wc_pool[RTE_MAX_LCORE];
26         uint16_t nb_wc;
27         char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]
28                         [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
29 };
30
31 #define RTE_CRYPTODEV_VDEV_NAME                 ("name")
32 #define RTE_CRYPTODEV_VDEV_SLAVE                ("slave")
33 #define RTE_CRYPTODEV_VDEV_MODE                 ("mode")
34 #define RTE_CRYPTODEV_VDEV_MODE_PARAM           ("mode_param")
35 #define RTE_CRYPTODEV_VDEV_ORDERING             ("ordering")
36 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG        ("max_nb_queue_pairs")
37 #define RTE_CRYPTODEV_VDEV_SOCKET_ID            ("socket_id")
38 #define RTE_CRYPTODEV_VDEV_COREMASK             ("coremask")
39 #define RTE_CRYPTODEV_VDEV_CORELIST             ("corelist")
40
41 static const char * const scheduler_valid_params[] = {
42         RTE_CRYPTODEV_VDEV_NAME,
43         RTE_CRYPTODEV_VDEV_SLAVE,
44         RTE_CRYPTODEV_VDEV_MODE,
45         RTE_CRYPTODEV_VDEV_MODE_PARAM,
46         RTE_CRYPTODEV_VDEV_ORDERING,
47         RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
48         RTE_CRYPTODEV_VDEV_SOCKET_ID,
49         RTE_CRYPTODEV_VDEV_COREMASK,
50         RTE_CRYPTODEV_VDEV_CORELIST
51 };
52
53 struct scheduler_parse_map {
54         const char *name;
55         uint32_t val;
56 };
57
58 const struct scheduler_parse_map scheduler_mode_map[] = {
59         {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN),
60                         CDEV_SCHED_MODE_ROUNDROBIN},
61         {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR),
62                         CDEV_SCHED_MODE_PKT_SIZE_DISTR},
63         {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER),
64                         CDEV_SCHED_MODE_FAILOVER},
65         {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE),
66                         CDEV_SCHED_MODE_MULTICORE}
67 };
68
69 const struct scheduler_parse_map scheduler_ordering_map[] = {
70                 {"enable", 1},
71                 {"disable", 0}
72 };
73
74 #define CDEV_SCHED_MODE_PARAM_SEP_CHAR          ':'
75
76 static int
77 cryptodev_scheduler_create(const char *name,
78                 struct rte_vdev_device *vdev,
79                 struct scheduler_init_params *init_params)
80 {
81         struct rte_cryptodev *dev;
82         struct scheduler_ctx *sched_ctx;
83         uint32_t i;
84         int ret;
85
86         dev = rte_cryptodev_pmd_create(name, &vdev->device,
87                         &init_params->def_p);
88         if (dev == NULL) {
89                 CR_SCHED_LOG(ERR, "driver %s: failed to create cryptodev vdev",
90                         name);
91                 return -EFAULT;
92         }
93
94         dev->driver_id = cryptodev_scheduler_driver_id;
95         dev->dev_ops = rte_crypto_scheduler_pmd_ops;
96
97         sched_ctx = dev->data->dev_private;
98         sched_ctx->max_nb_queue_pairs =
99                         init_params->def_p.max_nb_queue_pairs;
100
101         if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) {
102                 uint16_t i;
103
104                 sched_ctx->nb_wc = init_params->nb_wc;
105
106                 for (i = 0; i < sched_ctx->nb_wc; i++) {
107                         sched_ctx->wc_pool[i] = init_params->wc_pool[i];
108                         CR_SCHED_LOG(INFO, "  Worker core[%u]=%u added",
109                                 i, sched_ctx->wc_pool[i]);
110                 }
111         }
112
113         if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED &&
114                         init_params->mode < CDEV_SCHED_MODE_COUNT) {
115                 union {
116                         struct rte_cryptodev_scheduler_threshold_option
117                                         threshold_option;
118                 } option;
119                 enum rte_cryptodev_schedule_option_type option_type;
120                 char param_name[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0};
121                 char param_val[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0};
122                 char *s, *end;
123
124                 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id,
125                         init_params->mode);
126                 if (ret < 0) {
127                         rte_cryptodev_pmd_release_device(dev);
128                         return ret;
129                 }
130
131                 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
132                         if (scheduler_mode_map[i].val != sched_ctx->mode)
133                                 continue;
134
135                         CR_SCHED_LOG(INFO, "  Scheduling mode = %s",
136                                         scheduler_mode_map[i].name);
137                         break;
138                 }
139
140                 if (strlen(init_params->mode_param_str) > 0) {
141                         s = strchr(init_params->mode_param_str,
142                                         CDEV_SCHED_MODE_PARAM_SEP_CHAR);
143                         if (s == NULL) {
144                                 CR_SCHED_LOG(ERR, "Invalid mode param");
145                                 return -EINVAL;
146                         }
147
148                         strlcpy(param_name, init_params->mode_param_str,
149                                         s - init_params->mode_param_str + 1);
150                         s++;
151                         strlcpy(param_val, s,
152                                         RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN);
153
154                         switch (init_params->mode) {
155                         case CDEV_SCHED_MODE_PKT_SIZE_DISTR:
156                                 if (strcmp(param_name,
157                                         RTE_CRYPTODEV_SCHEDULER_PARAM_THRES)
158                                                 != 0) {
159                                         CR_SCHED_LOG(ERR, "Invalid mode param");
160                                         return -EINVAL;
161                                 }
162                                 option_type = CDEV_SCHED_OPTION_THRESHOLD;
163
164                                 option.threshold_option.threshold =
165                                                 strtoul(param_val, &end, 0);
166                                 break;
167                         default:
168                                 CR_SCHED_LOG(ERR, "Invalid mode param");
169                                 return -EINVAL;
170                         }
171
172                         if (sched_ctx->ops.option_set(dev, option_type,
173                                         (void *)&option) < 0) {
174                                 CR_SCHED_LOG(ERR, "Invalid mode param");
175                                 return -EINVAL;
176                         }
177
178                         RTE_LOG(INFO, PMD, "  Sched mode param (%s = %s)\n",
179                                         param_name, param_val);
180                 }
181         }
182
183         sched_ctx->reordering_enabled = init_params->enable_ordering;
184
185         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
186                 if (scheduler_ordering_map[i].val !=
187                                 sched_ctx->reordering_enabled)
188                         continue;
189
190                 CR_SCHED_LOG(INFO, "  Packet ordering = %s",
191                                 scheduler_ordering_map[i].name);
192
193                 break;
194         }
195
196         for (i = 0; i < init_params->nb_slaves; i++) {
197                 sched_ctx->init_slave_names[sched_ctx->nb_init_slaves] =
198                         rte_zmalloc_socket(
199                                 NULL,
200                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0,
201                                 SOCKET_ID_ANY);
202
203                 if (!sched_ctx->init_slave_names[
204                                 sched_ctx->nb_init_slaves]) {
205                         CR_SCHED_LOG(ERR, "driver %s: Insufficient memory",
206                                         name);
207                         return -ENOMEM;
208                 }
209
210                 strncpy(sched_ctx->init_slave_names[
211                                         sched_ctx->nb_init_slaves],
212                                 init_params->slave_names[i],
213                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
214
215                 sched_ctx->nb_init_slaves++;
216         }
217
218         /*
219          * Initialize capabilities structure as an empty structure,
220          * in case device information is requested when no slaves are attached
221          */
222         sched_ctx->capabilities = rte_zmalloc_socket(NULL,
223                         sizeof(struct rte_cryptodev_capabilities),
224                         0, SOCKET_ID_ANY);
225
226         if (!sched_ctx->capabilities) {
227                 CR_SCHED_LOG(ERR, "Not enough memory for capability "
228                                 "information");
229                 return -ENOMEM;
230         }
231
232         return 0;
233 }
234
235 static int
236 cryptodev_scheduler_remove(struct rte_vdev_device *vdev)
237 {
238         const char *name;
239         struct rte_cryptodev *dev;
240         struct scheduler_ctx *sched_ctx;
241
242         if (vdev == NULL)
243                 return -EINVAL;
244
245         name = rte_vdev_device_name(vdev);
246         dev = rte_cryptodev_pmd_get_named_dev(name);
247         if (dev == NULL)
248                 return -EINVAL;
249
250         sched_ctx = dev->data->dev_private;
251
252         if (sched_ctx->nb_slaves) {
253                 uint32_t i;
254
255                 for (i = 0; i < sched_ctx->nb_slaves; i++)
256                         rte_cryptodev_scheduler_slave_detach(dev->data->dev_id,
257                                         sched_ctx->slaves[i].dev_id);
258         }
259
260         return rte_cryptodev_pmd_destroy(dev);
261 }
262
263 /** Parse integer from integer argument */
264 static int
265 parse_integer_arg(const char *key __rte_unused,
266                 const char *value, void *extra_args)
267 {
268         int *i = (int *) extra_args;
269
270         *i = atoi(value);
271         if (*i < 0) {
272                 CR_SCHED_LOG(ERR, "Argument has to be positive.");
273                 return -EINVAL;
274         }
275
276         return 0;
277 }
278
279 /** Parse integer from hexadecimal integer argument */
280 static int
281 parse_coremask_arg(const char *key __rte_unused,
282                 const char *value, void *extra_args)
283 {
284         int i, j, val;
285         uint16_t idx = 0;
286         char c;
287         struct scheduler_init_params *params = extra_args;
288
289         params->nb_wc = 0;
290
291         if (value == NULL)
292                 return -1;
293         /* Remove all blank characters ahead and after .
294          * Remove 0x/0X if exists.
295          */
296         while (isblank(*value))
297                 value++;
298         if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X')))
299                 value += 2;
300         i = strlen(value);
301         while ((i > 0) && isblank(value[i - 1]))
302                 i--;
303
304         if (i == 0)
305                 return -1;
306
307         for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
308                 c = value[i];
309                 if (isxdigit(c) == 0) {
310                         /* invalid characters */
311                         return -1;
312                 }
313                 if (isdigit(c))
314                         val = c - '0';
315                 else if (isupper(c))
316                         val = c - 'A' + 10;
317                 else
318                         val = c - 'a' + 10;
319
320                 for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) {
321                         if ((1 << j) & val)
322                                 params->wc_pool[params->nb_wc++] = idx;
323                 }
324         }
325
326         return 0;
327 }
328
329 /** Parse integer from list of integers argument */
330 static int
331 parse_corelist_arg(const char *key __rte_unused,
332                 const char *value, void *extra_args)
333 {
334         struct scheduler_init_params *params = extra_args;
335
336         params->nb_wc = 0;
337
338         const char *token = value;
339
340         while (isdigit(token[0])) {
341                 char *rval;
342                 unsigned int core = strtoul(token, &rval, 10);
343
344                 if (core >= RTE_MAX_LCORE) {
345                         CR_SCHED_LOG(ERR, "Invalid worker core %u, should be smaller "
346                                    "than %u.", core, RTE_MAX_LCORE);
347                 }
348                 params->wc_pool[params->nb_wc++] = (uint16_t)core;
349                 token = (const char *)rval;
350                 if (token[0] == '\0')
351                         break;
352                 token++;
353         }
354
355         return 0;
356 }
357
358 /** Parse name */
359 static int
360 parse_name_arg(const char *key __rte_unused,
361                 const char *value, void *extra_args)
362 {
363         struct rte_cryptodev_pmd_init_params *params = extra_args;
364
365         if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
366                 CR_SCHED_LOG(ERR, "Invalid name %s, should be less than "
367                                 "%u bytes.", value,
368                                 RTE_CRYPTODEV_NAME_MAX_LEN - 1);
369                 return -EINVAL;
370         }
371
372         strlcpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);
373
374         return 0;
375 }
376
377 /** Parse slave */
378 static int
379 parse_slave_arg(const char *key __rte_unused,
380                 const char *value, void *extra_args)
381 {
382         struct scheduler_init_params *param = extra_args;
383
384         if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
385                 CR_SCHED_LOG(ERR, "Too many slaves.");
386                 return -ENOMEM;
387         }
388
389         strncpy(param->slave_names[param->nb_slaves++], value,
390                         RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
391
392         return 0;
393 }
394
395 static int
396 parse_mode_arg(const char *key __rte_unused,
397                 const char *value, void *extra_args)
398 {
399         struct scheduler_init_params *param = extra_args;
400         uint32_t i;
401
402         for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
403                 if (strcmp(value, scheduler_mode_map[i].name) == 0) {
404                         param->mode = (enum rte_cryptodev_scheduler_mode)
405                                         scheduler_mode_map[i].val;
406
407                         break;
408                 }
409         }
410
411         if (i == RTE_DIM(scheduler_mode_map)) {
412                 CR_SCHED_LOG(ERR, "Unrecognized input.");
413                 return -EINVAL;
414         }
415
416         return 0;
417 }
418
419 static int
420 parse_mode_param_arg(const char *key __rte_unused,
421                 const char *value, void *extra_args)
422 {
423         struct scheduler_init_params *param = extra_args;
424
425         strlcpy(param->mode_param_str, value,
426                         RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN);
427
428         return 0;
429 }
430
431 static int
432 parse_ordering_arg(const char *key __rte_unused,
433                 const char *value, void *extra_args)
434 {
435         struct scheduler_init_params *param = extra_args;
436         uint32_t i;
437
438         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
439                 if (strcmp(value, scheduler_ordering_map[i].name) == 0) {
440                         param->enable_ordering =
441                                         scheduler_ordering_map[i].val;
442                         break;
443                 }
444         }
445
446         if (i == RTE_DIM(scheduler_ordering_map)) {
447                 CR_SCHED_LOG(ERR, "Unrecognized input.");
448                 return -EINVAL;
449         }
450
451         return 0;
452 }
453
454 static int
455 scheduler_parse_init_params(struct scheduler_init_params *params,
456                 const char *input_args)
457 {
458         struct rte_kvargs *kvlist = NULL;
459         int ret = 0;
460
461         if (params == NULL)
462                 return -EINVAL;
463
464         if (input_args) {
465                 kvlist = rte_kvargs_parse(input_args,
466                                 scheduler_valid_params);
467                 if (kvlist == NULL)
468                         return -1;
469
470                 ret = rte_kvargs_process(kvlist,
471                                 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
472                                 &parse_integer_arg,
473                                 &params->def_p.max_nb_queue_pairs);
474                 if (ret < 0)
475                         goto free_kvlist;
476
477                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID,
478                                 &parse_integer_arg,
479                                 &params->def_p.socket_id);
480                 if (ret < 0)
481                         goto free_kvlist;
482
483                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK,
484                                 &parse_coremask_arg,
485                                 params);
486                 if (ret < 0)
487                         goto free_kvlist;
488
489                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST,
490                                 &parse_corelist_arg,
491                                 params);
492                 if (ret < 0)
493                         goto free_kvlist;
494
495                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME,
496                                 &parse_name_arg,
497                                 &params->def_p);
498                 if (ret < 0)
499                         goto free_kvlist;
500
501                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SLAVE,
502                                 &parse_slave_arg, params);
503                 if (ret < 0)
504                         goto free_kvlist;
505
506                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE,
507                                 &parse_mode_arg, params);
508                 if (ret < 0)
509                         goto free_kvlist;
510
511                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE_PARAM,
512                                 &parse_mode_param_arg, params);
513                 if (ret < 0)
514                         goto free_kvlist;
515
516                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING,
517                                 &parse_ordering_arg, params);
518                 if (ret < 0)
519                         goto free_kvlist;
520         }
521
522 free_kvlist:
523         rte_kvargs_free(kvlist);
524         return ret;
525 }
526
527 static int
528 cryptodev_scheduler_probe(struct rte_vdev_device *vdev)
529 {
530         struct scheduler_init_params init_params = {
531                 .def_p = {
532                         "",
533                         sizeof(struct scheduler_ctx),
534                         rte_socket_id(),
535                         RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
536                 },
537                 .nb_slaves = 0,
538                 .mode = CDEV_SCHED_MODE_NOT_SET,
539                 .enable_ordering = 0,
540                 .slave_names = { {0} }
541         };
542         const char *name;
543
544         name = rte_vdev_device_name(vdev);
545         if (name == NULL)
546                 return -EINVAL;
547
548         scheduler_parse_init_params(&init_params,
549                                     rte_vdev_device_args(vdev));
550
551
552         return cryptodev_scheduler_create(name,
553                                         vdev,
554                                         &init_params);
555 }
556
557 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = {
558         .probe = cryptodev_scheduler_probe,
559         .remove = cryptodev_scheduler_remove
560 };
561
562 static struct cryptodev_driver scheduler_crypto_drv;
563
564 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD,
565         cryptodev_scheduler_pmd_drv);
566 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD,
567         "max_nb_queue_pairs=<int> "
568         "socket_id=<int> "
569         "slave=<name>");
570 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv,
571                 cryptodev_scheduler_pmd_drv.driver,
572                 cryptodev_scheduler_driver_id);