drivers/crypto/scheduler/scheduler_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4 #include <rte_common.h>
   5 #include <rte_hexdump.h>
   6 #include <rte_cryptodev.h>
   7 #include <cryptodev_pmd.h>
   8 #include <rte_bus_vdev.h>
   9 #include <rte_malloc.h>
  10 #include <rte_cpuflags.h>
  11 #include <rte_reorder.h>
  12 #include <rte_string_fns.h>
  13
  14 #include "rte_cryptodev_scheduler.h"
  15 #include "scheduler_pmd_private.h"
  16
  17 uint8_t cryptodev_scheduler_driver_id;
  18
  19 struct scheduler_init_params {
  20         struct rte_cryptodev_pmd_init_params def_p;
  21         uint32_t nb_workers;
  22         enum rte_cryptodev_scheduler_mode mode;
  23         char mode_param_str[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
  24         uint32_t enable_ordering;
  25         uint16_t wc_pool[RTE_MAX_LCORE];
  26         uint16_t nb_wc;
  27         char worker_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS]
  28                         [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
  29 };
  30
  31 #define RTE_CRYPTODEV_VDEV_NAME                 ("name")
  32 #define RTE_CRYPTODEV_VDEV_WORKER               ("worker")
  33 #define RTE_CRYPTODEV_VDEV_MODE                 ("mode")
  34 #define RTE_CRYPTODEV_VDEV_MODE_PARAM           ("mode_param")
  35 #define RTE_CRYPTODEV_VDEV_ORDERING             ("ordering")
  36 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG        ("max_nb_queue_pairs")
  37 #define RTE_CRYPTODEV_VDEV_SOCKET_ID            ("socket_id")
  38 #define RTE_CRYPTODEV_VDEV_COREMASK             ("coremask")
  39 #define RTE_CRYPTODEV_VDEV_CORELIST             ("corelist")
  40
  41 static const char * const scheduler_valid_params[] = {
  42         RTE_CRYPTODEV_VDEV_NAME,
  43         RTE_CRYPTODEV_VDEV_WORKER,
  44         RTE_CRYPTODEV_VDEV_MODE,
  45         RTE_CRYPTODEV_VDEV_MODE_PARAM,
  46         RTE_CRYPTODEV_VDEV_ORDERING,
  47         RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
  48         RTE_CRYPTODEV_VDEV_SOCKET_ID,
  49         RTE_CRYPTODEV_VDEV_COREMASK,
  50         RTE_CRYPTODEV_VDEV_CORELIST
  51 };
  52
  53 struct scheduler_parse_map {
  54         const char *name;
  55         uint32_t val;
  56 };
  57
  58 const struct scheduler_parse_map scheduler_mode_map[] = {
  59         {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN),
  60                         CDEV_SCHED_MODE_ROUNDROBIN},
  61         {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR),
  62                         CDEV_SCHED_MODE_PKT_SIZE_DISTR},
  63         {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER),
  64                         CDEV_SCHED_MODE_FAILOVER},
  65         {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE),
  66                         CDEV_SCHED_MODE_MULTICORE}
  67 };
  68
  69 const struct scheduler_parse_map scheduler_ordering_map[] = {
  70                 {"enable", 1},
  71                 {"disable", 0}
  72 };
  73
  74 #define CDEV_SCHED_MODE_PARAM_SEP_CHAR          ':'
  75
  76 static int
  77 cryptodev_scheduler_create(const char *name,
  78                 struct rte_vdev_device *vdev,
  79                 struct scheduler_init_params *init_params)
  80 {
  81         struct rte_cryptodev *dev;
  82         struct scheduler_ctx *sched_ctx;
  83         uint32_t i;
  84         int ret;
  85
  86         dev = rte_cryptodev_pmd_create(name, &vdev->device,
  87                         &init_params->def_p);
  88         if (dev == NULL) {
  89                 CR_SCHED_LOG(ERR, "driver %s: failed to create cryptodev vdev",
  90                         name);
  91                 return -EFAULT;
  92         }
  93
  94         dev->driver_id = cryptodev_scheduler_driver_id;
  95         dev->dev_ops = rte_crypto_scheduler_pmd_ops;
  96
  97         sched_ctx = dev->data->dev_private;
  98         sched_ctx->max_nb_queue_pairs =
  99                         init_params->def_p.max_nb_queue_pairs;
 100
 101         if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) {
 102                 uint16_t i;
 103
 104                 sched_ctx->nb_wc = init_params->nb_wc;
 105
 106                 for (i = 0; i < sched_ctx->nb_wc; i++) {
 107                         sched_ctx->wc_pool[i] = init_params->wc_pool[i];
 108                         CR_SCHED_LOG(INFO, "  Worker core[%u]=%u added",
 109                                 i, sched_ctx->wc_pool[i]);
 110                 }
 111         }
 112
 113         if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED &&
 114                         init_params->mode < CDEV_SCHED_MODE_COUNT) {
 115                 union {
 116                         struct rte_cryptodev_scheduler_threshold_option
 117                                         threshold_option;
 118                 } option;
 119                 enum rte_cryptodev_schedule_option_type option_type;
 120                 char param_name[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0};
 121                 char param_val[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0};
 122                 char *s, *end;
 123
 124                 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id,
 125                         init_params->mode);
 126                 if (ret < 0) {
 127                         rte_cryptodev_pmd_release_device(dev);
 128                         return ret;
 129                 }
 130
 131                 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
 132                         if (scheduler_mode_map[i].val != sched_ctx->mode)
 133                                 continue;
 134
 135                         CR_SCHED_LOG(INFO, "  Scheduling mode = %s",
 136                                         scheduler_mode_map[i].name);
 137                         break;
 138                 }
 139
 140                 if (strlen(init_params->mode_param_str) > 0) {
 141                         s = strchr(init_params->mode_param_str,
 142                                         CDEV_SCHED_MODE_PARAM_SEP_CHAR);
 143                         if (s == NULL) {
 144                                 CR_SCHED_LOG(ERR, "Invalid mode param");
 145                                 return -EINVAL;
 146                         }
 147
 148                         strlcpy(param_name, init_params->mode_param_str,
 149                                         s - init_params->mode_param_str + 1);
 150                         s++;
 151                         strlcpy(param_val, s,
 152                                         RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN);
 153
 154                         switch (init_params->mode) {
 155                         case CDEV_SCHED_MODE_PKT_SIZE_DISTR:
 156                                 if (strcmp(param_name,
 157                                         RTE_CRYPTODEV_SCHEDULER_PARAM_THRES)
 158                                                 != 0) {
 159                                         CR_SCHED_LOG(ERR, "Invalid mode param");
 160                                         return -EINVAL;
 161                                 }
 162                                 option_type = CDEV_SCHED_OPTION_THRESHOLD;
 163
 164                                 option.threshold_option.threshold =
 165                                                 strtoul(param_val, &end, 0);
 166                                 break;
 167                         default:
 168                                 CR_SCHED_LOG(ERR, "Invalid mode param");
 169                                 return -EINVAL;
 170                         }
 171
 172                         if (sched_ctx->ops.option_set(dev, option_type,
 173                                         (void *)&option) < 0) {
 174                                 CR_SCHED_LOG(ERR, "Invalid mode param");
 175                                 return -EINVAL;
 176                         }
 177
 178                         RTE_LOG(INFO, PMD, "  Sched mode param (%s = %s)\n",
 179                                         param_name, param_val);
 180                 }
 181         }
 182
 183         sched_ctx->reordering_enabled = init_params->enable_ordering;
 184
 185         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
 186                 if (scheduler_ordering_map[i].val !=
 187                                 sched_ctx->reordering_enabled)
 188                         continue;
 189
 190                 CR_SCHED_LOG(INFO, "  Packet ordering = %s",
 191                                 scheduler_ordering_map[i].name);
 192
 193                 break;
 194         }
 195
 196         for (i = 0; i < init_params->nb_workers; i++) {
 197                 sched_ctx->init_worker_names[sched_ctx->nb_init_workers] =
 198                         rte_zmalloc_socket(
 199                                 NULL,
 200                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0,
 201                                 SOCKET_ID_ANY);
 202
 203                 if (!sched_ctx->init_worker_names[
 204                                 sched_ctx->nb_init_workers]) {
 205                         CR_SCHED_LOG(ERR, "driver %s: Insufficient memory",
 206                                         name);
 207                         return -ENOMEM;
 208                 }
 209
 210                 strncpy(sched_ctx->init_worker_names[
 211                                         sched_ctx->nb_init_workers],
 212                                 init_params->worker_names[i],
 213                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
 214
 215                 sched_ctx->nb_init_workers++;
 216         }
 217
 218         /*
 219          * Initialize capabilities structure as an empty structure,
 220          * in case device information is requested when no workers are attached
 221          */
 222         sched_ctx->capabilities = rte_zmalloc_socket(NULL,
 223                         sizeof(struct rte_cryptodev_capabilities),
 224                         0, SOCKET_ID_ANY);
 225
 226         if (!sched_ctx->capabilities) {
 227                 CR_SCHED_LOG(ERR, "Not enough memory for capability "
 228                                 "information");
 229                 return -ENOMEM;
 230         }
 231
 232         rte_cryptodev_pmd_probing_finish(dev);
 233
 234         return 0;
 235 }
 236
 237 static int
 238 cryptodev_scheduler_remove(struct rte_vdev_device *vdev)
 239 {
 240         const char *name;
 241         struct rte_cryptodev *dev;
 242         struct scheduler_ctx *sched_ctx;
 243
 244         if (vdev == NULL)
 245                 return -EINVAL;
 246
 247         name = rte_vdev_device_name(vdev);
 248         dev = rte_cryptodev_pmd_get_named_dev(name);
 249         if (dev == NULL)
 250                 return -EINVAL;
 251
 252         sched_ctx = dev->data->dev_private;
 253
 254         if (sched_ctx->nb_workers) {
 255                 uint32_t i;
 256
 257                 for (i = 0; i < sched_ctx->nb_workers; i++)
 258                         rte_cryptodev_scheduler_worker_detach(dev->data->dev_id,
 259                                         sched_ctx->workers[i].dev_id);
 260         }
 261
 262         return rte_cryptodev_pmd_destroy(dev);
 263 }
 264
 265 /** Parse integer from integer argument */
 266 static int
 267 parse_integer_arg(const char *key __rte_unused,
 268                 const char *value, void *extra_args)
 269 {
 270         int *i = (int *) extra_args;
 271
 272         *i = atoi(value);
 273         if (*i < 0) {
 274                 CR_SCHED_LOG(ERR, "Argument has to be positive.");
 275                 return -EINVAL;
 276         }
 277
 278         return 0;
 279 }
 280
 281 /** Parse integer from hexadecimal integer argument */
 282 static int
 283 parse_coremask_arg(const char *key __rte_unused,
 284                 const char *value, void *extra_args)
 285 {
 286         int i, j, val;
 287         uint16_t idx = 0;
 288         char c;
 289         struct scheduler_init_params *params = extra_args;
 290
 291         params->nb_wc = 0;
 292
 293         if (value == NULL)
 294                 return -1;
 295         /* Remove all blank characters ahead and after .
 296          * Remove 0x/0X if exists.
 297          */
 298         while (isblank(*value))
 299                 value++;
 300         if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X')))
 301                 value += 2;
 302         i = strlen(value);
 303         while ((i > 0) && isblank(value[i - 1]))
 304                 i--;
 305
 306         if (i == 0)
 307                 return -1;
 308
 309         for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
 310                 c = value[i];
 311                 if (isxdigit(c) == 0) {
 312                         /* invalid characters */
 313                         return -1;
 314                 }
 315                 if (isdigit(c))
 316                         val = c - '0';
 317                 else if (isupper(c))
 318                         val = c - 'A' + 10;
 319                 else
 320                         val = c - 'a' + 10;
 321
 322                 for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) {
 323                         if ((1 << j) & val)
 324                                 params->wc_pool[params->nb_wc++] = idx;
 325                 }
 326         }
 327
 328         return 0;
 329 }
 330
 331 /** Parse integer from list of integers argument */
 332 static int
 333 parse_corelist_arg(const char *key __rte_unused,
 334                 const char *value, void *extra_args)
 335 {
 336         struct scheduler_init_params *params = extra_args;
 337
 338         params->nb_wc = 0;
 339
 340         const char *token = value;
 341
 342         while (isdigit(token[0])) {
 343                 char *rval;
 344                 unsigned int core = strtoul(token, &rval, 10);
 345
 346                 if (core >= RTE_MAX_LCORE) {
 347                         CR_SCHED_LOG(ERR, "Invalid worker core %u, should be smaller "
 348                                    "than %u.", core, RTE_MAX_LCORE);
 349                 }
 350                 params->wc_pool[params->nb_wc++] = (uint16_t)core;
 351                 token = (const char *)rval;
 352                 if (token[0] == '\0')
 353                         break;
 354                 token++;
 355         }
 356
 357         return 0;
 358 }
 359
 360 /** Parse name */
 361 static int
 362 parse_name_arg(const char *key __rte_unused,
 363                 const char *value, void *extra_args)
 364 {
 365         struct rte_cryptodev_pmd_init_params *params = extra_args;
 366
 367         if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
 368                 CR_SCHED_LOG(ERR, "Invalid name %s, should be less than "
 369                                 "%u bytes.", value,
 370                                 RTE_CRYPTODEV_NAME_MAX_LEN - 1);
 371                 return -EINVAL;
 372         }
 373
 374         strlcpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);
 375
 376         return 0;
 377 }
 378
 379 /** Parse worker */
 380 static int
 381 parse_worker_arg(const char *key __rte_unused,
 382                 const char *value, void *extra_args)
 383 {
 384         struct scheduler_init_params *param = extra_args;
 385
 386         if (param->nb_workers >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS) {
 387                 CR_SCHED_LOG(ERR, "Too many workers.");
 388                 return -ENOMEM;
 389         }
 390
 391         strncpy(param->worker_names[param->nb_workers++], value,
 392                         RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
 393
 394         return 0;
 395 }
 396
 397 static int
 398 parse_mode_arg(const char *key __rte_unused,
 399                 const char *value, void *extra_args)
 400 {
 401         struct scheduler_init_params *param = extra_args;
 402         uint32_t i;
 403
 404         for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
 405                 if (strcmp(value, scheduler_mode_map[i].name) == 0) {
 406                         param->mode = (enum rte_cryptodev_scheduler_mode)
 407                                         scheduler_mode_map[i].val;
 408
 409                         break;
 410                 }
 411         }
 412
 413         if (i == RTE_DIM(scheduler_mode_map)) {
 414                 CR_SCHED_LOG(ERR, "Unrecognized input.");
 415                 return -EINVAL;
 416         }
 417
 418         return 0;
 419 }
 420
 421 static int
 422 parse_mode_param_arg(const char *key __rte_unused,
 423                 const char *value, void *extra_args)
 424 {
 425         struct scheduler_init_params *param = extra_args;
 426
 427         strlcpy(param->mode_param_str, value,
 428                         RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN);
 429
 430         return 0;
 431 }
 432
 433 static int
 434 parse_ordering_arg(const char *key __rte_unused,
 435                 const char *value, void *extra_args)
 436 {
 437         struct scheduler_init_params *param = extra_args;
 438         uint32_t i;
 439
 440         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
 441                 if (strcmp(value, scheduler_ordering_map[i].name) == 0) {
 442                         param->enable_ordering =
 443                                         scheduler_ordering_map[i].val;
 444                         break;
 445                 }
 446         }
 447
 448         if (i == RTE_DIM(scheduler_ordering_map)) {
 449                 CR_SCHED_LOG(ERR, "Unrecognized input.");
 450                 return -EINVAL;
 451         }
 452
 453         return 0;
 454 }
 455
 456 static int
 457 scheduler_parse_init_params(struct scheduler_init_params *params,
 458                 const char *input_args)
 459 {
 460         struct rte_kvargs *kvlist = NULL;
 461         int ret = 0;
 462
 463         if (params == NULL)
 464                 return -EINVAL;
 465
 466         if (input_args) {
 467                 kvlist = rte_kvargs_parse(input_args,
 468                                 scheduler_valid_params);
 469                 if (kvlist == NULL)
 470                         return -1;
 471
 472                 ret = rte_kvargs_process(kvlist,
 473                                 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
 474                                 &parse_integer_arg,
 475                                 &params->def_p.max_nb_queue_pairs);
 476                 if (ret < 0)
 477                         goto free_kvlist;
 478
 479                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID,
 480                                 &parse_integer_arg,
 481                                 &params->def_p.socket_id);
 482                 if (ret < 0)
 483                         goto free_kvlist;
 484
 485                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK,
 486                                 &parse_coremask_arg,
 487                                 params);
 488                 if (ret < 0)
 489                         goto free_kvlist;
 490
 491                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST,
 492                                 &parse_corelist_arg,
 493                                 params);
 494                 if (ret < 0)
 495                         goto free_kvlist;
 496
 497                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME,
 498                                 &parse_name_arg,
 499                                 &params->def_p);
 500                 if (ret < 0)
 501                         goto free_kvlist;
 502
 503                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_WORKER,
 504                                 &parse_worker_arg, params);
 505                 if (ret < 0)
 506                         goto free_kvlist;
 507
 508                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE,
 509                                 &parse_mode_arg, params);
 510                 if (ret < 0)
 511                         goto free_kvlist;
 512
 513                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE_PARAM,
 514                                 &parse_mode_param_arg, params);
 515                 if (ret < 0)
 516                         goto free_kvlist;
 517
 518                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING,
 519                                 &parse_ordering_arg, params);
 520                 if (ret < 0)
 521                         goto free_kvlist;
 522         }
 523
 524 free_kvlist:
 525         rte_kvargs_free(kvlist);
 526         return ret;
 527 }
 528
 529 static int
 530 cryptodev_scheduler_probe(struct rte_vdev_device *vdev)
 531 {
 532         struct scheduler_init_params init_params = {
 533                 .def_p = {
 534                         "",
 535                         sizeof(struct scheduler_ctx),
 536                         rte_socket_id(),
 537                         RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
 538                 },
 539                 .nb_workers = 0,
 540                 .mode = CDEV_SCHED_MODE_NOT_SET,
 541                 .enable_ordering = 0,
 542                 .worker_names = { {0} }
 543         };
 544         const char *name;
 545
 546         name = rte_vdev_device_name(vdev);
 547         if (name == NULL)
 548                 return -EINVAL;
 549
 550         scheduler_parse_init_params(&init_params,
 551                                     rte_vdev_device_args(vdev));
 552
 553
 554         return cryptodev_scheduler_create(name,
 555                                         vdev,
 556                                         &init_params);
 557 }
 558
 559 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = {
 560         .probe = cryptodev_scheduler_probe,
 561         .remove = cryptodev_scheduler_remove
 562 };
 563
 564 static struct cryptodev_driver scheduler_crypto_drv;
 565
 566 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD,
 567         cryptodev_scheduler_pmd_drv);
 568 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD,
 569         "max_nb_queue_pairs=<int> "
 570         "socket_id=<int> "
 571         "worker=<name>");
 572 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv,
 573                 cryptodev_scheduler_pmd_drv.driver,
 574                 cryptodev_scheduler_driver_id);