drivers/crypto/scheduler/scheduler_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4 #include <rte_common.h>
   5 #include <rte_hexdump.h>
   6 #include <rte_cryptodev.h>
   7 #include <rte_cryptodev_pmd.h>
   8 #include <rte_bus_vdev.h>
   9 #include <rte_malloc.h>
  10 #include <rte_cpuflags.h>
  11 #include <rte_reorder.h>
  12 #include <rte_string_fns.h>
  13
  14 #include "rte_cryptodev_scheduler.h"
  15 #include "scheduler_pmd_private.h"
  16
  17 uint8_t cryptodev_driver_id;
  18
  19 struct scheduler_init_params {
  20         struct rte_cryptodev_pmd_init_params def_p;
  21         uint32_t nb_slaves;
  22         enum rte_cryptodev_scheduler_mode mode;
  23         char mode_param_str[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
  24         uint32_t enable_ordering;
  25         uint16_t wc_pool[RTE_MAX_LCORE];
  26         uint16_t nb_wc;
  27         char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]
  28                         [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
  29 };
  30
  31 #define RTE_CRYPTODEV_VDEV_NAME                 ("name")
  32 #define RTE_CRYPTODEV_VDEV_SLAVE                ("slave")
  33 #define RTE_CRYPTODEV_VDEV_MODE                 ("mode")
  34 #define RTE_CRYPTODEV_VDEV_MODE_PARAM           ("mode_param")
  35 #define RTE_CRYPTODEV_VDEV_ORDERING             ("ordering")
  36 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG        ("max_nb_queue_pairs")
  37 #define RTE_CRYPTODEV_VDEV_SOCKET_ID            ("socket_id")
  38 #define RTE_CRYPTODEV_VDEV_COREMASK             ("coremask")
  39 #define RTE_CRYPTODEV_VDEV_CORELIST             ("corelist")
  40
  41 const char *scheduler_valid_params[] = {
  42         RTE_CRYPTODEV_VDEV_NAME,
  43         RTE_CRYPTODEV_VDEV_SLAVE,
  44         RTE_CRYPTODEV_VDEV_MODE,
  45         RTE_CRYPTODEV_VDEV_MODE_PARAM,
  46         RTE_CRYPTODEV_VDEV_ORDERING,
  47         RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
  48         RTE_CRYPTODEV_VDEV_SOCKET_ID,
  49         RTE_CRYPTODEV_VDEV_COREMASK,
  50         RTE_CRYPTODEV_VDEV_CORELIST
  51 };
  52
  53 struct scheduler_parse_map {
  54         const char *name;
  55         uint32_t val;
  56 };
  57
  58 const struct scheduler_parse_map scheduler_mode_map[] = {
  59         {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN),
  60                         CDEV_SCHED_MODE_ROUNDROBIN},
  61         {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR),
  62                         CDEV_SCHED_MODE_PKT_SIZE_DISTR},
  63         {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER),
  64                         CDEV_SCHED_MODE_FAILOVER},
  65         {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE),
  66                         CDEV_SCHED_MODE_MULTICORE}
  67 };
  68
  69 const struct scheduler_parse_map scheduler_ordering_map[] = {
  70                 {"enable", 1},
  71                 {"disable", 0}
  72 };
  73
  74 #define CDEV_SCHED_MODE_PARAM_SEP_CHAR          ':'
  75
  76 static int
  77 cryptodev_scheduler_create(const char *name,
  78                 struct rte_vdev_device *vdev,
  79                 struct scheduler_init_params *init_params)
  80 {
  81         struct rte_cryptodev *dev;
  82         struct scheduler_ctx *sched_ctx;
  83         uint32_t i;
  84         int ret;
  85
  86         dev = rte_cryptodev_pmd_create(name, &vdev->device,
  87                         &init_params->def_p);
  88         if (dev == NULL) {
  89                 CR_SCHED_LOG(ERR, "driver %s: failed to create cryptodev vdev",
  90                         name);
  91                 return -EFAULT;
  92         }
  93
  94         dev->driver_id = cryptodev_driver_id;
  95         dev->dev_ops = rte_crypto_scheduler_pmd_ops;
  96
  97         sched_ctx = dev->data->dev_private;
  98         sched_ctx->max_nb_queue_pairs =
  99                         init_params->def_p.max_nb_queue_pairs;
 100
 101         if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) {
 102                 uint16_t i;
 103
 104                 sched_ctx->nb_wc = init_params->nb_wc;
 105
 106                 for (i = 0; i < sched_ctx->nb_wc; i++) {
 107                         sched_ctx->wc_pool[i] = init_params->wc_pool[i];
 108                         CR_SCHED_LOG(INFO, "  Worker core[%u]=%u added",
 109                                 i, sched_ctx->wc_pool[i]);
 110                 }
 111         }
 112
 113         if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED &&
 114                         init_params->mode < CDEV_SCHED_MODE_COUNT) {
 115                 union {
 116                         struct rte_cryptodev_scheduler_threshold_option
 117                                         threshold_option;
 118                 } option;
 119                 enum rte_cryptodev_schedule_option_type option_type;
 120                 char param_name[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0};
 121                 char param_val[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN] = {0};
 122                 char *s, *end;
 123
 124                 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id,
 125                         init_params->mode);
 126                 if (ret < 0) {
 127                         rte_cryptodev_pmd_release_device(dev);
 128                         return ret;
 129                 }
 130
 131                 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
 132                         if (scheduler_mode_map[i].val != sched_ctx->mode)
 133                                 continue;
 134
 135                         CR_SCHED_LOG(INFO, "  Scheduling mode = %s",
 136                                         scheduler_mode_map[i].name);
 137                         break;
 138                 }
 139
 140                 if (strlen(init_params->mode_param_str) > 0) {
 141                         s = strchr(init_params->mode_param_str,
 142                                         CDEV_SCHED_MODE_PARAM_SEP_CHAR);
 143                         if (s == NULL) {
 144                                 CR_SCHED_LOG(ERR, "Invalid mode param");
 145                                 return -EINVAL;
 146                         }
 147
 148                         strlcpy(param_name, init_params->mode_param_str,
 149                                         s - init_params->mode_param_str + 1);
 150                         s++;
 151                         strlcpy(param_val, s,
 152                                         RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN);
 153
 154                         switch (init_params->mode) {
 155                         case CDEV_SCHED_MODE_PKT_SIZE_DISTR:
 156                                 if (strcmp(param_name,
 157                                         RTE_CRYPTODEV_SCHEDULER_PARAM_THRES)
 158                                                 != 0) {
 159                                         CR_SCHED_LOG(ERR, "Invalid mode param");
 160                                         return -EINVAL;
 161                                 }
 162                                 option_type = CDEV_SCHED_OPTION_THRESHOLD;
 163
 164                                 option.threshold_option.threshold =
 165                                                 strtoul(param_val, &end, 0);
 166                                 break;
 167                         default:
 168                                 CR_SCHED_LOG(ERR, "Invalid mode param");
 169                                 return -EINVAL;
 170                         }
 171
 172                         if (sched_ctx->ops.option_set(dev, option_type,
 173                                         (void *)&option) < 0) {
 174                                 CR_SCHED_LOG(ERR, "Invalid mode param");
 175                                 return -EINVAL;
 176                         }
 177
 178                         RTE_LOG(INFO, PMD, "  Sched mode param (%s = %s)\n",
 179                                         param_name, param_val);
 180                 }
 181         }
 182
 183         sched_ctx->reordering_enabled = init_params->enable_ordering;
 184
 185         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
 186                 if (scheduler_ordering_map[i].val !=
 187                                 sched_ctx->reordering_enabled)
 188                         continue;
 189
 190                 CR_SCHED_LOG(INFO, "  Packet ordering = %s",
 191                                 scheduler_ordering_map[i].name);
 192
 193                 break;
 194         }
 195
 196         for (i = 0; i < init_params->nb_slaves; i++) {
 197                 sched_ctx->init_slave_names[sched_ctx->nb_init_slaves] =
 198                         rte_zmalloc_socket(
 199                                 NULL,
 200                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0,
 201                                 SOCKET_ID_ANY);
 202
 203                 if (!sched_ctx->init_slave_names[
 204                                 sched_ctx->nb_init_slaves]) {
 205                         CR_SCHED_LOG(ERR, "driver %s: Insufficient memory",
 206                                         name);
 207                         return -ENOMEM;
 208                 }
 209
 210                 strncpy(sched_ctx->init_slave_names[
 211                                         sched_ctx->nb_init_slaves],
 212                                 init_params->slave_names[i],
 213                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
 214
 215                 sched_ctx->nb_init_slaves++;
 216         }
 217
 218         /*
 219          * Initialize capabilities structure as an empty structure,
 220          * in case device information is requested when no slaves are attached
 221          */
 222         sched_ctx->capabilities = rte_zmalloc_socket(NULL,
 223                         sizeof(struct rte_cryptodev_capabilities),
 224                         0, SOCKET_ID_ANY);
 225
 226         if (!sched_ctx->capabilities) {
 227                 CR_SCHED_LOG(ERR, "Not enough memory for capability "
 228                                 "information");
 229                 return -ENOMEM;
 230         }
 231
 232         return 0;
 233 }
 234
 235 static int
 236 cryptodev_scheduler_remove(struct rte_vdev_device *vdev)
 237 {
 238         const char *name;
 239         struct rte_cryptodev *dev;
 240         struct scheduler_ctx *sched_ctx;
 241
 242         if (vdev == NULL)
 243                 return -EINVAL;
 244
 245         name = rte_vdev_device_name(vdev);
 246         dev = rte_cryptodev_pmd_get_named_dev(name);
 247         if (dev == NULL)
 248                 return -EINVAL;
 249
 250         sched_ctx = dev->data->dev_private;
 251
 252         if (sched_ctx->nb_slaves) {
 253                 uint32_t i;
 254
 255                 for (i = 0; i < sched_ctx->nb_slaves; i++)
 256                         rte_cryptodev_scheduler_slave_detach(dev->data->dev_id,
 257                                         sched_ctx->slaves[i].dev_id);
 258         }
 259
 260         return rte_cryptodev_pmd_destroy(dev);
 261 }
 262
 263 /** Parse integer from integer argument */
 264 static int
 265 parse_integer_arg(const char *key __rte_unused,
 266                 const char *value, void *extra_args)
 267 {
 268         int *i = (int *) extra_args;
 269
 270         *i = atoi(value);
 271         if (*i < 0) {
 272                 CR_SCHED_LOG(ERR, "Argument has to be positive.");
 273                 return -EINVAL;
 274         }
 275
 276         return 0;
 277 }
 278
 279 /** Parse integer from hexadecimal integer argument */
 280 static int
 281 parse_coremask_arg(const char *key __rte_unused,
 282                 const char *value, void *extra_args)
 283 {
 284         int i, j, val;
 285         uint16_t idx = 0;
 286         char c;
 287         struct scheduler_init_params *params = extra_args;
 288
 289         params->nb_wc = 0;
 290
 291         if (value == NULL)
 292                 return -1;
 293         /* Remove all blank characters ahead and after .
 294          * Remove 0x/0X if exists.
 295          */
 296         while (isblank(*value))
 297                 value++;
 298         if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X')))
 299                 value += 2;
 300         i = strlen(value);
 301         while ((i > 0) && isblank(value[i - 1]))
 302                 i--;
 303
 304         if (i == 0)
 305                 return -1;
 306
 307         for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
 308                 c = value[i];
 309                 if (isxdigit(c) == 0) {
 310                         /* invalid characters */
 311                         return -1;
 312                 }
 313                 if (isdigit(c))
 314                         val = c - '0';
 315                 else if (isupper(c))
 316                         val = c - 'A' + 10;
 317                 else
 318                         val = c - 'a' + 10;
 319
 320                 for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) {
 321                         if ((1 << j) & val)
 322                                 params->wc_pool[params->nb_wc++] = idx;
 323                 }
 324         }
 325
 326         return 0;
 327 }
 328
 329 /** Parse integer from list of integers argument */
 330 static int
 331 parse_corelist_arg(const char *key __rte_unused,
 332                 const char *value, void *extra_args)
 333 {
 334         struct scheduler_init_params *params = extra_args;
 335
 336         params->nb_wc = 0;
 337
 338         const char *token = value;
 339
 340         while (isdigit(token[0])) {
 341                 char *rval;
 342                 unsigned int core = strtoul(token, &rval, 10);
 343
 344                 if (core >= RTE_MAX_LCORE) {
 345                         CR_SCHED_LOG(ERR, "Invalid worker core %u, should be smaller "
 346                                    "than %u.", core, RTE_MAX_LCORE);
 347                 }
 348                 params->wc_pool[params->nb_wc++] = (uint16_t)core;
 349                 token = (const char *)rval;
 350                 if (token[0] == '\0')
 351                         break;
 352                 token++;
 353         }
 354
 355         return 0;
 356 }
 357
 358 /** Parse name */
 359 static int
 360 parse_name_arg(const char *key __rte_unused,
 361                 const char *value, void *extra_args)
 362 {
 363         struct rte_cryptodev_pmd_init_params *params = extra_args;
 364
 365         if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
 366                 CR_SCHED_LOG(ERR, "Invalid name %s, should be less than "
 367                                 "%u bytes.", value,
 368                                 RTE_CRYPTODEV_NAME_MAX_LEN - 1);
 369                 return -EINVAL;
 370         }
 371
 372         strncpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);
 373
 374         return 0;
 375 }
 376
 377 /** Parse slave */
 378 static int
 379 parse_slave_arg(const char *key __rte_unused,
 380                 const char *value, void *extra_args)
 381 {
 382         struct scheduler_init_params *param = extra_args;
 383
 384         if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
 385                 CR_SCHED_LOG(ERR, "Too many slaves.");
 386                 return -ENOMEM;
 387         }
 388
 389         strncpy(param->slave_names[param->nb_slaves++], value,
 390                         RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
 391
 392         return 0;
 393 }
 394
 395 static int
 396 parse_mode_arg(const char *key __rte_unused,
 397                 const char *value, void *extra_args)
 398 {
 399         struct scheduler_init_params *param = extra_args;
 400         uint32_t i;
 401
 402         for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
 403                 if (strcmp(value, scheduler_mode_map[i].name) == 0) {
 404                         param->mode = (enum rte_cryptodev_scheduler_mode)
 405                                         scheduler_mode_map[i].val;
 406
 407                         break;
 408                 }
 409         }
 410
 411         if (i == RTE_DIM(scheduler_mode_map)) {
 412                 CR_SCHED_LOG(ERR, "Unrecognized input.");
 413                 return -EINVAL;
 414         }
 415
 416         return 0;
 417 }
 418
 419 static int
 420 parse_mode_param_arg(const char *key __rte_unused,
 421                 const char *value, void *extra_args)
 422 {
 423         struct scheduler_init_params *param = extra_args;
 424
 425         strlcpy(param->mode_param_str, value,
 426                         RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN);
 427
 428         return 0;
 429 }
 430
 431 static int
 432 parse_ordering_arg(const char *key __rte_unused,
 433                 const char *value, void *extra_args)
 434 {
 435         struct scheduler_init_params *param = extra_args;
 436         uint32_t i;
 437
 438         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
 439                 if (strcmp(value, scheduler_ordering_map[i].name) == 0) {
 440                         param->enable_ordering =
 441                                         scheduler_ordering_map[i].val;
 442                         break;
 443                 }
 444         }
 445
 446         if (i == RTE_DIM(scheduler_ordering_map)) {
 447                 CR_SCHED_LOG(ERR, "Unrecognized input.");
 448                 return -EINVAL;
 449         }
 450
 451         return 0;
 452 }
 453
 454 static int
 455 scheduler_parse_init_params(struct scheduler_init_params *params,
 456                 const char *input_args)
 457 {
 458         struct rte_kvargs *kvlist = NULL;
 459         int ret = 0;
 460
 461         if (params == NULL)
 462                 return -EINVAL;
 463
 464         if (input_args) {
 465                 kvlist = rte_kvargs_parse(input_args,
 466                                 scheduler_valid_params);
 467                 if (kvlist == NULL)
 468                         return -1;
 469
 470                 ret = rte_kvargs_process(kvlist,
 471                                 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
 472                                 &parse_integer_arg,
 473                                 &params->def_p.max_nb_queue_pairs);
 474                 if (ret < 0)
 475                         goto free_kvlist;
 476
 477                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID,
 478                                 &parse_integer_arg,
 479                                 &params->def_p.socket_id);
 480                 if (ret < 0)
 481                         goto free_kvlist;
 482
 483                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK,
 484                                 &parse_coremask_arg,
 485                                 params);
 486                 if (ret < 0)
 487                         goto free_kvlist;
 488
 489                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST,
 490                                 &parse_corelist_arg,
 491                                 params);
 492                 if (ret < 0)
 493                         goto free_kvlist;
 494
 495                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME,
 496                                 &parse_name_arg,
 497                                 &params->def_p);
 498                 if (ret < 0)
 499                         goto free_kvlist;
 500
 501                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SLAVE,
 502                                 &parse_slave_arg, params);
 503                 if (ret < 0)
 504                         goto free_kvlist;
 505
 506                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE,
 507                                 &parse_mode_arg, params);
 508                 if (ret < 0)
 509                         goto free_kvlist;
 510
 511                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE_PARAM,
 512                                 &parse_mode_param_arg, params);
 513                 if (ret < 0)
 514                         goto free_kvlist;
 515
 516                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING,
 517                                 &parse_ordering_arg, params);
 518                 if (ret < 0)
 519                         goto free_kvlist;
 520         }
 521
 522 free_kvlist:
 523         rte_kvargs_free(kvlist);
 524         return ret;
 525 }
 526
 527 static int
 528 cryptodev_scheduler_probe(struct rte_vdev_device *vdev)
 529 {
 530         struct scheduler_init_params init_params = {
 531                 .def_p = {
 532                         "",
 533                         sizeof(struct scheduler_ctx),
 534                         rte_socket_id(),
 535                         RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
 536                 },
 537                 .nb_slaves = 0,
 538                 .mode = CDEV_SCHED_MODE_NOT_SET,
 539                 .enable_ordering = 0,
 540                 .slave_names = { {0} }
 541         };
 542         const char *name;
 543
 544         name = rte_vdev_device_name(vdev);
 545         if (name == NULL)
 546                 return -EINVAL;
 547
 548         scheduler_parse_init_params(&init_params,
 549                                     rte_vdev_device_args(vdev));
 550
 551
 552         return cryptodev_scheduler_create(name,
 553                                         vdev,
 554                                         &init_params);
 555 }
 556
 557 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = {
 558         .probe = cryptodev_scheduler_probe,
 559         .remove = cryptodev_scheduler_remove
 560 };
 561
 562 static struct cryptodev_driver scheduler_crypto_drv;
 563
 564 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD,
 565         cryptodev_scheduler_pmd_drv);
 566 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD,
 567         "max_nb_queue_pairs=<int> "
 568         "socket_id=<int> "
 569         "slave=<name>");
 570 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv,
 571                 cryptodev_scheduler_pmd_drv.driver,
 572                 cryptodev_driver_id);