drivers/crypto/scheduler/scheduler_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4 #include <rte_common.h>
   5 #include <rte_hexdump.h>
   6 #include <rte_cryptodev.h>
   7 #include <rte_cryptodev_pmd.h>
   8 #include <rte_bus_vdev.h>
   9 #include <rte_malloc.h>
  10 #include <rte_cpuflags.h>
  11 #include <rte_reorder.h>
  12
  13 #include "rte_cryptodev_scheduler.h"
  14 #include "scheduler_pmd_private.h"
  15
  16 uint8_t cryptodev_driver_id;
  17
  18 struct scheduler_init_params {
  19         struct rte_cryptodev_pmd_init_params def_p;
  20         uint32_t nb_slaves;
  21         enum rte_cryptodev_scheduler_mode mode;
  22         uint32_t enable_ordering;
  23         uint16_t wc_pool[RTE_MAX_LCORE];
  24         uint16_t nb_wc;
  25         char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]
  26                         [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
  27 };
  28
  29 #define RTE_CRYPTODEV_VDEV_NAME                 ("name")
  30 #define RTE_CRYPTODEV_VDEV_SLAVE                ("slave")
  31 #define RTE_CRYPTODEV_VDEV_MODE                 ("mode")
  32 #define RTE_CRYPTODEV_VDEV_ORDERING             ("ordering")
  33 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG        ("max_nb_queue_pairs")
  34 #define RTE_CRYPTODEV_VDEV_SOCKET_ID            ("socket_id")
  35 #define RTE_CRYPTODEV_VDEV_COREMASK             ("coremask")
  36 #define RTE_CRYPTODEV_VDEV_CORELIST             ("corelist")
  37
  38 const char *scheduler_valid_params[] = {
  39         RTE_CRYPTODEV_VDEV_NAME,
  40         RTE_CRYPTODEV_VDEV_SLAVE,
  41         RTE_CRYPTODEV_VDEV_MODE,
  42         RTE_CRYPTODEV_VDEV_ORDERING,
  43         RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
  44         RTE_CRYPTODEV_VDEV_SOCKET_ID,
  45         RTE_CRYPTODEV_VDEV_COREMASK,
  46         RTE_CRYPTODEV_VDEV_CORELIST
  47 };
  48
  49 struct scheduler_parse_map {
  50         const char *name;
  51         uint32_t val;
  52 };
  53
  54 const struct scheduler_parse_map scheduler_mode_map[] = {
  55         {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN),
  56                         CDEV_SCHED_MODE_ROUNDROBIN},
  57         {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR),
  58                         CDEV_SCHED_MODE_PKT_SIZE_DISTR},
  59         {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER),
  60                         CDEV_SCHED_MODE_FAILOVER},
  61         {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE),
  62                         CDEV_SCHED_MODE_MULTICORE}
  63 };
  64
  65 const struct scheduler_parse_map scheduler_ordering_map[] = {
  66                 {"enable", 1},
  67                 {"disable", 0}
  68 };
  69
  70 static int
  71 cryptodev_scheduler_create(const char *name,
  72                 struct rte_vdev_device *vdev,
  73                 struct scheduler_init_params *init_params)
  74 {
  75         struct rte_cryptodev *dev;
  76         struct scheduler_ctx *sched_ctx;
  77         uint32_t i;
  78         int ret;
  79
  80         dev = rte_cryptodev_pmd_create(name, &vdev->device,
  81                         &init_params->def_p);
  82         if (dev == NULL) {
  83                 CR_SCHED_LOG(ERR, "driver %s: failed to create cryptodev vdev",
  84                         name);
  85                 return -EFAULT;
  86         }
  87
  88         dev->driver_id = cryptodev_driver_id;
  89         dev->dev_ops = rte_crypto_scheduler_pmd_ops;
  90
  91         sched_ctx = dev->data->dev_private;
  92         sched_ctx->max_nb_queue_pairs =
  93                         init_params->def_p.max_nb_queue_pairs;
  94
  95         if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) {
  96                 uint16_t i;
  97
  98                 sched_ctx->nb_wc = init_params->nb_wc;
  99
 100                 for (i = 0; i < sched_ctx->nb_wc; i++) {
 101                         sched_ctx->wc_pool[i] = init_params->wc_pool[i];
 102                         CR_SCHED_LOG(INFO, "  Worker core[%u]=%u added",
 103                                 i, sched_ctx->wc_pool[i]);
 104                 }
 105         }
 106
 107         if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED &&
 108                         init_params->mode < CDEV_SCHED_MODE_COUNT) {
 109                 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id,
 110                         init_params->mode);
 111                 if (ret < 0) {
 112                         rte_cryptodev_pmd_release_device(dev);
 113                         return ret;
 114                 }
 115
 116                 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
 117                         if (scheduler_mode_map[i].val != sched_ctx->mode)
 118                                 continue;
 119
 120                         CR_SCHED_LOG(INFO, "  Scheduling mode = %s",
 121                                         scheduler_mode_map[i].name);
 122                         break;
 123                 }
 124         }
 125
 126         sched_ctx->reordering_enabled = init_params->enable_ordering;
 127
 128         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
 129                 if (scheduler_ordering_map[i].val !=
 130                                 sched_ctx->reordering_enabled)
 131                         continue;
 132
 133                 CR_SCHED_LOG(INFO, "  Packet ordering = %s",
 134                                 scheduler_ordering_map[i].name);
 135
 136                 break;
 137         }
 138
 139         for (i = 0; i < init_params->nb_slaves; i++) {
 140                 sched_ctx->init_slave_names[sched_ctx->nb_init_slaves] =
 141                         rte_zmalloc_socket(
 142                                 NULL,
 143                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0,
 144                                 SOCKET_ID_ANY);
 145
 146                 if (!sched_ctx->init_slave_names[
 147                                 sched_ctx->nb_init_slaves]) {
 148                         CR_SCHED_LOG(ERR, "driver %s: Insufficient memory",
 149                                         name);
 150                         return -ENOMEM;
 151                 }
 152
 153                 strncpy(sched_ctx->init_slave_names[
 154                                         sched_ctx->nb_init_slaves],
 155                                 init_params->slave_names[i],
 156                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
 157
 158                 sched_ctx->nb_init_slaves++;
 159         }
 160
 161         /*
 162          * Initialize capabilities structure as an empty structure,
 163          * in case device information is requested when no slaves are attached
 164          */
 165         sched_ctx->capabilities = rte_zmalloc_socket(NULL,
 166                         sizeof(struct rte_cryptodev_capabilities),
 167                         0, SOCKET_ID_ANY);
 168
 169         if (!sched_ctx->capabilities) {
 170                 CR_SCHED_LOG(ERR, "Not enough memory for capability "
 171                                 "information");
 172                 return -ENOMEM;
 173         }
 174
 175         return 0;
 176 }
 177
 178 static int
 179 cryptodev_scheduler_remove(struct rte_vdev_device *vdev)
 180 {
 181         const char *name;
 182         struct rte_cryptodev *dev;
 183         struct scheduler_ctx *sched_ctx;
 184
 185         if (vdev == NULL)
 186                 return -EINVAL;
 187
 188         name = rte_vdev_device_name(vdev);
 189         dev = rte_cryptodev_pmd_get_named_dev(name);
 190         if (dev == NULL)
 191                 return -EINVAL;
 192
 193         sched_ctx = dev->data->dev_private;
 194
 195         if (sched_ctx->nb_slaves) {
 196                 uint32_t i;
 197
 198                 for (i = 0; i < sched_ctx->nb_slaves; i++)
 199                         rte_cryptodev_scheduler_slave_detach(dev->data->dev_id,
 200                                         sched_ctx->slaves[i].dev_id);
 201         }
 202
 203         return rte_cryptodev_pmd_destroy(dev);
 204 }
 205
 206 /** Parse integer from integer argument */
 207 static int
 208 parse_integer_arg(const char *key __rte_unused,
 209                 const char *value, void *extra_args)
 210 {
 211         int *i = (int *) extra_args;
 212
 213         *i = atoi(value);
 214         if (*i < 0) {
 215                 CR_SCHED_LOG(ERR, "Argument has to be positive.");
 216                 return -EINVAL;
 217         }
 218
 219         return 0;
 220 }
 221
 222 /** Parse integer from hexadecimal integer argument */
 223 static int
 224 parse_coremask_arg(const char *key __rte_unused,
 225                 const char *value, void *extra_args)
 226 {
 227         int i, j, val;
 228         uint16_t idx = 0;
 229         char c;
 230         struct scheduler_init_params *params = extra_args;
 231
 232         params->nb_wc = 0;
 233
 234         if (value == NULL)
 235                 return -1;
 236         /* Remove all blank characters ahead and after .
 237          * Remove 0x/0X if exists.
 238          */
 239         while (isblank(*value))
 240                 value++;
 241         if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X')))
 242                 value += 2;
 243         i = strlen(value);
 244         while ((i > 0) && isblank(value[i - 1]))
 245                 i--;
 246
 247         if (i == 0)
 248                 return -1;
 249
 250         for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
 251                 c = value[i];
 252                 if (isxdigit(c) == 0) {
 253                         /* invalid characters */
 254                         return -1;
 255                 }
 256                 if (isdigit(c))
 257                         val = c - '0';
 258                 else if (isupper(c))
 259                         val = c - 'A' + 10;
 260                 else
 261                         val = c - 'a' + 10;
 262
 263                 for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) {
 264                         if ((1 << j) & val)
 265                                 params->wc_pool[params->nb_wc++] = idx;
 266                 }
 267         }
 268
 269         return 0;
 270 }
 271
 272 /** Parse integer from list of integers argument */
 273 static int
 274 parse_corelist_arg(const char *key __rte_unused,
 275                 const char *value, void *extra_args)
 276 {
 277         struct scheduler_init_params *params = extra_args;
 278
 279         params->nb_wc = 0;
 280
 281         const char *token = value;
 282
 283         while (isdigit(token[0])) {
 284                 char *rval;
 285                 unsigned int core = strtoul(token, &rval, 10);
 286
 287                 if (core >= RTE_MAX_LCORE) {
 288                         CR_SCHED_LOG(ERR, "Invalid worker core %u, should be smaller "
 289                                    "than %u.", core, RTE_MAX_LCORE);
 290                 }
 291                 params->wc_pool[params->nb_wc++] = (uint16_t)core;
 292                 token = (const char *)rval;
 293                 if (token[0] == '\0')
 294                         break;
 295                 token++;
 296         }
 297
 298         return 0;
 299 }
 300
 301 /** Parse name */
 302 static int
 303 parse_name_arg(const char *key __rte_unused,
 304                 const char *value, void *extra_args)
 305 {
 306         struct rte_cryptodev_pmd_init_params *params = extra_args;
 307
 308         if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
 309                 CR_SCHED_LOG(ERR, "Invalid name %s, should be less than "
 310                                 "%u bytes.", value,
 311                                 RTE_CRYPTODEV_NAME_MAX_LEN - 1);
 312                 return -EINVAL;
 313         }
 314
 315         strncpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);
 316
 317         return 0;
 318 }
 319
 320 /** Parse slave */
 321 static int
 322 parse_slave_arg(const char *key __rte_unused,
 323                 const char *value, void *extra_args)
 324 {
 325         struct scheduler_init_params *param = extra_args;
 326
 327         if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
 328                 CR_SCHED_LOG(ERR, "Too many slaves.");
 329                 return -ENOMEM;
 330         }
 331
 332         strncpy(param->slave_names[param->nb_slaves++], value,
 333                         RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
 334
 335         return 0;
 336 }
 337
 338 static int
 339 parse_mode_arg(const char *key __rte_unused,
 340                 const char *value, void *extra_args)
 341 {
 342         struct scheduler_init_params *param = extra_args;
 343         uint32_t i;
 344
 345         for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
 346                 if (strcmp(value, scheduler_mode_map[i].name) == 0) {
 347                         param->mode = (enum rte_cryptodev_scheduler_mode)
 348                                         scheduler_mode_map[i].val;
 349                         break;
 350                 }
 351         }
 352
 353         if (i == RTE_DIM(scheduler_mode_map)) {
 354                 CR_SCHED_LOG(ERR, "Unrecognized input.");
 355                 return -EINVAL;
 356         }
 357
 358         return 0;
 359 }
 360
 361 static int
 362 parse_ordering_arg(const char *key __rte_unused,
 363                 const char *value, void *extra_args)
 364 {
 365         struct scheduler_init_params *param = extra_args;
 366         uint32_t i;
 367
 368         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
 369                 if (strcmp(value, scheduler_ordering_map[i].name) == 0) {
 370                         param->enable_ordering =
 371                                         scheduler_ordering_map[i].val;
 372                         break;
 373                 }
 374         }
 375
 376         if (i == RTE_DIM(scheduler_ordering_map)) {
 377                 CR_SCHED_LOG(ERR, "Unrecognized input.");
 378                 return -EINVAL;
 379         }
 380
 381         return 0;
 382 }
 383
 384 static int
 385 scheduler_parse_init_params(struct scheduler_init_params *params,
 386                 const char *input_args)
 387 {
 388         struct rte_kvargs *kvlist = NULL;
 389         int ret = 0;
 390
 391         if (params == NULL)
 392                 return -EINVAL;
 393
 394         if (input_args) {
 395                 kvlist = rte_kvargs_parse(input_args,
 396                                 scheduler_valid_params);
 397                 if (kvlist == NULL)
 398                         return -1;
 399
 400                 ret = rte_kvargs_process(kvlist,
 401                                 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
 402                                 &parse_integer_arg,
 403                                 &params->def_p.max_nb_queue_pairs);
 404                 if (ret < 0)
 405                         goto free_kvlist;
 406
 407                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID,
 408                                 &parse_integer_arg,
 409                                 &params->def_p.socket_id);
 410                 if (ret < 0)
 411                         goto free_kvlist;
 412
 413                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK,
 414                                 &parse_coremask_arg,
 415                                 params);
 416                 if (ret < 0)
 417                         goto free_kvlist;
 418
 419                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST,
 420                                 &parse_corelist_arg,
 421                                 params);
 422                 if (ret < 0)
 423                         goto free_kvlist;
 424
 425                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME,
 426                                 &parse_name_arg,
 427                                 &params->def_p);
 428                 if (ret < 0)
 429                         goto free_kvlist;
 430
 431                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SLAVE,
 432                                 &parse_slave_arg, params);
 433                 if (ret < 0)
 434                         goto free_kvlist;
 435
 436                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE,
 437                                 &parse_mode_arg, params);
 438                 if (ret < 0)
 439                         goto free_kvlist;
 440
 441                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING,
 442                                 &parse_ordering_arg, params);
 443                 if (ret < 0)
 444                         goto free_kvlist;
 445         }
 446
 447 free_kvlist:
 448         rte_kvargs_free(kvlist);
 449         return ret;
 450 }
 451
 452 static int
 453 cryptodev_scheduler_probe(struct rte_vdev_device *vdev)
 454 {
 455         struct scheduler_init_params init_params = {
 456                 .def_p = {
 457                         "",
 458                         sizeof(struct scheduler_ctx),
 459                         rte_socket_id(),
 460                         RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
 461                 },
 462                 .nb_slaves = 0,
 463                 .mode = CDEV_SCHED_MODE_NOT_SET,
 464                 .enable_ordering = 0,
 465                 .slave_names = { {0} }
 466         };
 467         const char *name;
 468
 469         name = rte_vdev_device_name(vdev);
 470         if (name == NULL)
 471                 return -EINVAL;
 472
 473         scheduler_parse_init_params(&init_params,
 474                                     rte_vdev_device_args(vdev));
 475
 476
 477         return cryptodev_scheduler_create(name,
 478                                         vdev,
 479                                         &init_params);
 480 }
 481
 482 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = {
 483         .probe = cryptodev_scheduler_probe,
 484         .remove = cryptodev_scheduler_remove
 485 };
 486
 487 static struct cryptodev_driver scheduler_crypto_drv;
 488
 489 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD,
 490         cryptodev_scheduler_pmd_drv);
 491 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD,
 492         "max_nb_queue_pairs=<int> "
 493         "socket_id=<int> "
 494         "slave=<name>");
 495 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv,
 496                 cryptodev_scheduler_pmd_drv.driver,
 497                 cryptodev_driver_id);