drivers/crypto/scheduler/scheduler_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4 #include <rte_common.h>
   5 #include <rte_hexdump.h>
   6 #include <rte_cryptodev.h>
   7 #include <rte_cryptodev_pmd.h>
   8 #include <rte_bus_vdev.h>
   9 #include <rte_malloc.h>
  10 #include <rte_cpuflags.h>
  11 #include <rte_reorder.h>
  12
  13 #include "rte_cryptodev_scheduler.h"
  14 #include "scheduler_pmd_private.h"
  15
  16 uint8_t cryptodev_driver_id;
  17
  18 struct scheduler_init_params {
  19         struct rte_cryptodev_pmd_init_params def_p;
  20         uint32_t nb_slaves;
  21         enum rte_cryptodev_scheduler_mode mode;
  22         uint32_t enable_ordering;
  23         uint16_t wc_pool[RTE_MAX_LCORE];
  24         uint16_t nb_wc;
  25         char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]
  26                         [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN];
  27 };
  28
  29 #define RTE_CRYPTODEV_VDEV_NAME                 ("name")
  30 #define RTE_CRYPTODEV_VDEV_SLAVE                ("slave")
  31 #define RTE_CRYPTODEV_VDEV_MODE                 ("mode")
  32 #define RTE_CRYPTODEV_VDEV_ORDERING             ("ordering")
  33 #define RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG        ("max_nb_queue_pairs")
  34 #define RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG      ("max_nb_sessions")
  35 #define RTE_CRYPTODEV_VDEV_SOCKET_ID            ("socket_id")
  36 #define RTE_CRYPTODEV_VDEV_COREMASK             ("coremask")
  37 #define RTE_CRYPTODEV_VDEV_CORELIST             ("corelist")
  38
  39 const char *scheduler_valid_params[] = {
  40         RTE_CRYPTODEV_VDEV_NAME,
  41         RTE_CRYPTODEV_VDEV_SLAVE,
  42         RTE_CRYPTODEV_VDEV_MODE,
  43         RTE_CRYPTODEV_VDEV_ORDERING,
  44         RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
  45         RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG,
  46         RTE_CRYPTODEV_VDEV_SOCKET_ID,
  47         RTE_CRYPTODEV_VDEV_COREMASK,
  48         RTE_CRYPTODEV_VDEV_CORELIST
  49 };
  50
  51 struct scheduler_parse_map {
  52         const char *name;
  53         uint32_t val;
  54 };
  55
  56 const struct scheduler_parse_map scheduler_mode_map[] = {
  57         {RTE_STR(SCHEDULER_MODE_NAME_ROUND_ROBIN),
  58                         CDEV_SCHED_MODE_ROUNDROBIN},
  59         {RTE_STR(SCHEDULER_MODE_NAME_PKT_SIZE_DISTR),
  60                         CDEV_SCHED_MODE_PKT_SIZE_DISTR},
  61         {RTE_STR(SCHEDULER_MODE_NAME_FAIL_OVER),
  62                         CDEV_SCHED_MODE_FAILOVER},
  63         {RTE_STR(SCHEDULER_MODE_NAME_MULTI_CORE),
  64                         CDEV_SCHED_MODE_MULTICORE}
  65 };
  66
  67 const struct scheduler_parse_map scheduler_ordering_map[] = {
  68                 {"enable", 1},
  69                 {"disable", 0}
  70 };
  71
  72 static int
  73 cryptodev_scheduler_create(const char *name,
  74                 struct rte_vdev_device *vdev,
  75                 struct scheduler_init_params *init_params)
  76 {
  77         struct rte_cryptodev *dev;
  78         struct scheduler_ctx *sched_ctx;
  79         uint32_t i;
  80         int ret;
  81
  82         dev = rte_cryptodev_pmd_create(name, &vdev->device,
  83                         &init_params->def_p);
  84         if (dev == NULL) {
  85                 CS_LOG_ERR("driver %s: failed to create cryptodev vdev",
  86                         name);
  87                 return -EFAULT;
  88         }
  89
  90         dev->driver_id = cryptodev_driver_id;
  91         dev->dev_ops = rte_crypto_scheduler_pmd_ops;
  92
  93         sched_ctx = dev->data->dev_private;
  94         sched_ctx->max_nb_queue_pairs =
  95                         init_params->def_p.max_nb_queue_pairs;
  96
  97         if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) {
  98                 uint16_t i;
  99
 100                 sched_ctx->nb_wc = init_params->nb_wc;
 101
 102                 for (i = 0; i < sched_ctx->nb_wc; i++) {
 103                         sched_ctx->wc_pool[i] = init_params->wc_pool[i];
 104                         RTE_LOG(INFO, PMD, "  Worker core[%u]=%u added\n",
 105                                 i, sched_ctx->wc_pool[i]);
 106                 }
 107         }
 108
 109         if (init_params->mode > CDEV_SCHED_MODE_USERDEFINED &&
 110                         init_params->mode < CDEV_SCHED_MODE_COUNT) {
 111                 ret = rte_cryptodev_scheduler_mode_set(dev->data->dev_id,
 112                         init_params->mode);
 113                 if (ret < 0) {
 114                         rte_cryptodev_pmd_release_device(dev);
 115                         return ret;
 116                 }
 117
 118                 for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
 119                         if (scheduler_mode_map[i].val != sched_ctx->mode)
 120                                 continue;
 121
 122                         RTE_LOG(INFO, PMD, "  Scheduling mode = %s\n",
 123                                         scheduler_mode_map[i].name);
 124                         break;
 125                 }
 126         }
 127
 128         sched_ctx->reordering_enabled = init_params->enable_ordering;
 129
 130         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
 131                 if (scheduler_ordering_map[i].val !=
 132                                 sched_ctx->reordering_enabled)
 133                         continue;
 134
 135                 RTE_LOG(INFO, PMD, "  Packet ordering = %s\n",
 136                                 scheduler_ordering_map[i].name);
 137
 138                 break;
 139         }
 140
 141         for (i = 0; i < init_params->nb_slaves; i++) {
 142                 sched_ctx->init_slave_names[sched_ctx->nb_init_slaves] =
 143                         rte_zmalloc_socket(
 144                                 NULL,
 145                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN, 0,
 146                                 SOCKET_ID_ANY);
 147
 148                 if (!sched_ctx->init_slave_names[
 149                                 sched_ctx->nb_init_slaves]) {
 150                         CS_LOG_ERR("driver %s: Insufficient memory",
 151                                         name);
 152                         return -ENOMEM;
 153                 }
 154
 155                 strncpy(sched_ctx->init_slave_names[
 156                                         sched_ctx->nb_init_slaves],
 157                                 init_params->slave_names[i],
 158                                 RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
 159
 160                 sched_ctx->nb_init_slaves++;
 161         }
 162
 163         /*
 164          * Initialize capabilities structure as an empty structure,
 165          * in case device information is requested when no slaves are attached
 166          */
 167         sched_ctx->capabilities = rte_zmalloc_socket(NULL,
 168                         sizeof(struct rte_cryptodev_capabilities),
 169                         0, SOCKET_ID_ANY);
 170
 171         if (!sched_ctx->capabilities) {
 172                 RTE_LOG(ERR, PMD, "Not enough memory for capability "
 173                                 "information\n");
 174                 return -ENOMEM;
 175         }
 176
 177         return 0;
 178 }
 179
 180 static int
 181 cryptodev_scheduler_remove(struct rte_vdev_device *vdev)
 182 {
 183         const char *name;
 184         struct rte_cryptodev *dev;
 185         struct scheduler_ctx *sched_ctx;
 186
 187         if (vdev == NULL)
 188                 return -EINVAL;
 189
 190         name = rte_vdev_device_name(vdev);
 191         dev = rte_cryptodev_pmd_get_named_dev(name);
 192         if (dev == NULL)
 193                 return -EINVAL;
 194
 195         sched_ctx = dev->data->dev_private;
 196
 197         if (sched_ctx->nb_slaves) {
 198                 uint32_t i;
 199
 200                 for (i = 0; i < sched_ctx->nb_slaves; i++)
 201                         rte_cryptodev_scheduler_slave_detach(dev->data->dev_id,
 202                                         sched_ctx->slaves[i].dev_id);
 203         }
 204
 205         return rte_cryptodev_pmd_destroy(dev);
 206 }
 207
 208 /** Parse integer from integer argument */
 209 static int
 210 parse_integer_arg(const char *key __rte_unused,
 211                 const char *value, void *extra_args)
 212 {
 213         int *i = (int *) extra_args;
 214
 215         *i = atoi(value);
 216         if (*i < 0) {
 217                 CS_LOG_ERR("Argument has to be positive.\n");
 218                 return -EINVAL;
 219         }
 220
 221         return 0;
 222 }
 223
 224 /** Parse integer from hexadecimal integer argument */
 225 static int
 226 parse_coremask_arg(const char *key __rte_unused,
 227                 const char *value, void *extra_args)
 228 {
 229         int i, j, val;
 230         uint16_t idx = 0;
 231         char c;
 232         struct scheduler_init_params *params = extra_args;
 233
 234         params->nb_wc = 0;
 235
 236         if (value == NULL)
 237                 return -1;
 238         /* Remove all blank characters ahead and after .
 239          * Remove 0x/0X if exists.
 240          */
 241         while (isblank(*value))
 242                 value++;
 243         if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X')))
 244                 value += 2;
 245         i = strlen(value);
 246         while ((i > 0) && isblank(value[i - 1]))
 247                 i--;
 248
 249         if (i == 0)
 250                 return -1;
 251
 252         for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
 253                 c = value[i];
 254                 if (isxdigit(c) == 0) {
 255                         /* invalid characters */
 256                         return -1;
 257                 }
 258                 if (isdigit(c))
 259                         val = c - '0';
 260                 else if (isupper(c))
 261                         val = c - 'A' + 10;
 262                 else
 263                         val = c - 'a' + 10;
 264
 265                 for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) {
 266                         if ((1 << j) & val)
 267                                 params->wc_pool[params->nb_wc++] = idx;
 268                 }
 269         }
 270
 271         return 0;
 272 }
 273
 274 /** Parse integer from list of integers argument */
 275 static int
 276 parse_corelist_arg(const char *key __rte_unused,
 277                 const char *value, void *extra_args)
 278 {
 279         struct scheduler_init_params *params = extra_args;
 280
 281         params->nb_wc = 0;
 282
 283         const char *token = value;
 284
 285         while (isdigit(token[0])) {
 286                 char *rval;
 287                 unsigned int core = strtoul(token, &rval, 10);
 288
 289                 if (core >= RTE_MAX_LCORE) {
 290                         CS_LOG_ERR("Invalid worker core %u, should be smaller "
 291                                    "than %u.\n", core, RTE_MAX_LCORE);
 292                 }
 293                 params->wc_pool[params->nb_wc++] = (uint16_t)core;
 294                 token = (const char *)rval;
 295                 if (token[0] == '\0')
 296                         break;
 297                 token++;
 298         }
 299
 300         return 0;
 301 }
 302
 303 /** Parse name */
 304 static int
 305 parse_name_arg(const char *key __rte_unused,
 306                 const char *value, void *extra_args)
 307 {
 308         struct rte_cryptodev_pmd_init_params *params = extra_args;
 309
 310         if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
 311                 CS_LOG_ERR("Invalid name %s, should be less than "
 312                                 "%u bytes.\n", value,
 313                                 RTE_CRYPTODEV_NAME_MAX_LEN - 1);
 314                 return -EINVAL;
 315         }
 316
 317         strncpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);
 318
 319         return 0;
 320 }
 321
 322 /** Parse slave */
 323 static int
 324 parse_slave_arg(const char *key __rte_unused,
 325                 const char *value, void *extra_args)
 326 {
 327         struct scheduler_init_params *param = extra_args;
 328
 329         if (param->nb_slaves >= RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
 330                 CS_LOG_ERR("Too many slaves.\n");
 331                 return -ENOMEM;
 332         }
 333
 334         strncpy(param->slave_names[param->nb_slaves++], value,
 335                         RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN - 1);
 336
 337         return 0;
 338 }
 339
 340 static int
 341 parse_mode_arg(const char *key __rte_unused,
 342                 const char *value, void *extra_args)
 343 {
 344         struct scheduler_init_params *param = extra_args;
 345         uint32_t i;
 346
 347         for (i = 0; i < RTE_DIM(scheduler_mode_map); i++) {
 348                 if (strcmp(value, scheduler_mode_map[i].name) == 0) {
 349                         param->mode = (enum rte_cryptodev_scheduler_mode)
 350                                         scheduler_mode_map[i].val;
 351                         break;
 352                 }
 353         }
 354
 355         if (i == RTE_DIM(scheduler_mode_map)) {
 356                 CS_LOG_ERR("Unrecognized input.\n");
 357                 return -EINVAL;
 358         }
 359
 360         return 0;
 361 }
 362
 363 static int
 364 parse_ordering_arg(const char *key __rte_unused,
 365                 const char *value, void *extra_args)
 366 {
 367         struct scheduler_init_params *param = extra_args;
 368         uint32_t i;
 369
 370         for (i = 0; i < RTE_DIM(scheduler_ordering_map); i++) {
 371                 if (strcmp(value, scheduler_ordering_map[i].name) == 0) {
 372                         param->enable_ordering =
 373                                         scheduler_ordering_map[i].val;
 374                         break;
 375                 }
 376         }
 377
 378         if (i == RTE_DIM(scheduler_ordering_map)) {
 379                 CS_LOG_ERR("Unrecognized input.\n");
 380                 return -EINVAL;
 381         }
 382
 383         return 0;
 384 }
 385
 386 static int
 387 scheduler_parse_init_params(struct scheduler_init_params *params,
 388                 const char *input_args)
 389 {
 390         struct rte_kvargs *kvlist = NULL;
 391         int ret = 0;
 392
 393         if (params == NULL)
 394                 return -EINVAL;
 395
 396         if (input_args) {
 397                 kvlist = rte_kvargs_parse(input_args,
 398                                 scheduler_valid_params);
 399                 if (kvlist == NULL)
 400                         return -1;
 401
 402                 ret = rte_kvargs_process(kvlist,
 403                                 RTE_CRYPTODEV_VDEV_MAX_NB_QP_ARG,
 404                                 &parse_integer_arg,
 405                                 &params->def_p.max_nb_queue_pairs);
 406                 if (ret < 0)
 407                         goto free_kvlist;
 408
 409                 ret = rte_kvargs_process(kvlist,
 410                                 RTE_CRYPTODEV_VDEV_MAX_NB_SESS_ARG,
 411                                 &parse_integer_arg,
 412                                 &params->def_p.max_nb_sessions);
 413                 if (ret < 0)
 414                         goto free_kvlist;
 415
 416                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SOCKET_ID,
 417                                 &parse_integer_arg,
 418                                 &params->def_p.socket_id);
 419                 if (ret < 0)
 420                         goto free_kvlist;
 421
 422                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_COREMASK,
 423                                 &parse_coremask_arg,
 424                                 params);
 425                 if (ret < 0)
 426                         goto free_kvlist;
 427
 428                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_CORELIST,
 429                                 &parse_corelist_arg,
 430                                 params);
 431                 if (ret < 0)
 432                         goto free_kvlist;
 433
 434                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_NAME,
 435                                 &parse_name_arg,
 436                                 &params->def_p);
 437                 if (ret < 0)
 438                         goto free_kvlist;
 439
 440                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_SLAVE,
 441                                 &parse_slave_arg, params);
 442                 if (ret < 0)
 443                         goto free_kvlist;
 444
 445                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_MODE,
 446                                 &parse_mode_arg, params);
 447                 if (ret < 0)
 448                         goto free_kvlist;
 449
 450                 ret = rte_kvargs_process(kvlist, RTE_CRYPTODEV_VDEV_ORDERING,
 451                                 &parse_ordering_arg, params);
 452                 if (ret < 0)
 453                         goto free_kvlist;
 454         }
 455
 456 free_kvlist:
 457         rte_kvargs_free(kvlist);
 458         return ret;
 459 }
 460
 461 static int
 462 cryptodev_scheduler_probe(struct rte_vdev_device *vdev)
 463 {
 464         struct scheduler_init_params init_params = {
 465                 .def_p = {
 466                         "",
 467                         sizeof(struct scheduler_ctx),
 468                         rte_socket_id(),
 469                         RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
 470                         RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
 471                 },
 472                 .nb_slaves = 0,
 473                 .mode = CDEV_SCHED_MODE_NOT_SET,
 474                 .enable_ordering = 0,
 475                 .slave_names = { {0} }
 476         };
 477         const char *name;
 478
 479         name = rte_vdev_device_name(vdev);
 480         if (name == NULL)
 481                 return -EINVAL;
 482
 483         scheduler_parse_init_params(&init_params,
 484                                     rte_vdev_device_args(vdev));
 485
 486
 487         return cryptodev_scheduler_create(name,
 488                                         vdev,
 489                                         &init_params);
 490 }
 491
 492 static struct rte_vdev_driver cryptodev_scheduler_pmd_drv = {
 493         .probe = cryptodev_scheduler_probe,
 494         .remove = cryptodev_scheduler_remove
 495 };
 496
 497 static struct cryptodev_driver scheduler_crypto_drv;
 498
 499 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_SCHEDULER_PMD,
 500         cryptodev_scheduler_pmd_drv);
 501 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_SCHEDULER_PMD,
 502         "max_nb_queue_pairs=<int> "
 503         "max_nb_sessions=<int> "
 504         "socket_id=<int> "
 505         "slave=<name>");
 506 RTE_PMD_REGISTER_CRYPTO_DRIVER(scheduler_crypto_drv,
 507                 cryptodev_scheduler_pmd_drv.driver,
 508                 cryptodev_driver_id);