drivers/crypto/scheduler/rte_cryptodev_scheduler.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4 #include <rte_reorder.h>
   5 #include <rte_cryptodev.h>
   6 #include <rte_cryptodev_pmd.h>
   7 #include <rte_malloc.h>
   8
   9 #include "rte_cryptodev_scheduler.h"
  10 #include "scheduler_pmd_private.h"
  11
  12 /** update the scheduler pmd's capability with attaching device's
  13  *  capability.
  14  *  For each device to be attached, the scheduler's capability should be
  15  *  the common capability set of all slaves
  16  **/
  17 static uint32_t
  18 sync_caps(struct rte_cryptodev_capabilities *caps,
  19                 uint32_t nb_caps,
  20                 const struct rte_cryptodev_capabilities *slave_caps)
  21 {
  22         uint32_t sync_nb_caps = nb_caps, nb_slave_caps = 0;
  23         uint32_t i;
  24
  25         while (slave_caps[nb_slave_caps].op != RTE_CRYPTO_OP_TYPE_UNDEFINED)
  26                 nb_slave_caps++;
  27
  28         if (nb_caps == 0) {
  29                 rte_memcpy(caps, slave_caps, sizeof(*caps) * nb_slave_caps);
  30                 return nb_slave_caps;
  31         }
  32
  33         for (i = 0; i < sync_nb_caps; i++) {
  34                 struct rte_cryptodev_capabilities *cap = &caps[i];
  35                 uint32_t j;
  36
  37                 for (j = 0; j < nb_slave_caps; j++) {
  38                         const struct rte_cryptodev_capabilities *s_cap =
  39                                         &slave_caps[j];
  40
  41                         if (s_cap->op != cap->op || s_cap->sym.xform_type !=
  42                                         cap->sym.xform_type)
  43                                 continue;
  44
  45                         if (s_cap->sym.xform_type ==
  46                                         RTE_CRYPTO_SYM_XFORM_AUTH) {
  47                                 if (s_cap->sym.auth.algo !=
  48                                                 cap->sym.auth.algo)
  49                                         continue;
  50
  51                                 cap->sym.auth.digest_size.min =
  52                                         s_cap->sym.auth.digest_size.min <
  53                                         cap->sym.auth.digest_size.min ?
  54                                         s_cap->sym.auth.digest_size.min :
  55                                         cap->sym.auth.digest_size.min;
  56                                 cap->sym.auth.digest_size.max =
  57                                         s_cap->sym.auth.digest_size.max <
  58                                         cap->sym.auth.digest_size.max ?
  59                                         s_cap->sym.auth.digest_size.max :
  60                                         cap->sym.auth.digest_size.max;
  61
  62                         }
  63
  64                         if (s_cap->sym.xform_type ==
  65                                         RTE_CRYPTO_SYM_XFORM_CIPHER)
  66                                 if (s_cap->sym.cipher.algo !=
  67                                                 cap->sym.cipher.algo)
  68                                         continue;
  69
  70                         /* no common cap found */
  71                         break;
  72                 }
  73
  74                 if (j < nb_slave_caps)
  75                         continue;
  76
  77                 /* remove a uncommon cap from the array */
  78                 for (j = i; j < sync_nb_caps - 1; j++)
  79                         rte_memcpy(&caps[j], &caps[j+1], sizeof(*cap));
  80
  81                 memset(&caps[sync_nb_caps - 1], 0, sizeof(*cap));
  82                 sync_nb_caps--;
  83         }
  84
  85         return sync_nb_caps;
  86 }
  87
  88 static int
  89 update_scheduler_capability(struct scheduler_ctx *sched_ctx)
  90 {
  91         struct rte_cryptodev_capabilities tmp_caps[256] = { {0} };
  92         uint32_t nb_caps = 0, i;
  93
  94         if (sched_ctx->capabilities) {
  95                 rte_free(sched_ctx->capabilities);
  96                 sched_ctx->capabilities = NULL;
  97         }
  98
  99         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 100                 struct rte_cryptodev_info dev_info;
 101
 102                 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info);
 103
 104                 nb_caps = sync_caps(tmp_caps, nb_caps, dev_info.capabilities);
 105                 if (nb_caps == 0)
 106                         return -1;
 107         }
 108
 109         sched_ctx->capabilities = rte_zmalloc_socket(NULL,
 110                         sizeof(struct rte_cryptodev_capabilities) *
 111                         (nb_caps + 1), 0, SOCKET_ID_ANY);
 112         if (!sched_ctx->capabilities)
 113                 return -ENOMEM;
 114
 115         rte_memcpy(sched_ctx->capabilities, tmp_caps,
 116                         sizeof(struct rte_cryptodev_capabilities) * nb_caps);
 117
 118         return 0;
 119 }
 120
 121 static void
 122 update_scheduler_feature_flag(struct rte_cryptodev *dev)
 123 {
 124         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 125         uint32_t i;
 126
 127         dev->feature_flags = 0;
 128
 129         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 130                 struct rte_cryptodev_info dev_info;
 131
 132                 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info);
 133
 134                 dev->feature_flags |= dev_info.feature_flags;
 135         }
 136 }
 137
 138 static void
 139 update_max_nb_qp(struct scheduler_ctx *sched_ctx)
 140 {
 141         uint32_t i;
 142         uint32_t max_nb_qp;
 143
 144         if (!sched_ctx->nb_slaves)
 145                 return;
 146
 147         max_nb_qp = sched_ctx->nb_slaves ? UINT32_MAX : 0;
 148
 149         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 150                 struct rte_cryptodev_info dev_info;
 151
 152                 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info);
 153                 max_nb_qp = dev_info.max_nb_queue_pairs < max_nb_qp ?
 154                                 dev_info.max_nb_queue_pairs : max_nb_qp;
 155         }
 156
 157         sched_ctx->max_nb_queue_pairs = max_nb_qp;
 158 }
 159
 160 /** Attach a device to the scheduler. */
 161 int
 162 rte_cryptodev_scheduler_slave_attach(uint8_t scheduler_id, uint8_t slave_id)
 163 {
 164         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 165         struct scheduler_ctx *sched_ctx;
 166         struct scheduler_slave *slave;
 167         struct rte_cryptodev_info dev_info;
 168         uint32_t i;
 169
 170         if (!dev) {
 171                 CS_LOG_ERR("Operation not supported");
 172                 return -ENOTSUP;
 173         }
 174
 175         if (dev->driver_id != cryptodev_driver_id) {
 176                 CS_LOG_ERR("Operation not supported");
 177                 return -ENOTSUP;
 178         }
 179
 180         if (dev->data->dev_started) {
 181                 CS_LOG_ERR("Illegal operation");
 182                 return -EBUSY;
 183         }
 184
 185         sched_ctx = dev->data->dev_private;
 186         if (sched_ctx->nb_slaves >=
 187                         RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
 188                 CS_LOG_ERR("Too many slaves attached");
 189                 return -ENOMEM;
 190         }
 191
 192         for (i = 0; i < sched_ctx->nb_slaves; i++)
 193                 if (sched_ctx->slaves[i].dev_id == slave_id) {
 194                         CS_LOG_ERR("Slave already added");
 195                         return -ENOTSUP;
 196                 }
 197
 198         slave = &sched_ctx->slaves[sched_ctx->nb_slaves];
 199
 200         rte_cryptodev_info_get(slave_id, &dev_info);
 201
 202         slave->dev_id = slave_id;
 203         slave->driver_id = dev_info.driver_id;
 204         sched_ctx->nb_slaves++;
 205
 206         if (update_scheduler_capability(sched_ctx) < 0) {
 207                 slave->dev_id = 0;
 208                 slave->driver_id = 0;
 209                 sched_ctx->nb_slaves--;
 210
 211                 CS_LOG_ERR("capabilities update failed");
 212                 return -ENOTSUP;
 213         }
 214
 215         update_scheduler_feature_flag(dev);
 216
 217         update_max_nb_qp(sched_ctx);
 218
 219         return 0;
 220 }
 221
 222 int
 223 rte_cryptodev_scheduler_slave_detach(uint8_t scheduler_id, uint8_t slave_id)
 224 {
 225         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 226         struct scheduler_ctx *sched_ctx;
 227         uint32_t i, slave_pos;
 228
 229         if (!dev) {
 230                 CS_LOG_ERR("Operation not supported");
 231                 return -ENOTSUP;
 232         }
 233
 234         if (dev->driver_id != cryptodev_driver_id) {
 235                 CS_LOG_ERR("Operation not supported");
 236                 return -ENOTSUP;
 237         }
 238
 239         if (dev->data->dev_started) {
 240                 CS_LOG_ERR("Illegal operation");
 241                 return -EBUSY;
 242         }
 243
 244         sched_ctx = dev->data->dev_private;
 245
 246         for (slave_pos = 0; slave_pos < sched_ctx->nb_slaves; slave_pos++)
 247                 if (sched_ctx->slaves[slave_pos].dev_id == slave_id)
 248                         break;
 249         if (slave_pos == sched_ctx->nb_slaves) {
 250                 CS_LOG_ERR("Cannot find slave");
 251                 return -ENOTSUP;
 252         }
 253
 254         if (sched_ctx->ops.slave_detach(dev, slave_id) < 0) {
 255                 CS_LOG_ERR("Failed to detach slave");
 256                 return -ENOTSUP;
 257         }
 258
 259         for (i = slave_pos; i < sched_ctx->nb_slaves - 1; i++) {
 260                 memcpy(&sched_ctx->slaves[i], &sched_ctx->slaves[i+1],
 261                                 sizeof(struct scheduler_slave));
 262         }
 263         memset(&sched_ctx->slaves[sched_ctx->nb_slaves - 1], 0,
 264                         sizeof(struct scheduler_slave));
 265         sched_ctx->nb_slaves--;
 266
 267         if (update_scheduler_capability(sched_ctx) < 0) {
 268                 CS_LOG_ERR("capabilities update failed");
 269                 return -ENOTSUP;
 270         }
 271
 272         update_scheduler_feature_flag(dev);
 273
 274         update_max_nb_qp(sched_ctx);
 275
 276         return 0;
 277 }
 278
 279 int
 280 rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id,
 281                 enum rte_cryptodev_scheduler_mode mode)
 282 {
 283         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 284         struct scheduler_ctx *sched_ctx;
 285
 286         if (!dev) {
 287                 CS_LOG_ERR("Operation not supported");
 288                 return -ENOTSUP;
 289         }
 290
 291         if (dev->driver_id != cryptodev_driver_id) {
 292                 CS_LOG_ERR("Operation not supported");
 293                 return -ENOTSUP;
 294         }
 295
 296         if (dev->data->dev_started) {
 297                 CS_LOG_ERR("Illegal operation");
 298                 return -EBUSY;
 299         }
 300
 301         sched_ctx = dev->data->dev_private;
 302
 303         if (mode == sched_ctx->mode)
 304                 return 0;
 305
 306         switch (mode) {
 307         case CDEV_SCHED_MODE_ROUNDROBIN:
 308                 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
 309                                 roundrobin_scheduler) < 0) {
 310                         CS_LOG_ERR("Failed to load scheduler");
 311                         return -1;
 312                 }
 313                 break;
 314         case CDEV_SCHED_MODE_PKT_SIZE_DISTR:
 315                 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
 316                                 pkt_size_based_distr_scheduler) < 0) {
 317                         CS_LOG_ERR("Failed to load scheduler");
 318                         return -1;
 319                 }
 320                 break;
 321         case CDEV_SCHED_MODE_FAILOVER:
 322                 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
 323                                 failover_scheduler) < 0) {
 324                         CS_LOG_ERR("Failed to load scheduler");
 325                         return -1;
 326                 }
 327                 break;
 328         case CDEV_SCHED_MODE_MULTICORE:
 329                 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
 330                                 multicore_scheduler) < 0) {
 331                         CS_LOG_ERR("Failed to load scheduler");
 332                         return -1;
 333                 }
 334                 break;
 335         default:
 336                 CS_LOG_ERR("Not yet supported");
 337                 return -ENOTSUP;
 338         }
 339
 340         return 0;
 341 }
 342
 343 enum rte_cryptodev_scheduler_mode
 344 rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id)
 345 {
 346         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 347         struct scheduler_ctx *sched_ctx;
 348
 349         if (!dev) {
 350                 CS_LOG_ERR("Operation not supported");
 351                 return -ENOTSUP;
 352         }
 353
 354         if (dev->driver_id != cryptodev_driver_id) {
 355                 CS_LOG_ERR("Operation not supported");
 356                 return -ENOTSUP;
 357         }
 358
 359         sched_ctx = dev->data->dev_private;
 360
 361         return sched_ctx->mode;
 362 }
 363
 364 int
 365 rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id,
 366                 uint32_t enable_reorder)
 367 {
 368         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 369         struct scheduler_ctx *sched_ctx;
 370
 371         if (!dev) {
 372                 CS_LOG_ERR("Operation not supported");
 373                 return -ENOTSUP;
 374         }
 375
 376         if (dev->driver_id != cryptodev_driver_id) {
 377                 CS_LOG_ERR("Operation not supported");
 378                 return -ENOTSUP;
 379         }
 380
 381         if (dev->data->dev_started) {
 382                 CS_LOG_ERR("Illegal operation");
 383                 return -EBUSY;
 384         }
 385
 386         sched_ctx = dev->data->dev_private;
 387
 388         sched_ctx->reordering_enabled = enable_reorder;
 389
 390         return 0;
 391 }
 392
 393 int
 394 rte_cryptodev_scheduler_ordering_get(uint8_t scheduler_id)
 395 {
 396         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 397         struct scheduler_ctx *sched_ctx;
 398
 399         if (!dev) {
 400                 CS_LOG_ERR("Operation not supported");
 401                 return -ENOTSUP;
 402         }
 403
 404         if (dev->driver_id != cryptodev_driver_id) {
 405                 CS_LOG_ERR("Operation not supported");
 406                 return -ENOTSUP;
 407         }
 408
 409         sched_ctx = dev->data->dev_private;
 410
 411         return (int)sched_ctx->reordering_enabled;
 412 }
 413
 414 int
 415 rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id,
 416                 struct rte_cryptodev_scheduler *scheduler) {
 417
 418         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 419         struct scheduler_ctx *sched_ctx;
 420
 421         if (!dev) {
 422                 CS_LOG_ERR("Operation not supported");
 423                 return -ENOTSUP;
 424         }
 425
 426         if (dev->driver_id != cryptodev_driver_id) {
 427                 CS_LOG_ERR("Operation not supported");
 428                 return -ENOTSUP;
 429         }
 430
 431         if (dev->data->dev_started) {
 432                 CS_LOG_ERR("Illegal operation");
 433                 return -EBUSY;
 434         }
 435
 436         sched_ctx = dev->data->dev_private;
 437
 438         if (strlen(scheduler->name) > RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
 439                 CS_LOG_ERR("Invalid name %s, should be less than "
 440                                 "%u bytes.\n", scheduler->name,
 441                                 RTE_CRYPTODEV_NAME_MAX_LEN);
 442                 return -EINVAL;
 443         }
 444         snprintf(sched_ctx->name, sizeof(sched_ctx->name), "%s",
 445                         scheduler->name);
 446
 447         if (strlen(scheduler->description) >
 448                         RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1) {
 449                 CS_LOG_ERR("Invalid description %s, should be less than "
 450                                 "%u bytes.\n", scheduler->description,
 451                                 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1);
 452                 return -EINVAL;
 453         }
 454         snprintf(sched_ctx->description, sizeof(sched_ctx->description), "%s",
 455                         scheduler->description);
 456
 457         /* load scheduler instance operations functions */
 458         sched_ctx->ops.config_queue_pair = scheduler->ops->config_queue_pair;
 459         sched_ctx->ops.create_private_ctx = scheduler->ops->create_private_ctx;
 460         sched_ctx->ops.scheduler_start = scheduler->ops->scheduler_start;
 461         sched_ctx->ops.scheduler_stop = scheduler->ops->scheduler_stop;
 462         sched_ctx->ops.slave_attach = scheduler->ops->slave_attach;
 463         sched_ctx->ops.slave_detach = scheduler->ops->slave_detach;
 464         sched_ctx->ops.option_set = scheduler->ops->option_set;
 465         sched_ctx->ops.option_get = scheduler->ops->option_get;
 466
 467         if (sched_ctx->private_ctx) {
 468                 rte_free(sched_ctx->private_ctx);
 469                 sched_ctx->private_ctx = NULL;
 470         }
 471
 472         if (sched_ctx->ops.create_private_ctx) {
 473                 int ret = (*sched_ctx->ops.create_private_ctx)(dev);
 474
 475                 if (ret < 0) {
 476                         CS_LOG_ERR("Unable to create scheduler private "
 477                                         "context");
 478                         return ret;
 479                 }
 480         }
 481
 482         sched_ctx->mode = scheduler->mode;
 483
 484         return 0;
 485 }
 486
 487 int
 488 rte_cryptodev_scheduler_slaves_get(uint8_t scheduler_id, uint8_t *slaves)
 489 {
 490         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 491         struct scheduler_ctx *sched_ctx;
 492         uint32_t nb_slaves = 0;
 493
 494         if (!dev) {
 495                 CS_LOG_ERR("Operation not supported");
 496                 return -ENOTSUP;
 497         }
 498
 499         if (dev->driver_id != cryptodev_driver_id) {
 500                 CS_LOG_ERR("Operation not supported");
 501                 return -ENOTSUP;
 502         }
 503
 504         sched_ctx = dev->data->dev_private;
 505
 506         nb_slaves = sched_ctx->nb_slaves;
 507
 508         if (slaves && nb_slaves) {
 509                 uint32_t i;
 510
 511                 for (i = 0; i < nb_slaves; i++)
 512                         slaves[i] = sched_ctx->slaves[i].dev_id;
 513         }
 514
 515         return (int)nb_slaves;
 516 }
 517
 518 int
 519 rte_cryptodev_scheduler_option_set(uint8_t scheduler_id,
 520                 enum rte_cryptodev_schedule_option_type option_type,
 521                 void *option)
 522 {
 523         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 524         struct scheduler_ctx *sched_ctx;
 525
 526         if (option_type == CDEV_SCHED_OPTION_NOT_SET ||
 527                         option_type >= CDEV_SCHED_OPTION_COUNT) {
 528                 CS_LOG_ERR("Invalid option parameter");
 529                 return -EINVAL;
 530         }
 531
 532         if (!option) {
 533                 CS_LOG_ERR("Invalid option parameter");
 534                 return -EINVAL;
 535         }
 536
 537         if (dev->data->dev_started) {
 538                 CS_LOG_ERR("Illegal operation");
 539                 return -EBUSY;
 540         }
 541
 542         sched_ctx = dev->data->dev_private;
 543
 544         RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_set, -ENOTSUP);
 545
 546         return (*sched_ctx->ops.option_set)(dev, option_type, option);
 547 }
 548
 549 int
 550 rte_cryptodev_scheduler_option_get(uint8_t scheduler_id,
 551                 enum rte_cryptodev_schedule_option_type option_type,
 552                 void *option)
 553 {
 554         struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
 555         struct scheduler_ctx *sched_ctx;
 556
 557         if (!dev) {
 558                 CS_LOG_ERR("Operation not supported");
 559                 return -ENOTSUP;
 560         }
 561
 562         if (!option) {
 563                 CS_LOG_ERR("Invalid option parameter");
 564                 return -EINVAL;
 565         }
 566
 567         if (dev->driver_id != cryptodev_driver_id) {
 568                 CS_LOG_ERR("Operation not supported");
 569                 return -ENOTSUP;
 570         }
 571
 572         sched_ctx = dev->data->dev_private;
 573
 574         RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_get, -ENOTSUP);
 575
 576         return (*sched_ctx->ops.option_get)(dev, option_type, option);
 577 }