drivers/crypto/scheduler/scheduler_pmd_ops.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32 #include <string.h>
  33
  34 #include <rte_config.h>
  35 #include <rte_common.h>
  36 #include <rte_malloc.h>
  37 #include <rte_dev.h>
  38 #include <rte_cryptodev.h>
  39 #include <rte_cryptodev_pmd.h>
  40 #include <rte_cryptodev_vdev.h>
  41 #include <rte_reorder.h>
  42
  43 #include "scheduler_pmd_private.h"
  44
  45 /** attaching the slaves predefined by scheduler's EAL options */
  46 static int
  47 scheduler_attach_init_slave(struct rte_cryptodev *dev)
  48 {
  49         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
  50         uint8_t scheduler_id = dev->data->dev_id;
  51         int i;
  52
  53         for (i = sched_ctx->nb_init_slaves - 1; i >= 0; i--) {
  54                 const char *dev_name = sched_ctx->init_slave_names[i];
  55                 struct rte_cryptodev *slave_dev =
  56                                 rte_cryptodev_pmd_get_named_dev(dev_name);
  57                 int status;
  58
  59                 if (!slave_dev) {
  60                         CS_LOG_ERR("Failed to locate slave dev %s",
  61                                         dev_name);
  62                         return -EINVAL;
  63                 }
  64
  65                 status = rte_cryptodev_scheduler_slave_attach(
  66                                 scheduler_id, slave_dev->data->dev_id);
  67
  68                 if (status < 0) {
  69                         CS_LOG_ERR("Failed to attach slave cryptodev %u",
  70                                         slave_dev->data->dev_id);
  71                         return status;
  72                 }
  73
  74                 CS_LOG_INFO("Scheduler %s attached slave %s\n",
  75                                 dev->data->name,
  76                                 sched_ctx->init_slave_names[i]);
  77
  78                 rte_free(sched_ctx->init_slave_names[i]);
  79
  80                 sched_ctx->nb_init_slaves -= 1;
  81         }
  82
  83         return 0;
  84 }
  85 /** Configure device */
  86 static int
  87 scheduler_pmd_config(struct rte_cryptodev *dev,
  88                 struct rte_cryptodev_config *config __rte_unused)
  89 {
  90         int ret;
  91
  92         /* although scheduler_attach_init_slave presents multiple times,
  93          * there will be only 1 meaningful execution.
  94          */
  95         ret = scheduler_attach_init_slave(dev);
  96         if (ret < 0)
  97                 return ret;
  98
  99         return ret;
 100 }
 101
 102 static int
 103 update_order_ring(struct rte_cryptodev *dev, uint16_t qp_id)
 104 {
 105         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 106         struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
 107
 108         if (sched_ctx->reordering_enabled) {
 109                 char order_ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];
 110                 uint32_t buff_size = rte_align32pow2(
 111                         sched_ctx->nb_slaves * PER_SLAVE_BUFF_SIZE);
 112
 113                 if (qp_ctx->order_ring) {
 114                         rte_ring_free(qp_ctx->order_ring);
 115                         qp_ctx->order_ring = NULL;
 116                 }
 117
 118                 if (!buff_size)
 119                         return 0;
 120
 121                 if (snprintf(order_ring_name, RTE_CRYPTODEV_NAME_MAX_LEN,
 122                         "%s_rb_%u_%u", RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD),
 123                         dev->data->dev_id, qp_id) < 0) {
 124                         CS_LOG_ERR("failed to create unique reorder buffer "
 125                                         "name");
 126                         return -ENOMEM;
 127                 }
 128
 129                 qp_ctx->order_ring = rte_ring_create(order_ring_name,
 130                                 buff_size, rte_socket_id(),
 131                                 RING_F_SP_ENQ | RING_F_SC_DEQ);
 132                 if (!qp_ctx->order_ring) {
 133                         CS_LOG_ERR("failed to create order ring");
 134                         return -ENOMEM;
 135                 }
 136         } else {
 137                 if (qp_ctx->order_ring) {
 138                         rte_ring_free(qp_ctx->order_ring);
 139                         qp_ctx->order_ring = NULL;
 140                 }
 141         }
 142
 143         return 0;
 144 }
 145
 146 /** Start device */
 147 static int
 148 scheduler_pmd_start(struct rte_cryptodev *dev)
 149 {
 150         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 151         uint32_t i;
 152         int ret;
 153
 154         if (dev->data->dev_started)
 155                 return 0;
 156
 157         /* although scheduler_attach_init_slave presents multiple times,
 158          * there will be only 1 meaningful execution.
 159          */
 160         ret = scheduler_attach_init_slave(dev);
 161         if (ret < 0)
 162                 return ret;
 163
 164         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 165                 ret = update_order_ring(dev, i);
 166                 if (ret < 0) {
 167                         CS_LOG_ERR("Failed to update reorder buffer");
 168                         return ret;
 169                 }
 170         }
 171
 172         if (sched_ctx->mode == CDEV_SCHED_MODE_NOT_SET) {
 173                 CS_LOG_ERR("Scheduler mode is not set");
 174                 return -1;
 175         }
 176
 177         if (!sched_ctx->nb_slaves) {
 178                 CS_LOG_ERR("No slave in the scheduler");
 179                 return -1;
 180         }
 181
 182         RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.slave_attach, -ENOTSUP);
 183
 184         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 185                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 186
 187                 if ((*sched_ctx->ops.slave_attach)(dev, slave_dev_id) < 0) {
 188                         CS_LOG_ERR("Failed to attach slave");
 189                         return -ENOTSUP;
 190                 }
 191         }
 192
 193         RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.scheduler_start, -ENOTSUP);
 194
 195         if ((*sched_ctx->ops.scheduler_start)(dev) < 0) {
 196                 CS_LOG_ERR("Scheduler start failed");
 197                 return -1;
 198         }
 199
 200         /* start all slaves */
 201         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 202                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 203                 struct rte_cryptodev *slave_dev =
 204                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 205
 206                 ret = (*slave_dev->dev_ops->dev_start)(slave_dev);
 207                 if (ret < 0) {
 208                         CS_LOG_ERR("Failed to start slave dev %u",
 209                                         slave_dev_id);
 210                         return ret;
 211                 }
 212         }
 213
 214         return 0;
 215 }
 216
 217 /** Stop device */
 218 static void
 219 scheduler_pmd_stop(struct rte_cryptodev *dev)
 220 {
 221         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 222         uint32_t i;
 223
 224         if (!dev->data->dev_started)
 225                 return;
 226
 227         /* stop all slaves first */
 228         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 229                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 230                 struct rte_cryptodev *slave_dev =
 231                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 232
 233                 (*slave_dev->dev_ops->dev_stop)(slave_dev);
 234         }
 235
 236         if (*sched_ctx->ops.scheduler_stop)
 237                 (*sched_ctx->ops.scheduler_stop)(dev);
 238
 239         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 240                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 241
 242                 if (*sched_ctx->ops.slave_detach)
 243                         (*sched_ctx->ops.slave_detach)(dev, slave_dev_id);
 244         }
 245 }
 246
 247 /** Close device */
 248 static int
 249 scheduler_pmd_close(struct rte_cryptodev *dev)
 250 {
 251         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 252         uint32_t i;
 253         int ret;
 254
 255         /* the dev should be stopped before being closed */
 256         if (dev->data->dev_started)
 257                 return -EBUSY;
 258
 259         /* close all slaves first */
 260         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 261                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 262                 struct rte_cryptodev *slave_dev =
 263                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 264
 265                 ret = (*slave_dev->dev_ops->dev_close)(slave_dev);
 266                 if (ret < 0)
 267                         return ret;
 268         }
 269
 270         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 271                 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i];
 272
 273                 if (qp_ctx->order_ring) {
 274                         rte_ring_free(qp_ctx->order_ring);
 275                         qp_ctx->order_ring = NULL;
 276                 }
 277
 278                 if (qp_ctx->private_qp_ctx) {
 279                         rte_free(qp_ctx->private_qp_ctx);
 280                         qp_ctx->private_qp_ctx = NULL;
 281                 }
 282         }
 283
 284         if (sched_ctx->private_ctx)
 285                 rte_free(sched_ctx->private_ctx);
 286
 287         if (sched_ctx->capabilities)
 288                 rte_free(sched_ctx->capabilities);
 289
 290         return 0;
 291 }
 292
 293 /** Get device statistics */
 294 static void
 295 scheduler_pmd_stats_get(struct rte_cryptodev *dev,
 296         struct rte_cryptodev_stats *stats)
 297 {
 298         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 299         uint32_t i;
 300
 301         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 302                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 303                 struct rte_cryptodev *slave_dev =
 304                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 305                 struct rte_cryptodev_stats slave_stats = {0};
 306
 307                 (*slave_dev->dev_ops->stats_get)(slave_dev, &slave_stats);
 308
 309                 stats->enqueued_count += slave_stats.enqueued_count;
 310                 stats->dequeued_count += slave_stats.dequeued_count;
 311
 312                 stats->enqueue_err_count += slave_stats.enqueue_err_count;
 313                 stats->dequeue_err_count += slave_stats.dequeue_err_count;
 314         }
 315 }
 316
 317 /** Reset device statistics */
 318 static void
 319 scheduler_pmd_stats_reset(struct rte_cryptodev *dev)
 320 {
 321         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 322         uint32_t i;
 323
 324         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 325                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 326                 struct rte_cryptodev *slave_dev =
 327                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 328
 329                 (*slave_dev->dev_ops->stats_reset)(slave_dev);
 330         }
 331 }
 332
 333 /** Get device info */
 334 static void
 335 scheduler_pmd_info_get(struct rte_cryptodev *dev,
 336                 struct rte_cryptodev_info *dev_info)
 337 {
 338         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 339         uint32_t max_nb_sessions = sched_ctx->nb_slaves ?
 340                         UINT32_MAX : RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS;
 341         uint32_t i;
 342
 343         if (!dev_info)
 344                 return;
 345
 346         /* although scheduler_attach_init_slave presents multiple times,
 347          * there will be only 1 meaningful execution.
 348          */
 349         scheduler_attach_init_slave(dev);
 350
 351         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 352                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 353                 struct rte_cryptodev_info slave_info;
 354
 355                 rte_cryptodev_info_get(slave_dev_id, &slave_info);
 356                 max_nb_sessions = slave_info.sym.max_nb_sessions <
 357                                 max_nb_sessions ?
 358                                 slave_info.sym.max_nb_sessions :
 359                                 max_nb_sessions;
 360         }
 361
 362         dev_info->driver_id = dev->driver_id;
 363         dev_info->feature_flags = dev->feature_flags;
 364         dev_info->capabilities = sched_ctx->capabilities;
 365         dev_info->max_nb_queue_pairs = sched_ctx->max_nb_queue_pairs;
 366         dev_info->sym.max_nb_sessions = max_nb_sessions;
 367 }
 368
 369 /** Release queue pair */
 370 static int
 371 scheduler_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
 372 {
 373         struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
 374
 375         if (!qp_ctx)
 376                 return 0;
 377
 378         if (qp_ctx->order_ring)
 379                 rte_ring_free(qp_ctx->order_ring);
 380         if (qp_ctx->private_qp_ctx)
 381                 rte_free(qp_ctx->private_qp_ctx);
 382
 383         rte_free(qp_ctx);
 384         dev->data->queue_pairs[qp_id] = NULL;
 385
 386         return 0;
 387 }
 388
 389 /** Setup a queue pair */
 390 static int
 391 scheduler_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 392         const struct rte_cryptodev_qp_conf *qp_conf, int socket_id)
 393 {
 394         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 395         struct scheduler_qp_ctx *qp_ctx;
 396         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
 397         uint32_t i;
 398         int ret;
 399
 400         if (snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN,
 401                         "CRYTO_SCHE PMD %u QP %u",
 402                         dev->data->dev_id, qp_id) < 0) {
 403                 CS_LOG_ERR("Failed to create unique queue pair name");
 404                 return -EFAULT;
 405         }
 406
 407         /* Free memory prior to re-allocation if needed. */
 408         if (dev->data->queue_pairs[qp_id] != NULL)
 409                 scheduler_pmd_qp_release(dev, qp_id);
 410
 411         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 412                 uint8_t slave_id = sched_ctx->slaves[i].dev_id;
 413
 414                 ret = rte_cryptodev_queue_pair_setup(slave_id, qp_id,
 415                                 qp_conf, socket_id);
 416                 if (ret < 0)
 417                         return ret;
 418         }
 419
 420         /* Allocate the queue pair data structure. */
 421         qp_ctx = rte_zmalloc_socket(name, sizeof(*qp_ctx), RTE_CACHE_LINE_SIZE,
 422                         socket_id);
 423         if (qp_ctx == NULL)
 424                 return -ENOMEM;
 425
 426         /* The actual available object number = nb_descriptors - 1 */
 427         qp_ctx->max_nb_objs = qp_conf->nb_descriptors - 1;
 428
 429         dev->data->queue_pairs[qp_id] = qp_ctx;
 430
 431         /* although scheduler_attach_init_slave presents multiple times,
 432          * there will be only 1 meaningful execution.
 433          */
 434         ret = scheduler_attach_init_slave(dev);
 435         if (ret < 0) {
 436                 CS_LOG_ERR("Failed to attach slave");
 437                 scheduler_pmd_qp_release(dev, qp_id);
 438                 return ret;
 439         }
 440
 441         if (*sched_ctx->ops.config_queue_pair) {
 442                 if ((*sched_ctx->ops.config_queue_pair)(dev, qp_id) < 0) {
 443                         CS_LOG_ERR("Unable to configure queue pair");
 444                         return -1;
 445                 }
 446         }
 447
 448         return 0;
 449 }
 450
 451 /** Start queue pair */
 452 static int
 453 scheduler_pmd_qp_start(__rte_unused struct rte_cryptodev *dev,
 454                 __rte_unused uint16_t queue_pair_id)
 455 {
 456         return -ENOTSUP;
 457 }
 458
 459 /** Stop queue pair */
 460 static int
 461 scheduler_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev,
 462                 __rte_unused uint16_t queue_pair_id)
 463 {
 464         return -ENOTSUP;
 465 }
 466
 467 /** Return the number of allocated queue pairs */
 468 static uint32_t
 469 scheduler_pmd_qp_count(struct rte_cryptodev *dev)
 470 {
 471         return dev->data->nb_queue_pairs;
 472 }
 473
 474 static uint32_t
 475 scheduler_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
 476 {
 477         return sizeof(struct scheduler_session);
 478 }
 479
 480 static int
 481 config_slave_sess(struct scheduler_ctx *sched_ctx,
 482                 struct rte_crypto_sym_xform *xform,
 483                 struct scheduler_session *sess,
 484                 uint32_t create)
 485 {
 486         uint32_t i;
 487
 488         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 489                 struct scheduler_slave *slave = &sched_ctx->slaves[i];
 490
 491                 if (sess->sessions[i]) {
 492                         if (create)
 493                                 continue;
 494                         /* !create */
 495                         sess->sessions[i] = rte_cryptodev_sym_session_free(
 496                                         slave->dev_id, sess->sessions[i]);
 497                 } else {
 498                         if (!create)
 499                                 continue;
 500                         /* create */
 501                         sess->sessions[i] =
 502                                         rte_cryptodev_sym_session_create(
 503                                                         slave->dev_id, xform);
 504                         if (!sess->sessions[i]) {
 505                                 config_slave_sess(sched_ctx, NULL, sess, 0);
 506                                 return -1;
 507                         }
 508                 }
 509         }
 510
 511         return 0;
 512 }
 513
 514 /** Clear the memory of session so it doesn't leave key material behind */
 515 static void
 516 scheduler_pmd_session_clear(struct rte_cryptodev *dev,
 517         void *sess)
 518 {
 519         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 520
 521         config_slave_sess(sched_ctx, NULL, sess, 0);
 522
 523         memset(sess, 0, sizeof(struct scheduler_session));
 524 }
 525
 526 static void *
 527 scheduler_pmd_session_configure(struct rte_cryptodev *dev,
 528         struct rte_crypto_sym_xform *xform, void *sess)
 529 {
 530         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 531
 532         if (config_slave_sess(sched_ctx, xform, sess, 1) < 0) {
 533                 CS_LOG_ERR("unabled to config sym session");
 534                 return NULL;
 535         }
 536
 537         return sess;
 538 }
 539
 540 struct rte_cryptodev_ops scheduler_pmd_ops = {
 541                 .dev_configure          = scheduler_pmd_config,
 542                 .dev_start              = scheduler_pmd_start,
 543                 .dev_stop               = scheduler_pmd_stop,
 544                 .dev_close              = scheduler_pmd_close,
 545
 546                 .stats_get              = scheduler_pmd_stats_get,
 547                 .stats_reset            = scheduler_pmd_stats_reset,
 548
 549                 .dev_infos_get          = scheduler_pmd_info_get,
 550
 551                 .queue_pair_setup       = scheduler_pmd_qp_setup,
 552                 .queue_pair_release     = scheduler_pmd_qp_release,
 553                 .queue_pair_start       = scheduler_pmd_qp_start,
 554                 .queue_pair_stop        = scheduler_pmd_qp_stop,
 555                 .queue_pair_count       = scheduler_pmd_qp_count,
 556
 557                 .session_get_size       = scheduler_pmd_session_get_size,
 558                 .session_configure      = scheduler_pmd_session_configure,
 559                 .session_clear          = scheduler_pmd_session_clear,
 560 };
 561
 562 struct rte_cryptodev_ops *rte_crypto_scheduler_pmd_ops = &scheduler_pmd_ops;