drivers/crypto/scheduler/scheduler_pmd_ops.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32 #include <string.h>
  33
  34 #include <rte_config.h>
  35 #include <rte_common.h>
  36 #include <rte_malloc.h>
  37 #include <rte_dev.h>
  38 #include <rte_cryptodev.h>
  39 #include <rte_cryptodev_pmd.h>
  40 #include <rte_reorder.h>
  41
  42 #include "scheduler_pmd_private.h"
  43
  44 /** Configure device */
  45 static int
  46 scheduler_pmd_config(struct rte_cryptodev *dev,
  47                 struct rte_cryptodev_config *config)
  48 {
  49         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
  50         uint32_t i;
  51         int ret = 0;
  52
  53         for (i = 0; i < sched_ctx->nb_slaves; i++) {
  54                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
  55                 struct rte_cryptodev *slave_dev =
  56                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
  57
  58                 ret = (*slave_dev->dev_ops->dev_configure)(slave_dev,
  59                                 config);
  60                 if (ret < 0)
  61                         break;
  62         }
  63
  64         return ret;
  65 }
  66
  67 static int
  68 update_order_ring(struct rte_cryptodev *dev, uint16_t qp_id)
  69 {
  70         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
  71         struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
  72
  73         if (sched_ctx->reordering_enabled) {
  74                 char order_ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];
  75                 uint32_t buff_size = rte_align32pow2(
  76                         sched_ctx->nb_slaves * PER_SLAVE_BUFF_SIZE);
  77
  78                 if (qp_ctx->order_ring) {
  79                         rte_ring_free(qp_ctx->order_ring);
  80                         qp_ctx->order_ring = NULL;
  81                 }
  82
  83                 if (!buff_size)
  84                         return 0;
  85
  86                 if (snprintf(order_ring_name, RTE_CRYPTODEV_NAME_MAX_LEN,
  87                         "%s_rb_%u_%u", RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD),
  88                         dev->data->dev_id, qp_id) < 0) {
  89                         CS_LOG_ERR("failed to create unique reorder buffer "
  90                                         "name");
  91                         return -ENOMEM;
  92                 }
  93
  94                 qp_ctx->order_ring = rte_ring_create(order_ring_name,
  95                                 buff_size, rte_socket_id(),
  96                                 RING_F_SP_ENQ | RING_F_SC_DEQ);
  97                 if (!qp_ctx->order_ring) {
  98                         CS_LOG_ERR("failed to create order ring");
  99                         return -ENOMEM;
 100                 }
 101         } else {
 102                 if (qp_ctx->order_ring) {
 103                         rte_ring_free(qp_ctx->order_ring);
 104                         qp_ctx->order_ring = NULL;
 105                 }
 106         }
 107
 108         return 0;
 109 }
 110
 111 /** Start device */
 112 static int
 113 scheduler_pmd_start(struct rte_cryptodev *dev)
 114 {
 115         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 116         uint32_t i;
 117         int ret;
 118
 119         if (dev->data->dev_started)
 120                 return 0;
 121
 122         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 123                 ret = update_order_ring(dev, i);
 124                 if (ret < 0) {
 125                         CS_LOG_ERR("Failed to update reorder buffer");
 126                         return ret;
 127                 }
 128         }
 129
 130         if (sched_ctx->mode == CDEV_SCHED_MODE_NOT_SET) {
 131                 CS_LOG_ERR("Scheduler mode is not set");
 132                 return -1;
 133         }
 134
 135         if (!sched_ctx->nb_slaves) {
 136                 CS_LOG_ERR("No slave in the scheduler");
 137                 return -1;
 138         }
 139
 140         RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.slave_attach, -ENOTSUP);
 141
 142         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 143                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 144
 145                 if ((*sched_ctx->ops.slave_attach)(dev, slave_dev_id) < 0) {
 146                         CS_LOG_ERR("Failed to attach slave");
 147                         return -ENOTSUP;
 148                 }
 149         }
 150
 151         RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.scheduler_start, -ENOTSUP);
 152
 153         if ((*sched_ctx->ops.scheduler_start)(dev) < 0) {
 154                 CS_LOG_ERR("Scheduler start failed");
 155                 return -1;
 156         }
 157
 158         /* start all slaves */
 159         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 160                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 161                 struct rte_cryptodev *slave_dev =
 162                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 163
 164                 ret = (*slave_dev->dev_ops->dev_start)(slave_dev);
 165                 if (ret < 0) {
 166                         CS_LOG_ERR("Failed to start slave dev %u",
 167                                         slave_dev_id);
 168                         return ret;
 169                 }
 170         }
 171
 172         return 0;
 173 }
 174
 175 /** Stop device */
 176 static void
 177 scheduler_pmd_stop(struct rte_cryptodev *dev)
 178 {
 179         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 180         uint32_t i;
 181
 182         if (!dev->data->dev_started)
 183                 return;
 184
 185         /* stop all slaves first */
 186         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 187                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 188                 struct rte_cryptodev *slave_dev =
 189                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 190
 191                 (*slave_dev->dev_ops->dev_stop)(slave_dev);
 192         }
 193
 194         if (*sched_ctx->ops.scheduler_stop)
 195                 (*sched_ctx->ops.scheduler_stop)(dev);
 196
 197         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 198                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 199
 200                 if (*sched_ctx->ops.slave_detach)
 201                         (*sched_ctx->ops.slave_detach)(dev, slave_dev_id);
 202         }
 203 }
 204
 205 /** Close device */
 206 static int
 207 scheduler_pmd_close(struct rte_cryptodev *dev)
 208 {
 209         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 210         uint32_t i;
 211         int ret;
 212
 213         /* the dev should be stopped before being closed */
 214         if (dev->data->dev_started)
 215                 return -EBUSY;
 216
 217         /* close all slaves first */
 218         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 219                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 220                 struct rte_cryptodev *slave_dev =
 221                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 222
 223                 ret = (*slave_dev->dev_ops->dev_close)(slave_dev);
 224                 if (ret < 0)
 225                         return ret;
 226         }
 227
 228         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 229                 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i];
 230
 231                 if (qp_ctx->order_ring) {
 232                         rte_ring_free(qp_ctx->order_ring);
 233                         qp_ctx->order_ring = NULL;
 234                 }
 235
 236                 if (qp_ctx->private_qp_ctx) {
 237                         rte_free(qp_ctx->private_qp_ctx);
 238                         qp_ctx->private_qp_ctx = NULL;
 239                 }
 240         }
 241
 242         if (sched_ctx->private_ctx)
 243                 rte_free(sched_ctx->private_ctx);
 244
 245         if (sched_ctx->capabilities)
 246                 rte_free(sched_ctx->capabilities);
 247
 248         return 0;
 249 }
 250
 251 /** Get device statistics */
 252 static void
 253 scheduler_pmd_stats_get(struct rte_cryptodev *dev,
 254         struct rte_cryptodev_stats *stats)
 255 {
 256         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 257         uint32_t i;
 258
 259         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 260                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 261                 struct rte_cryptodev *slave_dev =
 262                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 263                 struct rte_cryptodev_stats slave_stats = {0};
 264
 265                 (*slave_dev->dev_ops->stats_get)(slave_dev, &slave_stats);
 266
 267                 stats->enqueued_count += slave_stats.enqueued_count;
 268                 stats->dequeued_count += slave_stats.dequeued_count;
 269
 270                 stats->enqueue_err_count += slave_stats.enqueue_err_count;
 271                 stats->dequeue_err_count += slave_stats.dequeue_err_count;
 272         }
 273 }
 274
 275 /** Reset device statistics */
 276 static void
 277 scheduler_pmd_stats_reset(struct rte_cryptodev *dev)
 278 {
 279         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 280         uint32_t i;
 281
 282         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 283                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 284                 struct rte_cryptodev *slave_dev =
 285                                 rte_cryptodev_pmd_get_dev(slave_dev_id);
 286
 287                 (*slave_dev->dev_ops->stats_reset)(slave_dev);
 288         }
 289 }
 290
 291 /** Get device info */
 292 static void
 293 scheduler_pmd_info_get(struct rte_cryptodev *dev,
 294                 struct rte_cryptodev_info *dev_info)
 295 {
 296         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 297         uint32_t max_nb_sessions = sched_ctx->nb_slaves ?
 298                         UINT32_MAX : RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS;
 299         uint32_t i;
 300
 301         if (!dev_info)
 302                 return;
 303
 304         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 305                 uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id;
 306                 struct rte_cryptodev_info slave_info;
 307
 308                 rte_cryptodev_info_get(slave_dev_id, &slave_info);
 309                 max_nb_sessions = slave_info.sym.max_nb_sessions <
 310                                 max_nb_sessions ?
 311                                 slave_info.sym.max_nb_sessions :
 312                                 max_nb_sessions;
 313         }
 314
 315         dev_info->dev_type = dev->dev_type;
 316         dev_info->feature_flags = dev->feature_flags;
 317         dev_info->capabilities = sched_ctx->capabilities;
 318         dev_info->max_nb_queue_pairs = sched_ctx->max_nb_queue_pairs;
 319         dev_info->sym.max_nb_sessions = max_nb_sessions;
 320 }
 321
 322 /** Release queue pair */
 323 static int
 324 scheduler_pmd_qp_release(struct rte_cryptodev *dev, uint16_t qp_id)
 325 {
 326         struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
 327
 328         if (!qp_ctx)
 329                 return 0;
 330
 331         if (qp_ctx->order_ring)
 332                 rte_ring_free(qp_ctx->order_ring);
 333         if (qp_ctx->private_qp_ctx)
 334                 rte_free(qp_ctx->private_qp_ctx);
 335
 336         rte_free(qp_ctx);
 337         dev->data->queue_pairs[qp_id] = NULL;
 338
 339         return 0;
 340 }
 341
 342 /** Setup a queue pair */
 343 static int
 344 scheduler_pmd_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 345         __rte_unused const struct rte_cryptodev_qp_conf *qp_conf, int socket_id)
 346 {
 347         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 348         struct scheduler_qp_ctx *qp_ctx;
 349         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
 350
 351         if (snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN,
 352                         "CRYTO_SCHE PMD %u QP %u",
 353                         dev->data->dev_id, qp_id) < 0) {
 354                 CS_LOG_ERR("Failed to create unique queue pair name");
 355                 return -EFAULT;
 356         }
 357
 358         /* Free memory prior to re-allocation if needed. */
 359         if (dev->data->queue_pairs[qp_id] != NULL)
 360                 scheduler_pmd_qp_release(dev, qp_id);
 361
 362         /* Allocate the queue pair data structure. */
 363         qp_ctx = rte_zmalloc_socket(name, sizeof(*qp_ctx), RTE_CACHE_LINE_SIZE,
 364                         socket_id);
 365         if (qp_ctx == NULL)
 366                 return -ENOMEM;
 367
 368         dev->data->queue_pairs[qp_id] = qp_ctx;
 369
 370         if (*sched_ctx->ops.config_queue_pair) {
 371                 if ((*sched_ctx->ops.config_queue_pair)(dev, qp_id) < 0) {
 372                         CS_LOG_ERR("Unable to configure queue pair");
 373                         return -1;
 374                 }
 375         }
 376
 377         return 0;
 378 }
 379
 380 /** Start queue pair */
 381 static int
 382 scheduler_pmd_qp_start(__rte_unused struct rte_cryptodev *dev,
 383                 __rte_unused uint16_t queue_pair_id)
 384 {
 385         return -ENOTSUP;
 386 }
 387
 388 /** Stop queue pair */
 389 static int
 390 scheduler_pmd_qp_stop(__rte_unused struct rte_cryptodev *dev,
 391                 __rte_unused uint16_t queue_pair_id)
 392 {
 393         return -ENOTSUP;
 394 }
 395
 396 /** Return the number of allocated queue pairs */
 397 static uint32_t
 398 scheduler_pmd_qp_count(struct rte_cryptodev *dev)
 399 {
 400         return dev->data->nb_queue_pairs;
 401 }
 402
 403 static uint32_t
 404 scheduler_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused)
 405 {
 406         return sizeof(struct scheduler_session);
 407 }
 408
 409 static int
 410 config_slave_sess(struct scheduler_ctx *sched_ctx,
 411                 struct rte_crypto_sym_xform *xform,
 412                 struct scheduler_session *sess,
 413                 uint32_t create)
 414 {
 415         uint32_t i;
 416
 417         for (i = 0; i < sched_ctx->nb_slaves; i++) {
 418                 struct scheduler_slave *slave = &sched_ctx->slaves[i];
 419                 struct rte_cryptodev *dev =
 420                                 rte_cryptodev_pmd_get_dev(slave->dev_id);
 421
 422                 if (sess->sessions[i]) {
 423                         if (create)
 424                                 continue;
 425                         /* !create */
 426                         (*dev->dev_ops->session_clear)(dev,
 427                                         (void *)sess->sessions[i]);
 428                         sess->sessions[i] = NULL;
 429                 } else {
 430                         if (!create)
 431                                 continue;
 432                         /* create */
 433                         sess->sessions[i] =
 434                                         rte_cryptodev_sym_session_create(
 435                                                         slave->dev_id, xform);
 436                         if (!sess->sessions[i]) {
 437                                 config_slave_sess(sched_ctx, NULL, sess, 0);
 438                                 return -1;
 439                         }
 440                 }
 441         }
 442
 443         return 0;
 444 }
 445
 446 /** Clear the memory of session so it doesn't leave key material behind */
 447 static void
 448 scheduler_pmd_session_clear(struct rte_cryptodev *dev,
 449         void *sess)
 450 {
 451         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 452
 453         config_slave_sess(sched_ctx, NULL, sess, 0);
 454
 455         memset(sess, 0, sizeof(struct scheduler_session));
 456 }
 457
 458 static void *
 459 scheduler_pmd_session_configure(struct rte_cryptodev *dev,
 460         struct rte_crypto_sym_xform *xform, void *sess)
 461 {
 462         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 463
 464         if (config_slave_sess(sched_ctx, xform, sess, 1) < 0) {
 465                 CS_LOG_ERR("unabled to config sym session");
 466                 return NULL;
 467         }
 468
 469         return sess;
 470 }
 471
 472 struct rte_cryptodev_ops scheduler_pmd_ops = {
 473                 .dev_configure          = scheduler_pmd_config,
 474                 .dev_start              = scheduler_pmd_start,
 475                 .dev_stop               = scheduler_pmd_stop,
 476                 .dev_close              = scheduler_pmd_close,
 477
 478                 .stats_get              = scheduler_pmd_stats_get,
 479                 .stats_reset            = scheduler_pmd_stats_reset,
 480
 481                 .dev_infos_get          = scheduler_pmd_info_get,
 482
 483                 .queue_pair_setup       = scheduler_pmd_qp_setup,
 484                 .queue_pair_release     = scheduler_pmd_qp_release,
 485                 .queue_pair_start       = scheduler_pmd_qp_start,
 486                 .queue_pair_stop        = scheduler_pmd_qp_stop,
 487                 .queue_pair_count       = scheduler_pmd_qp_count,
 488
 489                 .session_get_size       = scheduler_pmd_session_get_size,
 490                 .session_configure      = scheduler_pmd_session_configure,
 491                 .session_clear          = scheduler_pmd_session_clear,
 492 };
 493
 494 struct rte_cryptodev_ops *rte_crypto_scheduler_pmd_ops = &scheduler_pmd_ops;