drivers/crypto/scheduler/scheduler_failover.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <cryptodev_pmd.h>
   6 #include <rte_malloc.h>
   7
   8 #include "rte_cryptodev_scheduler_operations.h"
   9 #include "scheduler_pmd_private.h"
  10
  11 #define PRIMARY_WORKER_IDX      0
  12 #define SECONDARY_WORKER_IDX    1
  13 #define NB_FAILOVER_WORKERS     2
  14 #define WORKER_SWITCH_MASK      (0x01)
  15
  16 struct fo_scheduler_qp_ctx {
  17         struct scheduler_worker primary_worker;
  18         struct scheduler_worker secondary_worker;
  19
  20         uint8_t deq_idx;
  21 };
  22
  23 static __rte_always_inline uint16_t
  24 failover_worker_enqueue(struct scheduler_worker *worker,
  25                 struct rte_crypto_op **ops, uint16_t nb_ops)
  26 {
  27         uint16_t i, processed_ops;
  28
  29         for (i = 0; i < nb_ops && i < 4; i++)
  30                 rte_prefetch0(ops[i]->sym->session);
  31
  32         processed_ops = rte_cryptodev_enqueue_burst(worker->dev_id,
  33                         worker->qp_id, ops, nb_ops);
  34         worker->nb_inflight_cops += processed_ops;
  35
  36         return processed_ops;
  37 }
  38
  39 static uint16_t
  40 schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
  41 {
  42         struct fo_scheduler_qp_ctx *qp_ctx =
  43                         ((struct scheduler_qp_ctx *)qp)->private_qp_ctx;
  44         uint16_t enqueued_ops;
  45
  46         if (unlikely(nb_ops == 0))
  47                 return 0;
  48
  49         enqueued_ops = failover_worker_enqueue(&qp_ctx->primary_worker,
  50                         ops, nb_ops);
  51
  52         if (enqueued_ops < nb_ops)
  53                 enqueued_ops += failover_worker_enqueue(
  54                                 &qp_ctx->secondary_worker,
  55                                 &ops[enqueued_ops],
  56                                 nb_ops - enqueued_ops);
  57
  58         return enqueued_ops;
  59 }
  60
  61
  62 static uint16_t
  63 schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops,
  64                 uint16_t nb_ops)
  65 {
  66         struct rte_ring *order_ring =
  67                         ((struct scheduler_qp_ctx *)qp)->order_ring;
  68         uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring,
  69                         nb_ops);
  70         uint16_t nb_ops_enqd = schedule_enqueue(qp, ops,
  71                         nb_ops_to_enq);
  72
  73         scheduler_order_insert(order_ring, ops, nb_ops_enqd);
  74
  75         return nb_ops_enqd;
  76 }
  77
  78 static uint16_t
  79 schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
  80 {
  81         struct fo_scheduler_qp_ctx *qp_ctx =
  82                         ((struct scheduler_qp_ctx *)qp)->private_qp_ctx;
  83         struct scheduler_worker *workers[NB_FAILOVER_WORKERS] = {
  84                         &qp_ctx->primary_worker, &qp_ctx->secondary_worker};
  85         struct scheduler_worker *worker = workers[qp_ctx->deq_idx];
  86         uint16_t nb_deq_ops = 0, nb_deq_ops2 = 0;
  87
  88         if (worker->nb_inflight_cops) {
  89                 nb_deq_ops = rte_cryptodev_dequeue_burst(worker->dev_id,
  90                         worker->qp_id, ops, nb_ops);
  91                 worker->nb_inflight_cops -= nb_deq_ops;
  92         }
  93
  94         qp_ctx->deq_idx = (~qp_ctx->deq_idx) & WORKER_SWITCH_MASK;
  95
  96         if (nb_deq_ops == nb_ops)
  97                 return nb_deq_ops;
  98
  99         worker = workers[qp_ctx->deq_idx];
 100
 101         if (worker->nb_inflight_cops) {
 102                 nb_deq_ops2 = rte_cryptodev_dequeue_burst(worker->dev_id,
 103                         worker->qp_id, &ops[nb_deq_ops], nb_ops - nb_deq_ops);
 104                 worker->nb_inflight_cops -= nb_deq_ops2;
 105         }
 106
 107         return nb_deq_ops + nb_deq_ops2;
 108 }
 109
 110 static uint16_t
 111 schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops,
 112                 uint16_t nb_ops)
 113 {
 114         struct rte_ring *order_ring =
 115                         ((struct scheduler_qp_ctx *)qp)->order_ring;
 116
 117         schedule_dequeue(qp, ops, nb_ops);
 118
 119         return scheduler_order_drain(order_ring, ops, nb_ops);
 120 }
 121
 122 static int
 123 worker_attach(__rte_unused struct rte_cryptodev *dev,
 124                 __rte_unused uint8_t worker_id)
 125 {
 126         return 0;
 127 }
 128
 129 static int
 130 worker_detach(__rte_unused struct rte_cryptodev *dev,
 131                 __rte_unused uint8_t worker_id)
 132 {
 133         return 0;
 134 }
 135
 136 static int
 137 scheduler_start(struct rte_cryptodev *dev)
 138 {
 139         struct scheduler_ctx *sched_ctx = dev->data->dev_private;
 140         uint16_t i;
 141
 142         if (sched_ctx->nb_workers < 2) {
 143                 CR_SCHED_LOG(ERR, "Number of workers shall no less than 2");
 144                 return -ENOMEM;
 145         }
 146
 147         if (sched_ctx->reordering_enabled) {
 148                 dev->enqueue_burst = schedule_enqueue_ordering;
 149                 dev->dequeue_burst = schedule_dequeue_ordering;
 150         } else {
 151                 dev->enqueue_burst = schedule_enqueue;
 152                 dev->dequeue_burst = schedule_dequeue;
 153         }
 154
 155         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 156                 struct fo_scheduler_qp_ctx *qp_ctx =
 157                         ((struct scheduler_qp_ctx *)
 158                                 dev->data->queue_pairs[i])->private_qp_ctx;
 159
 160                 sched_ctx->workers[PRIMARY_WORKER_IDX].qp_id = i;
 161                 sched_ctx->workers[SECONDARY_WORKER_IDX].qp_id = i;
 162
 163                 rte_memcpy(&qp_ctx->primary_worker,
 164                                 &sched_ctx->workers[PRIMARY_WORKER_IDX],
 165                                 sizeof(struct scheduler_worker));
 166                 rte_memcpy(&qp_ctx->secondary_worker,
 167                                 &sched_ctx->workers[SECONDARY_WORKER_IDX],
 168                                 sizeof(struct scheduler_worker));
 169         }
 170
 171         return 0;
 172 }
 173
 174 static int
 175 scheduler_stop(__rte_unused struct rte_cryptodev *dev)
 176 {
 177         return 0;
 178 }
 179
 180 static int
 181 scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id)
 182 {
 183         struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
 184         struct fo_scheduler_qp_ctx *fo_qp_ctx;
 185
 186         fo_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*fo_qp_ctx), 0,
 187                         rte_socket_id());
 188         if (!fo_qp_ctx) {
 189                 CR_SCHED_LOG(ERR, "failed allocate memory for private queue pair");
 190                 return -ENOMEM;
 191         }
 192
 193         qp_ctx->private_qp_ctx = (void *)fo_qp_ctx;
 194
 195         return 0;
 196 }
 197
 198 static int
 199 scheduler_create_private_ctx(__rte_unused struct rte_cryptodev *dev)
 200 {
 201         return 0;
 202 }
 203
 204 static struct rte_cryptodev_scheduler_ops scheduler_fo_ops = {
 205         worker_attach,
 206         worker_detach,
 207         scheduler_start,
 208         scheduler_stop,
 209         scheduler_config_qp,
 210         scheduler_create_private_ctx,
 211         NULL,   /* option_set */
 212         NULL    /*option_get */
 213 };
 214
 215 static struct rte_cryptodev_scheduler fo_scheduler = {
 216                 .name = "failover-scheduler",
 217                 .description = "scheduler which enqueues to the primary worker, "
 218                                 "and only then enqueues to the secondary worker "
 219                                 "upon failing on enqueuing to primary",
 220                 .mode = CDEV_SCHED_MODE_FAILOVER,
 221                 .ops = &scheduler_fo_ops
 222 };
 223
 224 struct rte_cryptodev_scheduler *crypto_scheduler_failover = &fo_scheduler;