1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
5 #include <cryptodev_pmd.h>
6 #include <rte_malloc.h>
8 #include "rte_cryptodev_scheduler_operations.h"
9 #include "scheduler_pmd_private.h"
11 #define PRIMARY_WORKER_IDX 0
12 #define SECONDARY_WORKER_IDX 1
13 #define NB_FAILOVER_WORKERS 2
14 #define WORKER_SWITCH_MASK (0x01)
16 struct fo_scheduler_qp_ctx {
17 struct scheduler_worker primary_worker;
18 struct scheduler_worker secondary_worker;
23 static __rte_always_inline uint16_t
24 failover_worker_enqueue(struct scheduler_worker *worker,
25 struct rte_crypto_op **ops, uint16_t nb_ops)
27 uint16_t i, processed_ops;
29 for (i = 0; i < nb_ops && i < 4; i++)
30 rte_prefetch0(ops[i]->sym->session);
32 processed_ops = rte_cryptodev_enqueue_burst(worker->dev_id,
33 worker->qp_id, ops, nb_ops);
34 worker->nb_inflight_cops += processed_ops;
40 schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
42 struct fo_scheduler_qp_ctx *qp_ctx =
43 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx;
44 uint16_t enqueued_ops;
46 if (unlikely(nb_ops == 0))
49 enqueued_ops = failover_worker_enqueue(&qp_ctx->primary_worker,
52 if (enqueued_ops < nb_ops)
53 enqueued_ops += failover_worker_enqueue(
54 &qp_ctx->secondary_worker,
56 nb_ops - enqueued_ops);
63 schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops,
66 struct rte_ring *order_ring =
67 ((struct scheduler_qp_ctx *)qp)->order_ring;
68 uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring,
70 uint16_t nb_ops_enqd = schedule_enqueue(qp, ops,
73 scheduler_order_insert(order_ring, ops, nb_ops_enqd);
79 schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
81 struct fo_scheduler_qp_ctx *qp_ctx =
82 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx;
83 struct scheduler_worker *workers[NB_FAILOVER_WORKERS] = {
84 &qp_ctx->primary_worker, &qp_ctx->secondary_worker};
85 struct scheduler_worker *worker = workers[qp_ctx->deq_idx];
86 uint16_t nb_deq_ops = 0, nb_deq_ops2 = 0;
88 if (worker->nb_inflight_cops) {
89 nb_deq_ops = rte_cryptodev_dequeue_burst(worker->dev_id,
90 worker->qp_id, ops, nb_ops);
91 worker->nb_inflight_cops -= nb_deq_ops;
94 qp_ctx->deq_idx = (~qp_ctx->deq_idx) & WORKER_SWITCH_MASK;
96 if (nb_deq_ops == nb_ops)
99 worker = workers[qp_ctx->deq_idx];
101 if (worker->nb_inflight_cops) {
102 nb_deq_ops2 = rte_cryptodev_dequeue_burst(worker->dev_id,
103 worker->qp_id, &ops[nb_deq_ops], nb_ops - nb_deq_ops);
104 worker->nb_inflight_cops -= nb_deq_ops2;
107 return nb_deq_ops + nb_deq_ops2;
111 schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops,
114 struct rte_ring *order_ring =
115 ((struct scheduler_qp_ctx *)qp)->order_ring;
117 schedule_dequeue(qp, ops, nb_ops);
119 return scheduler_order_drain(order_ring, ops, nb_ops);
123 worker_attach(__rte_unused struct rte_cryptodev *dev,
124 __rte_unused uint8_t worker_id)
130 worker_detach(__rte_unused struct rte_cryptodev *dev,
131 __rte_unused uint8_t worker_id)
137 scheduler_start(struct rte_cryptodev *dev)
139 struct scheduler_ctx *sched_ctx = dev->data->dev_private;
142 if (sched_ctx->nb_workers < 2) {
143 CR_SCHED_LOG(ERR, "Number of workers shall no less than 2");
147 if (sched_ctx->reordering_enabled) {
148 dev->enqueue_burst = schedule_enqueue_ordering;
149 dev->dequeue_burst = schedule_dequeue_ordering;
151 dev->enqueue_burst = schedule_enqueue;
152 dev->dequeue_burst = schedule_dequeue;
155 for (i = 0; i < dev->data->nb_queue_pairs; i++) {
156 struct fo_scheduler_qp_ctx *qp_ctx =
157 ((struct scheduler_qp_ctx *)
158 dev->data->queue_pairs[i])->private_qp_ctx;
160 rte_memcpy(&qp_ctx->primary_worker,
161 &sched_ctx->workers[PRIMARY_WORKER_IDX],
162 sizeof(struct scheduler_worker));
163 rte_memcpy(&qp_ctx->secondary_worker,
164 &sched_ctx->workers[SECONDARY_WORKER_IDX],
165 sizeof(struct scheduler_worker));
172 scheduler_stop(__rte_unused struct rte_cryptodev *dev)
178 scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id)
180 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
181 struct fo_scheduler_qp_ctx *fo_qp_ctx;
183 fo_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*fo_qp_ctx), 0,
186 CR_SCHED_LOG(ERR, "failed allocate memory for private queue pair");
190 qp_ctx->private_qp_ctx = (void *)fo_qp_ctx;
196 scheduler_create_private_ctx(__rte_unused struct rte_cryptodev *dev)
201 static struct rte_cryptodev_scheduler_ops scheduler_fo_ops = {
207 scheduler_create_private_ctx,
208 NULL, /* option_set */
212 static struct rte_cryptodev_scheduler fo_scheduler = {
213 .name = "failover-scheduler",
214 .description = "scheduler which enqueues to the primary worker, "
215 "and only then enqueues to the secondary worker "
216 "upon failing on enqueuing to primary",
217 .mode = CDEV_SCHED_MODE_FAILOVER,
218 .ops = &scheduler_fo_ops
221 struct rte_cryptodev_scheduler *crypto_scheduler_failover = &fo_scheduler;