1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
5 #include <cryptodev_pmd.h>
6 #include <rte_malloc.h>
8 #include "rte_cryptodev_scheduler_operations.h"
9 #include "scheduler_pmd_private.h"
11 struct rr_scheduler_qp_ctx {
12 struct scheduler_worker workers[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS];
15 uint32_t last_enq_worker_idx;
16 uint32_t last_deq_worker_idx;
20 schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
22 struct rr_scheduler_qp_ctx *rr_qp_ctx =
23 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx;
24 uint32_t worker_idx = rr_qp_ctx->last_enq_worker_idx;
25 struct scheduler_worker *worker = &rr_qp_ctx->workers[worker_idx];
26 uint16_t i, processed_ops;
28 if (unlikely(nb_ops == 0))
31 for (i = 0; i < nb_ops && i < 4; i++)
32 rte_prefetch0(ops[i]->sym->session);
34 processed_ops = rte_cryptodev_enqueue_burst(worker->dev_id,
35 worker->qp_id, ops, nb_ops);
37 worker->nb_inflight_cops += processed_ops;
39 rr_qp_ctx->last_enq_worker_idx += 1;
40 rr_qp_ctx->last_enq_worker_idx %= rr_qp_ctx->nb_workers;
46 schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops,
49 struct rte_ring *order_ring =
50 ((struct scheduler_qp_ctx *)qp)->order_ring;
51 uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring,
53 uint16_t nb_ops_enqd = schedule_enqueue(qp, ops,
56 scheduler_order_insert(order_ring, ops, nb_ops_enqd);
63 schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
65 struct rr_scheduler_qp_ctx *rr_qp_ctx =
66 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx;
67 struct scheduler_worker *worker;
68 uint32_t last_worker_idx = rr_qp_ctx->last_deq_worker_idx;
71 if (unlikely(rr_qp_ctx->workers[last_worker_idx].nb_inflight_cops
76 if (unlikely(last_worker_idx >= rr_qp_ctx->nb_workers))
78 /* looped back, means no inflight cops in the queue */
79 if (last_worker_idx == rr_qp_ctx->last_deq_worker_idx)
81 } while (rr_qp_ctx->workers[last_worker_idx].nb_inflight_cops
85 worker = &rr_qp_ctx->workers[last_worker_idx];
87 nb_deq_ops = rte_cryptodev_dequeue_burst(worker->dev_id,
88 worker->qp_id, ops, nb_ops);
91 last_worker_idx %= rr_qp_ctx->nb_workers;
93 rr_qp_ctx->last_deq_worker_idx = last_worker_idx;
95 worker->nb_inflight_cops -= nb_deq_ops;
101 schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops,
104 struct rte_ring *order_ring =
105 ((struct scheduler_qp_ctx *)qp)->order_ring;
107 schedule_dequeue(qp, ops, nb_ops);
109 return scheduler_order_drain(order_ring, ops, nb_ops);
113 worker_attach(__rte_unused struct rte_cryptodev *dev,
114 __rte_unused uint8_t worker_id)
120 worker_detach(__rte_unused struct rte_cryptodev *dev,
121 __rte_unused uint8_t worker_id)
127 scheduler_start(struct rte_cryptodev *dev)
129 struct scheduler_ctx *sched_ctx = dev->data->dev_private;
132 if (sched_ctx->reordering_enabled) {
133 dev->enqueue_burst = &schedule_enqueue_ordering;
134 dev->dequeue_burst = &schedule_dequeue_ordering;
136 dev->enqueue_burst = &schedule_enqueue;
137 dev->dequeue_burst = &schedule_dequeue;
140 for (i = 0; i < dev->data->nb_queue_pairs; i++) {
141 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[i];
142 struct rr_scheduler_qp_ctx *rr_qp_ctx =
143 qp_ctx->private_qp_ctx;
146 memset(rr_qp_ctx->workers, 0,
147 RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKERS *
148 sizeof(struct scheduler_worker));
149 for (j = 0; j < sched_ctx->nb_workers; j++) {
150 rr_qp_ctx->workers[j].dev_id =
151 sched_ctx->workers[j].dev_id;
152 rr_qp_ctx->workers[j].qp_id = i;
155 rr_qp_ctx->nb_workers = sched_ctx->nb_workers;
157 rr_qp_ctx->last_enq_worker_idx = 0;
158 rr_qp_ctx->last_deq_worker_idx = 0;
165 scheduler_stop(__rte_unused struct rte_cryptodev *dev)
171 scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id)
173 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
174 struct rr_scheduler_qp_ctx *rr_qp_ctx;
176 rr_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*rr_qp_ctx), 0,
179 CR_SCHED_LOG(ERR, "failed allocate memory for private queue pair");
183 qp_ctx->private_qp_ctx = (void *)rr_qp_ctx;
189 scheduler_create_private_ctx(__rte_unused struct rte_cryptodev *dev)
194 static struct rte_cryptodev_scheduler_ops scheduler_rr_ops = {
200 scheduler_create_private_ctx,
201 NULL, /* option_set */
202 NULL /* option_get */
205 static struct rte_cryptodev_scheduler scheduler = {
206 .name = "roundrobin-scheduler",
207 .description = "scheduler which will round robin burst across "
208 "worker crypto devices",
209 .mode = CDEV_SCHED_MODE_ROUNDROBIN,
210 .ops = &scheduler_rr_ops
213 struct rte_cryptodev_scheduler *crypto_scheduler_roundrobin = &scheduler;