4 * Copyright(c) 2017 Intel Corporation. All rights reserved.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * * Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in
14 * the documentation and/or other materials provided with the
16 * * Neither the name of Intel Corporation nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 #include <rte_cryptodev.h>
34 #include <rte_malloc.h>
36 #include "rte_cryptodev_scheduler_operations.h"
37 #include "scheduler_pmd_private.h"
39 #define PRIMARY_SLAVE_IDX 0
40 #define SECONDARY_SLAVE_IDX 1
41 #define NB_FAILOVER_SLAVES 2
42 #define SLAVE_SWITCH_MASK (0x01)
44 struct fo_scheduler_qp_ctx {
45 struct scheduler_slave primary_slave;
46 struct scheduler_slave secondary_slave;
51 static __rte_always_inline uint16_t
52 failover_slave_enqueue(struct scheduler_slave *slave,
53 struct rte_crypto_op **ops, uint16_t nb_ops)
55 uint16_t i, processed_ops;
57 for (i = 0; i < nb_ops && i < 4; i++)
58 rte_prefetch0(ops[i]->sym->session);
60 processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id,
61 slave->qp_id, ops, nb_ops);
62 slave->nb_inflight_cops += processed_ops;
68 schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
70 struct fo_scheduler_qp_ctx *qp_ctx =
71 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx;
72 uint16_t enqueued_ops;
74 if (unlikely(nb_ops == 0))
77 enqueued_ops = failover_slave_enqueue(&qp_ctx->primary_slave,
80 if (enqueued_ops < nb_ops)
81 enqueued_ops += failover_slave_enqueue(&qp_ctx->secondary_slave,
83 nb_ops - enqueued_ops);
90 schedule_enqueue_ordering(void *qp, struct rte_crypto_op **ops,
93 struct rte_ring *order_ring =
94 ((struct scheduler_qp_ctx *)qp)->order_ring;
95 uint16_t nb_ops_to_enq = get_max_enqueue_order_count(order_ring,
97 uint16_t nb_ops_enqd = schedule_enqueue(qp, ops,
100 scheduler_order_insert(order_ring, ops, nb_ops_enqd);
106 schedule_dequeue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
108 struct fo_scheduler_qp_ctx *qp_ctx =
109 ((struct scheduler_qp_ctx *)qp)->private_qp_ctx;
110 struct scheduler_slave *slaves[NB_FAILOVER_SLAVES] = {
111 &qp_ctx->primary_slave, &qp_ctx->secondary_slave};
112 struct scheduler_slave *slave = slaves[qp_ctx->deq_idx];
113 uint16_t nb_deq_ops = 0, nb_deq_ops2 = 0;
115 if (slave->nb_inflight_cops) {
116 nb_deq_ops = rte_cryptodev_dequeue_burst(slave->dev_id,
117 slave->qp_id, ops, nb_ops);
118 slave->nb_inflight_cops -= nb_deq_ops;
121 qp_ctx->deq_idx = (~qp_ctx->deq_idx) & SLAVE_SWITCH_MASK;
123 if (nb_deq_ops == nb_ops)
126 slave = slaves[qp_ctx->deq_idx];
128 if (slave->nb_inflight_cops) {
129 nb_deq_ops2 = rte_cryptodev_dequeue_burst(slave->dev_id,
130 slave->qp_id, &ops[nb_deq_ops], nb_ops - nb_deq_ops);
131 slave->nb_inflight_cops -= nb_deq_ops2;
134 return nb_deq_ops + nb_deq_ops2;
138 schedule_dequeue_ordering(void *qp, struct rte_crypto_op **ops,
141 struct rte_ring *order_ring =
142 ((struct scheduler_qp_ctx *)qp)->order_ring;
144 schedule_dequeue(qp, ops, nb_ops);
146 return scheduler_order_drain(order_ring, ops, nb_ops);
150 slave_attach(__rte_unused struct rte_cryptodev *dev,
151 __rte_unused uint8_t slave_id)
157 slave_detach(__rte_unused struct rte_cryptodev *dev,
158 __rte_unused uint8_t slave_id)
164 scheduler_start(struct rte_cryptodev *dev)
166 struct scheduler_ctx *sched_ctx = dev->data->dev_private;
169 if (sched_ctx->nb_slaves < 2) {
170 CS_LOG_ERR("Number of slaves shall no less than 2");
174 if (sched_ctx->reordering_enabled) {
175 dev->enqueue_burst = schedule_enqueue_ordering;
176 dev->dequeue_burst = schedule_dequeue_ordering;
178 dev->enqueue_burst = schedule_enqueue;
179 dev->dequeue_burst = schedule_dequeue;
182 for (i = 0; i < dev->data->nb_queue_pairs; i++) {
183 struct fo_scheduler_qp_ctx *qp_ctx =
184 ((struct scheduler_qp_ctx *)
185 dev->data->queue_pairs[i])->private_qp_ctx;
187 rte_memcpy(&qp_ctx->primary_slave,
188 &sched_ctx->slaves[PRIMARY_SLAVE_IDX],
189 sizeof(struct scheduler_slave));
190 rte_memcpy(&qp_ctx->secondary_slave,
191 &sched_ctx->slaves[SECONDARY_SLAVE_IDX],
192 sizeof(struct scheduler_slave));
199 scheduler_stop(__rte_unused struct rte_cryptodev *dev)
205 scheduler_config_qp(struct rte_cryptodev *dev, uint16_t qp_id)
207 struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
208 struct fo_scheduler_qp_ctx *fo_qp_ctx;
210 fo_qp_ctx = rte_zmalloc_socket(NULL, sizeof(*fo_qp_ctx), 0,
213 CS_LOG_ERR("failed allocate memory for private queue pair");
217 qp_ctx->private_qp_ctx = (void *)fo_qp_ctx;
223 scheduler_create_private_ctx(__rte_unused struct rte_cryptodev *dev)
228 struct rte_cryptodev_scheduler_ops scheduler_fo_ops = {
234 scheduler_create_private_ctx,
235 NULL, /* option_set */
239 struct rte_cryptodev_scheduler fo_scheduler = {
240 .name = "failover-scheduler",
241 .description = "scheduler which enqueues to the primary slave, "
242 "and only then enqueues to the secondary slave "
243 "upon failing on enqueuing to primary",
244 .mode = CDEV_SCHED_MODE_FAILOVER,
245 .ops = &scheduler_fo_ops
248 struct rte_cryptodev_scheduler *failover_scheduler = &fo_scheduler;