1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
4 #include <rte_reorder.h>
5 #include <rte_cryptodev.h>
6 #include <rte_cryptodev_pmd.h>
7 #include <rte_malloc.h>
9 #include "rte_cryptodev_scheduler.h"
10 #include "scheduler_pmd_private.h"
12 /** update the scheduler pmd's capability with attaching device's
14 * For each device to be attached, the scheduler's capability should be
15 * the common capability set of all slaves
18 sync_caps(struct rte_cryptodev_capabilities *caps,
20 const struct rte_cryptodev_capabilities *slave_caps)
22 uint32_t sync_nb_caps = nb_caps, nb_slave_caps = 0;
25 while (slave_caps[nb_slave_caps].op != RTE_CRYPTO_OP_TYPE_UNDEFINED)
29 rte_memcpy(caps, slave_caps, sizeof(*caps) * nb_slave_caps);
33 for (i = 0; i < sync_nb_caps; i++) {
34 struct rte_cryptodev_capabilities *cap = &caps[i];
37 for (j = 0; j < nb_slave_caps; j++) {
38 const struct rte_cryptodev_capabilities *s_cap =
41 if (s_cap->op != cap->op || s_cap->sym.xform_type !=
45 if (s_cap->sym.xform_type ==
46 RTE_CRYPTO_SYM_XFORM_AUTH) {
47 if (s_cap->sym.auth.algo !=
51 cap->sym.auth.digest_size.min =
52 s_cap->sym.auth.digest_size.min <
53 cap->sym.auth.digest_size.min ?
54 s_cap->sym.auth.digest_size.min :
55 cap->sym.auth.digest_size.min;
56 cap->sym.auth.digest_size.max =
57 s_cap->sym.auth.digest_size.max <
58 cap->sym.auth.digest_size.max ?
59 s_cap->sym.auth.digest_size.max :
60 cap->sym.auth.digest_size.max;
64 if (s_cap->sym.xform_type ==
65 RTE_CRYPTO_SYM_XFORM_CIPHER)
66 if (s_cap->sym.cipher.algo !=
70 /* no common cap found */
74 if (j < nb_slave_caps)
77 /* remove a uncommon cap from the array */
78 for (j = i; j < sync_nb_caps - 1; j++)
79 rte_memcpy(&caps[j], &caps[j+1], sizeof(*cap));
81 memset(&caps[sync_nb_caps - 1], 0, sizeof(*cap));
89 update_scheduler_capability(struct scheduler_ctx *sched_ctx)
91 struct rte_cryptodev_capabilities tmp_caps[256] = { {0} };
92 uint32_t nb_caps = 0, i;
94 if (sched_ctx->capabilities) {
95 rte_free(sched_ctx->capabilities);
96 sched_ctx->capabilities = NULL;
99 for (i = 0; i < sched_ctx->nb_slaves; i++) {
100 struct rte_cryptodev_info dev_info;
102 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info);
104 nb_caps = sync_caps(tmp_caps, nb_caps, dev_info.capabilities);
109 sched_ctx->capabilities = rte_zmalloc_socket(NULL,
110 sizeof(struct rte_cryptodev_capabilities) *
111 (nb_caps + 1), 0, SOCKET_ID_ANY);
112 if (!sched_ctx->capabilities)
115 rte_memcpy(sched_ctx->capabilities, tmp_caps,
116 sizeof(struct rte_cryptodev_capabilities) * nb_caps);
122 update_scheduler_feature_flag(struct rte_cryptodev *dev)
124 struct scheduler_ctx *sched_ctx = dev->data->dev_private;
127 dev->feature_flags = 0;
129 for (i = 0; i < sched_ctx->nb_slaves; i++) {
130 struct rte_cryptodev_info dev_info;
132 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info);
134 dev->feature_flags |= dev_info.feature_flags;
139 update_max_nb_qp(struct scheduler_ctx *sched_ctx)
144 if (!sched_ctx->nb_slaves)
147 max_nb_qp = sched_ctx->nb_slaves ? UINT32_MAX : 0;
149 for (i = 0; i < sched_ctx->nb_slaves; i++) {
150 struct rte_cryptodev_info dev_info;
152 rte_cryptodev_info_get(sched_ctx->slaves[i].dev_id, &dev_info);
153 max_nb_qp = dev_info.max_nb_queue_pairs < max_nb_qp ?
154 dev_info.max_nb_queue_pairs : max_nb_qp;
157 sched_ctx->max_nb_queue_pairs = max_nb_qp;
160 /** Attach a device to the scheduler. */
162 rte_cryptodev_scheduler_slave_attach(uint8_t scheduler_id, uint8_t slave_id)
164 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
165 struct scheduler_ctx *sched_ctx;
166 struct scheduler_slave *slave;
167 struct rte_cryptodev_info dev_info;
171 CS_LOG_ERR("Operation not supported");
175 if (dev->driver_id != cryptodev_driver_id) {
176 CS_LOG_ERR("Operation not supported");
180 if (dev->data->dev_started) {
181 CS_LOG_ERR("Illegal operation");
185 sched_ctx = dev->data->dev_private;
186 if (sched_ctx->nb_slaves >=
187 RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES) {
188 CS_LOG_ERR("Too many slaves attached");
192 for (i = 0; i < sched_ctx->nb_slaves; i++)
193 if (sched_ctx->slaves[i].dev_id == slave_id) {
194 CS_LOG_ERR("Slave already added");
198 slave = &sched_ctx->slaves[sched_ctx->nb_slaves];
200 rte_cryptodev_info_get(slave_id, &dev_info);
202 slave->dev_id = slave_id;
203 slave->driver_id = dev_info.driver_id;
204 sched_ctx->nb_slaves++;
206 if (update_scheduler_capability(sched_ctx) < 0) {
208 slave->driver_id = 0;
209 sched_ctx->nb_slaves--;
211 CS_LOG_ERR("capabilities update failed");
215 update_scheduler_feature_flag(dev);
217 update_max_nb_qp(sched_ctx);
223 rte_cryptodev_scheduler_slave_detach(uint8_t scheduler_id, uint8_t slave_id)
225 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
226 struct scheduler_ctx *sched_ctx;
227 uint32_t i, slave_pos;
230 CS_LOG_ERR("Operation not supported");
234 if (dev->driver_id != cryptodev_driver_id) {
235 CS_LOG_ERR("Operation not supported");
239 if (dev->data->dev_started) {
240 CS_LOG_ERR("Illegal operation");
244 sched_ctx = dev->data->dev_private;
246 for (slave_pos = 0; slave_pos < sched_ctx->nb_slaves; slave_pos++)
247 if (sched_ctx->slaves[slave_pos].dev_id == slave_id)
249 if (slave_pos == sched_ctx->nb_slaves) {
250 CS_LOG_ERR("Cannot find slave");
254 if (sched_ctx->ops.slave_detach(dev, slave_id) < 0) {
255 CS_LOG_ERR("Failed to detach slave");
259 for (i = slave_pos; i < sched_ctx->nb_slaves - 1; i++) {
260 memcpy(&sched_ctx->slaves[i], &sched_ctx->slaves[i+1],
261 sizeof(struct scheduler_slave));
263 memset(&sched_ctx->slaves[sched_ctx->nb_slaves - 1], 0,
264 sizeof(struct scheduler_slave));
265 sched_ctx->nb_slaves--;
267 if (update_scheduler_capability(sched_ctx) < 0) {
268 CS_LOG_ERR("capabilities update failed");
272 update_scheduler_feature_flag(dev);
274 update_max_nb_qp(sched_ctx);
280 rte_cryptodev_scheduler_mode_set(uint8_t scheduler_id,
281 enum rte_cryptodev_scheduler_mode mode)
283 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
284 struct scheduler_ctx *sched_ctx;
287 CS_LOG_ERR("Operation not supported");
291 if (dev->driver_id != cryptodev_driver_id) {
292 CS_LOG_ERR("Operation not supported");
296 if (dev->data->dev_started) {
297 CS_LOG_ERR("Illegal operation");
301 sched_ctx = dev->data->dev_private;
303 if (mode == sched_ctx->mode)
307 case CDEV_SCHED_MODE_ROUNDROBIN:
308 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
309 roundrobin_scheduler) < 0) {
310 CS_LOG_ERR("Failed to load scheduler");
314 case CDEV_SCHED_MODE_PKT_SIZE_DISTR:
315 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
316 pkt_size_based_distr_scheduler) < 0) {
317 CS_LOG_ERR("Failed to load scheduler");
321 case CDEV_SCHED_MODE_FAILOVER:
322 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
323 failover_scheduler) < 0) {
324 CS_LOG_ERR("Failed to load scheduler");
328 case CDEV_SCHED_MODE_MULTICORE:
329 if (rte_cryptodev_scheduler_load_user_scheduler(scheduler_id,
330 multicore_scheduler) < 0) {
331 CS_LOG_ERR("Failed to load scheduler");
336 CS_LOG_ERR("Not yet supported");
343 enum rte_cryptodev_scheduler_mode
344 rte_cryptodev_scheduler_mode_get(uint8_t scheduler_id)
346 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
347 struct scheduler_ctx *sched_ctx;
350 CS_LOG_ERR("Operation not supported");
354 if (dev->driver_id != cryptodev_driver_id) {
355 CS_LOG_ERR("Operation not supported");
359 sched_ctx = dev->data->dev_private;
361 return sched_ctx->mode;
365 rte_cryptodev_scheduler_ordering_set(uint8_t scheduler_id,
366 uint32_t enable_reorder)
368 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
369 struct scheduler_ctx *sched_ctx;
372 CS_LOG_ERR("Operation not supported");
376 if (dev->driver_id != cryptodev_driver_id) {
377 CS_LOG_ERR("Operation not supported");
381 if (dev->data->dev_started) {
382 CS_LOG_ERR("Illegal operation");
386 sched_ctx = dev->data->dev_private;
388 sched_ctx->reordering_enabled = enable_reorder;
394 rte_cryptodev_scheduler_ordering_get(uint8_t scheduler_id)
396 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
397 struct scheduler_ctx *sched_ctx;
400 CS_LOG_ERR("Operation not supported");
404 if (dev->driver_id != cryptodev_driver_id) {
405 CS_LOG_ERR("Operation not supported");
409 sched_ctx = dev->data->dev_private;
411 return (int)sched_ctx->reordering_enabled;
415 rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id,
416 struct rte_cryptodev_scheduler *scheduler) {
418 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
419 struct scheduler_ctx *sched_ctx;
422 CS_LOG_ERR("Operation not supported");
426 if (dev->driver_id != cryptodev_driver_id) {
427 CS_LOG_ERR("Operation not supported");
431 if (dev->data->dev_started) {
432 CS_LOG_ERR("Illegal operation");
436 sched_ctx = dev->data->dev_private;
438 if (strlen(scheduler->name) > RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
439 CS_LOG_ERR("Invalid name %s, should be less than "
440 "%u bytes.\n", scheduler->name,
441 RTE_CRYPTODEV_NAME_MAX_LEN);
444 snprintf(sched_ctx->name, sizeof(sched_ctx->name), "%s",
447 if (strlen(scheduler->description) >
448 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1) {
449 CS_LOG_ERR("Invalid description %s, should be less than "
450 "%u bytes.\n", scheduler->description,
451 RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN - 1);
454 snprintf(sched_ctx->description, sizeof(sched_ctx->description), "%s",
455 scheduler->description);
457 /* load scheduler instance operations functions */
458 sched_ctx->ops.config_queue_pair = scheduler->ops->config_queue_pair;
459 sched_ctx->ops.create_private_ctx = scheduler->ops->create_private_ctx;
460 sched_ctx->ops.scheduler_start = scheduler->ops->scheduler_start;
461 sched_ctx->ops.scheduler_stop = scheduler->ops->scheduler_stop;
462 sched_ctx->ops.slave_attach = scheduler->ops->slave_attach;
463 sched_ctx->ops.slave_detach = scheduler->ops->slave_detach;
464 sched_ctx->ops.option_set = scheduler->ops->option_set;
465 sched_ctx->ops.option_get = scheduler->ops->option_get;
467 if (sched_ctx->private_ctx) {
468 rte_free(sched_ctx->private_ctx);
469 sched_ctx->private_ctx = NULL;
472 if (sched_ctx->ops.create_private_ctx) {
473 int ret = (*sched_ctx->ops.create_private_ctx)(dev);
476 CS_LOG_ERR("Unable to create scheduler private "
482 sched_ctx->mode = scheduler->mode;
488 rte_cryptodev_scheduler_slaves_get(uint8_t scheduler_id, uint8_t *slaves)
490 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
491 struct scheduler_ctx *sched_ctx;
492 uint32_t nb_slaves = 0;
495 CS_LOG_ERR("Operation not supported");
499 if (dev->driver_id != cryptodev_driver_id) {
500 CS_LOG_ERR("Operation not supported");
504 sched_ctx = dev->data->dev_private;
506 nb_slaves = sched_ctx->nb_slaves;
508 if (slaves && nb_slaves) {
511 for (i = 0; i < nb_slaves; i++)
512 slaves[i] = sched_ctx->slaves[i].dev_id;
515 return (int)nb_slaves;
519 rte_cryptodev_scheduler_option_set(uint8_t scheduler_id,
520 enum rte_cryptodev_schedule_option_type option_type,
523 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
524 struct scheduler_ctx *sched_ctx;
526 if (option_type == CDEV_SCHED_OPTION_NOT_SET ||
527 option_type >= CDEV_SCHED_OPTION_COUNT) {
528 CS_LOG_ERR("Invalid option parameter");
533 CS_LOG_ERR("Invalid option parameter");
537 if (dev->data->dev_started) {
538 CS_LOG_ERR("Illegal operation");
542 sched_ctx = dev->data->dev_private;
544 RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_set, -ENOTSUP);
546 return (*sched_ctx->ops.option_set)(dev, option_type, option);
550 rte_cryptodev_scheduler_option_get(uint8_t scheduler_id,
551 enum rte_cryptodev_schedule_option_type option_type,
554 struct rte_cryptodev *dev = rte_cryptodev_pmd_get_dev(scheduler_id);
555 struct scheduler_ctx *sched_ctx;
558 CS_LOG_ERR("Operation not supported");
563 CS_LOG_ERR("Invalid option parameter");
567 if (dev->driver_id != cryptodev_driver_id) {
568 CS_LOG_ERR("Operation not supported");
572 sched_ctx = dev->data->dev_private;
574 RTE_FUNC_PTR_OR_ERR_RET(*sched_ctx->ops.option_get, -ENOTSUP);
576 return (*sched_ctx->ops.option_get)(dev, option_type, option);