1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2017-2018 NXP
10 #include <rte_byteorder.h>
11 #include <rte_common.h>
12 #include <rte_cryptodev_pmd.h>
13 #include <rte_crypto.h>
14 #include <rte_cryptodev.h>
15 #include <rte_bus_vdev.h>
16 #include <rte_malloc.h>
17 #include <rte_security_driver.h>
18 #include <rte_hexdump.h>
20 #include <caam_jr_config.h>
21 #include <caam_jr_hw_specific.h>
22 #include <caam_jr_pvt.h>
23 #include <caam_jr_log.h>
25 /* RTA header files */
26 #include <hw/desc/common.h>
29 #define CRYPTODEV_NAME_CAAM_JR_PMD crypto_caam_jr
30 static uint8_t cryptodev_driver_id;
33 enum rta_sec_era rta_sec_era;
35 /* Lists the states possible for the SEC user space driver. */
36 enum sec_driver_state_e {
37 SEC_DRIVER_STATE_IDLE, /* Driver not initialized */
38 SEC_DRIVER_STATE_STARTED, /* Driver initialized and can be used*/
39 SEC_DRIVER_STATE_RELEASE, /* Driver release is in progress */
42 /* Job rings used for communication with SEC HW */
43 static struct sec_job_ring_t g_job_rings[MAX_SEC_JOB_RINGS];
45 /* The current state of SEC user space driver */
46 static enum sec_driver_state_e g_driver_state = SEC_DRIVER_STATE_IDLE;
48 /* The number of job rings used by SEC user space driver */
49 static int g_job_rings_no;
50 static int g_job_rings_max;
52 /* @brief Poll the HW for already processed jobs in the JR
53 * and silently discard the available jobs or notify them to UA
54 * with indicated error code.
56 * @param [in,out] job_ring The job ring to poll.
57 * @param [in] do_notify Can be #TRUE or #FALSE. Indicates if
58 * descriptors are to be discarded
59 * or notified to UA with given error_code.
60 * @param [out] notified_descs Number of notified descriptors. Can be NULL
61 * if do_notify is #FALSE
64 hw_flush_job_ring(struct sec_job_ring_t *job_ring,
66 uint32_t *notified_descs)
68 int32_t jobs_no_to_discard = 0;
69 int32_t discarded_descs_no = 0;
71 PMD_INIT_FUNC_TRACE();
72 CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Flushing jr notify desc=[%d]",
73 job_ring, job_ring->pidx, job_ring->cidx, do_notify);
75 jobs_no_to_discard = hw_get_no_finished_jobs(job_ring);
77 /* Discard all jobs */
78 CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Discarding %d descs",
79 job_ring, job_ring->pidx, job_ring->cidx,
82 while (jobs_no_to_discard > discarded_descs_no) {
84 /* Now increment the consumer index for the current job ring,
85 * AFTER saving job in temporary location!
86 * Increment the consumer index for the current job ring
88 job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
91 hw_remove_entries(job_ring, 1);
94 if (do_notify == true) {
95 ASSERT(notified_descs != NULL);
96 *notified_descs = discarded_descs_no;
102 caam_jr_dev_configure(struct rte_cryptodev *dev,
103 struct rte_cryptodev_config *config __rte_unused)
106 struct sec_job_ring_t *internals;
108 PMD_INIT_FUNC_TRACE();
110 internals = dev->data->dev_private;
111 sprintf(str, "ctx_pool_%d", dev->data->dev_id);
112 if (!internals->ctx_pool) {
113 internals->ctx_pool = rte_mempool_create((const char *)str,
115 sizeof(struct caam_jr_op_ctx),
116 CTX_POOL_CACHE_SIZE, 0,
117 NULL, NULL, NULL, NULL,
119 if (!internals->ctx_pool) {
120 CAAM_JR_ERR("%s create failed\n", str);
124 CAAM_JR_INFO("mempool already created for dev_id : %d",
131 caam_jr_dev_start(struct rte_cryptodev *dev __rte_unused)
133 PMD_INIT_FUNC_TRACE();
138 caam_jr_dev_stop(struct rte_cryptodev *dev __rte_unused)
140 PMD_INIT_FUNC_TRACE();
144 caam_jr_dev_close(struct rte_cryptodev *dev)
146 struct sec_job_ring_t *internals;
148 PMD_INIT_FUNC_TRACE();
153 internals = dev->data->dev_private;
154 rte_mempool_free(internals->ctx_pool);
155 internals->ctx_pool = NULL;
161 caam_jr_dev_infos_get(struct rte_cryptodev *dev,
162 struct rte_cryptodev_info *info)
164 struct sec_job_ring_t *internals = dev->data->dev_private;
166 PMD_INIT_FUNC_TRACE();
168 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
169 info->feature_flags = dev->feature_flags;
170 info->sym.max_nb_sessions = internals->max_nb_sessions;
171 info->driver_id = cryptodev_driver_id;
175 static struct rte_cryptodev_ops caam_jr_ops = {
176 .dev_configure = caam_jr_dev_configure,
177 .dev_start = caam_jr_dev_start,
178 .dev_stop = caam_jr_dev_stop,
179 .dev_close = caam_jr_dev_close,
180 .dev_infos_get = caam_jr_dev_infos_get,
184 /* @brief Flush job rings of any processed descs.
185 * The processed descs are silently dropped,
186 * WITHOUT being notified to UA.
189 close_job_ring(struct sec_job_ring_t *job_ring)
191 PMD_INIT_FUNC_TRACE();
192 if (job_ring->irq_fd) {
193 /* Producer index is frozen. If consumer index is not equal
194 * with producer index, then we have descs to flush.
196 while (job_ring->pidx != job_ring->cidx)
197 hw_flush_job_ring(job_ring, false, NULL);
199 /* free the uio job ring */
200 free_job_ring(job_ring->irq_fd);
201 job_ring->irq_fd = 0;
202 caam_jr_dma_free(job_ring->input_ring);
203 caam_jr_dma_free(job_ring->output_ring);
208 /** @brief Release the software and hardware resources tied to a job ring.
209 * @param [in] job_ring The job ring
211 * @retval 0 for success
212 * @retval -1 for error
215 shutdown_job_ring(struct sec_job_ring_t *job_ring)
219 PMD_INIT_FUNC_TRACE();
220 ASSERT(job_ring != NULL);
221 ret = hw_shutdown_job_ring(job_ring);
222 SEC_ASSERT(ret == 0, ret,
223 "Failed to shutdown hardware job ring %p",
226 if (job_ring->coalescing_en)
227 hw_job_ring_disable_coalescing(job_ring);
229 if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL) {
230 ret = caam_jr_disable_irqs(job_ring->irq_fd);
231 SEC_ASSERT(ret == 0, ret,
232 "Failed to disable irqs for job ring %p",
240 * @brief Release the resources used by the SEC user space driver.
242 * Reset and release SEC's job rings indicated by the User Application at
243 * init_job_ring() and free any memory allocated internally.
244 * Call once during application tear down.
246 * @note In case there are any descriptors in-flight (descriptors received by
247 * SEC driver for processing and for which no response was yet provided to UA),
248 * the descriptors are discarded without any notifications to User Application.
250 * @retval ::0 is returned for a successful execution
251 * @retval ::-1 is returned if SEC driver release is in progress
254 caam_jr_dev_uninit(struct rte_cryptodev *dev)
256 struct sec_job_ring_t *internals;
258 PMD_INIT_FUNC_TRACE();
262 internals = dev->data->dev_private;
263 rte_free(dev->security_ctx);
265 /* If any descriptors in flight , poll and wait
266 * until all descriptors are received and silently discarded.
269 shutdown_job_ring(internals);
270 close_job_ring(internals);
271 rte_mempool_free(internals->ctx_pool);
274 CAAM_JR_INFO("Closing crypto device %s", dev->data->name);
276 /* last caam jr instance) */
277 if (g_job_rings_no == 0)
278 g_driver_state = SEC_DRIVER_STATE_IDLE;
283 /* @brief Initialize the software and hardware resources tied to a job ring.
284 * @param [in] jr_mode; Model to be used by SEC Driver to receive
285 * notifications from SEC. Can be either
286 * of the three: #SEC_NOTIFICATION_TYPE_NAPI
287 * #SEC_NOTIFICATION_TYPE_IRQ or
288 * #SEC_NOTIFICATION_TYPE_POLL
289 * @param [in] NAPI_mode The NAPI work mode to configure a job ring at
290 * startup. Used only when #SEC_NOTIFICATION_TYPE
291 * is set to #SEC_NOTIFICATION_TYPE_NAPI.
292 * @param [in] irq_coalescing_timer This value determines the maximum
293 * amount of time after processing a
294 * descriptor before raising an interrupt.
295 * @param [in] irq_coalescing_count This value determines how many
296 * descriptors are completed before
297 * raising an interrupt.
298 * @param [in] reg_base_addr, The job ring base address register
299 * @param [in] irq_id The job ring interrupt identification number.
300 * @retval job_ring_handle for successful job ring configuration
301 * @retval NULL on error
305 init_job_ring(void *reg_base_addr, uint32_t irq_id)
307 struct sec_job_ring_t *job_ring = NULL;
309 int jr_mode = SEC_NOTIFICATION_TYPE_POLL;
311 int irq_coalescing_timer = 0;
312 int irq_coalescing_count = 0;
314 for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
315 if (g_job_rings[i].irq_fd == 0) {
316 job_ring = &g_job_rings[i];
321 if (job_ring == NULL) {
322 CAAM_JR_ERR("No free job ring\n");
326 job_ring->register_base_addr = reg_base_addr;
327 job_ring->jr_mode = jr_mode;
328 job_ring->napi_mode = 0;
329 job_ring->irq_fd = irq_id;
331 /* Allocate mem for input and output ring */
333 /* Allocate memory for input ring */
334 job_ring->input_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
335 SEC_DMA_MEM_INPUT_RING_SIZE);
336 memset(job_ring->input_ring, 0, SEC_DMA_MEM_INPUT_RING_SIZE);
338 /* Allocate memory for output ring */
339 job_ring->output_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
340 SEC_DMA_MEM_OUTPUT_RING_SIZE);
341 memset(job_ring->output_ring, 0, SEC_DMA_MEM_OUTPUT_RING_SIZE);
343 /* Reset job ring in SEC hw and configure job ring registers */
344 ret = hw_reset_job_ring(job_ring);
346 CAAM_JR_ERR("Failed to reset hardware job ring");
350 if (jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
351 /* When SEC US driver works in NAPI mode, the UA can select
352 * if the driver starts with IRQs on or off.
354 if (napi_mode == SEC_STARTUP_INTERRUPT_MODE) {
355 CAAM_JR_INFO("Enabling DONE IRQ generationon job ring - %p",
357 ret = caam_jr_enable_irqs(job_ring->irq_fd);
359 CAAM_JR_ERR("Failed to enable irqs for job ring");
363 } else if (jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
364 /* When SEC US driver works in pure interrupt mode,
365 * IRQ's are always enabled.
367 CAAM_JR_INFO("Enabling DONE IRQ generation on job ring - %p",
369 ret = caam_jr_enable_irqs(job_ring->irq_fd);
371 CAAM_JR_ERR("Failed to enable irqs for job ring");
375 if (irq_coalescing_timer || irq_coalescing_count) {
376 hw_job_ring_set_coalescing_param(job_ring,
377 irq_coalescing_timer,
378 irq_coalescing_count);
380 hw_job_ring_enable_coalescing(job_ring);
381 job_ring->coalescing_en = 1;
384 job_ring->jr_state = SEC_JOB_RING_STATE_STARTED;
385 job_ring->max_nb_queue_pairs = RTE_CAAM_MAX_NB_SEC_QPS;
386 job_ring->max_nb_sessions = RTE_CAAM_JR_PMD_MAX_NB_SESSIONS;
390 caam_jr_dma_free(job_ring->output_ring);
391 caam_jr_dma_free(job_ring->input_ring);
397 caam_jr_dev_init(const char *name,
398 struct rte_vdev_device *vdev,
399 struct rte_cryptodev_pmd_init_params *init_params)
401 struct rte_cryptodev *dev;
402 struct uio_job_ring *job_ring;
403 char str[RTE_CRYPTODEV_NAME_MAX_LEN];
405 PMD_INIT_FUNC_TRACE();
407 /* Validate driver state */
408 if (g_driver_state == SEC_DRIVER_STATE_IDLE) {
409 g_job_rings_max = sec_configure();
410 if (!g_job_rings_max) {
411 CAAM_JR_ERR("No job ring detected on UIO !!!!");
414 /* Update driver state */
415 g_driver_state = SEC_DRIVER_STATE_STARTED;
418 if (g_job_rings_no >= g_job_rings_max) {
419 CAAM_JR_ERR("No more job rings available max=%d!!!!",
424 job_ring = config_job_ring();
425 if (job_ring == NULL) {
426 CAAM_JR_ERR("failed to create job ring");
430 snprintf(str, sizeof(str), "caam_jr%d", job_ring->jr_id);
432 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
434 CAAM_JR_ERR("failed to create cryptodev vdev");
437 /*TODO free it during teardown*/
438 dev->data->dev_private = init_job_ring(job_ring->register_base_addr,
441 if (!dev->data->dev_private) {
442 CAAM_JR_ERR("Ring memory allocation failed\n");
446 dev->driver_id = cryptodev_driver_id;
447 dev->dev_ops = &caam_jr_ops;
449 /* register rx/tx burst functions for data path */
450 dev->dequeue_burst = NULL;
451 dev->enqueue_burst = NULL;
452 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
453 RTE_CRYPTODEV_FF_HW_ACCELERATED |
454 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
455 RTE_CRYPTODEV_FF_SECURITY |
456 RTE_CRYPTODEV_FF_IN_PLACE_SGL |
457 RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
458 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
459 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
460 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
462 /* For secondary processes, we don't initialise any further as primary
463 * has already done this work. Only check we don't need a different
466 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
467 CAAM_JR_WARN("Device already init by primary process");
471 RTE_LOG(INFO, PMD, "%s cryptodev init\n", dev->data->name);
476 caam_jr_dev_uninit(dev);
477 rte_cryptodev_pmd_release_device(dev);
479 free_job_ring(job_ring->uio_fd);
481 CAAM_JR_ERR("driver %s: cryptodev_caam_jr_create failed",
487 /** Initialise CAAM JR crypto device */
489 cryptodev_caam_jr_probe(struct rte_vdev_device *vdev)
491 struct rte_cryptodev_pmd_init_params init_params = {
493 sizeof(struct sec_job_ring_t),
495 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
498 const char *input_args;
500 name = rte_vdev_device_name(vdev);
504 input_args = rte_vdev_device_args(vdev);
505 rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
507 /* if sec device version is not configured */
508 if (!rta_get_sec_era()) {
509 const struct device_node *caam_node;
511 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
512 const uint32_t *prop = of_get_property(caam_node,
517 INTL_SEC_ERA(cpu_to_caam32(*prop)));
522 #ifdef RTE_LIBRTE_PMD_CAAM_JR_BE
523 if (rta_get_sec_era() > RTA_SEC_ERA_8) {
525 "CAAM is compiled in BE mode for device with sec era > 8???\n");
530 return caam_jr_dev_init(name, vdev, &init_params);
533 /** Uninitialise CAAM JR crypto device */
535 cryptodev_caam_jr_remove(struct rte_vdev_device *vdev)
537 struct rte_cryptodev *cryptodev;
540 name = rte_vdev_device_name(vdev);
544 cryptodev = rte_cryptodev_pmd_get_named_dev(name);
545 if (cryptodev == NULL)
548 caam_jr_dev_uninit(cryptodev);
550 return rte_cryptodev_pmd_destroy(cryptodev);
553 static struct rte_vdev_driver cryptodev_caam_jr_drv = {
554 .probe = cryptodev_caam_jr_probe,
555 .remove = cryptodev_caam_jr_remove
558 static struct cryptodev_driver caam_jr_crypto_drv;
560 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CAAM_JR_PMD, cryptodev_caam_jr_drv);
561 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CAAM_JR_PMD,
562 "max_nb_queue_pairs=<int>"
564 RTE_PMD_REGISTER_CRYPTO_DRIVER(caam_jr_crypto_drv, cryptodev_caam_jr_drv.driver,
565 cryptodev_driver_id);
567 RTE_INIT(caam_jr_init_log)
569 caam_jr_logtype = rte_log_register("pmd.crypto.caam");
570 if (caam_jr_logtype >= 0)
571 rte_log_set_level(caam_jr_logtype, RTE_LOG_NOTICE);