1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2017-2018 NXP
10 #include <rte_byteorder.h>
11 #include <rte_common.h>
12 #include <rte_cryptodev_pmd.h>
13 #include <rte_crypto.h>
14 #include <rte_cryptodev.h>
15 #include <rte_bus_vdev.h>
16 #include <rte_malloc.h>
17 #include <rte_security_driver.h>
18 #include <rte_hexdump.h>
20 #include <caam_jr_config.h>
21 #include <caam_jr_hw_specific.h>
22 #include <caam_jr_pvt.h>
23 #include <caam_jr_log.h>
25 /* RTA header files */
26 #include <hw/desc/common.h>
29 #define CRYPTODEV_NAME_CAAM_JR_PMD crypto_caam_jr
30 static uint8_t cryptodev_driver_id;
33 enum rta_sec_era rta_sec_era;
35 /* Lists the states possible for the SEC user space driver. */
36 enum sec_driver_state_e {
37 SEC_DRIVER_STATE_IDLE, /* Driver not initialized */
38 SEC_DRIVER_STATE_STARTED, /* Driver initialized and can be used*/
39 SEC_DRIVER_STATE_RELEASE, /* Driver release is in progress */
42 /* Job rings used for communication with SEC HW */
43 static struct sec_job_ring_t g_job_rings[MAX_SEC_JOB_RINGS];
45 /* The current state of SEC user space driver */
46 static enum sec_driver_state_e g_driver_state = SEC_DRIVER_STATE_IDLE;
48 /* The number of job rings used by SEC user space driver */
49 static int g_job_rings_no;
50 static int g_job_rings_max;
52 /* @brief Poll the HW for already processed jobs in the JR
53 * and silently discard the available jobs or notify them to UA
54 * with indicated error code.
56 * @param [in,out] job_ring The job ring to poll.
57 * @param [in] do_notify Can be #TRUE or #FALSE. Indicates if
58 * descriptors are to be discarded
59 * or notified to UA with given error_code.
60 * @param [out] notified_descs Number of notified descriptors. Can be NULL
61 * if do_notify is #FALSE
64 hw_flush_job_ring(struct sec_job_ring_t *job_ring,
66 uint32_t *notified_descs)
68 int32_t jobs_no_to_discard = 0;
69 int32_t discarded_descs_no = 0;
71 PMD_INIT_FUNC_TRACE();
72 CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Flushing jr notify desc=[%d]",
73 job_ring, job_ring->pidx, job_ring->cidx, do_notify);
75 jobs_no_to_discard = hw_get_no_finished_jobs(job_ring);
77 /* Discard all jobs */
78 CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Discarding %d descs",
79 job_ring, job_ring->pidx, job_ring->cidx,
82 while (jobs_no_to_discard > discarded_descs_no) {
84 /* Now increment the consumer index for the current job ring,
85 * AFTER saving job in temporary location!
86 * Increment the consumer index for the current job ring
88 job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
91 hw_remove_entries(job_ring, 1);
94 if (do_notify == true) {
95 ASSERT(notified_descs != NULL);
96 *notified_descs = discarded_descs_no;
101 /* @brief Flush job rings of any processed descs.
102 * The processed descs are silently dropped,
103 * WITHOUT being notified to UA.
106 close_job_ring(struct sec_job_ring_t *job_ring)
108 PMD_INIT_FUNC_TRACE();
109 if (job_ring->irq_fd) {
110 /* Producer index is frozen. If consumer index is not equal
111 * with producer index, then we have descs to flush.
113 while (job_ring->pidx != job_ring->cidx)
114 hw_flush_job_ring(job_ring, false, NULL);
116 /* free the uio job ring */
117 free_job_ring(job_ring->irq_fd);
118 job_ring->irq_fd = 0;
119 caam_jr_dma_free(job_ring->input_ring);
120 caam_jr_dma_free(job_ring->output_ring);
125 /** @brief Release the software and hardware resources tied to a job ring.
126 * @param [in] job_ring The job ring
128 * @retval 0 for success
129 * @retval -1 for error
132 shutdown_job_ring(struct sec_job_ring_t *job_ring)
136 PMD_INIT_FUNC_TRACE();
137 ASSERT(job_ring != NULL);
138 ret = hw_shutdown_job_ring(job_ring);
139 SEC_ASSERT(ret == 0, ret,
140 "Failed to shutdown hardware job ring %p",
143 if (job_ring->coalescing_en)
144 hw_job_ring_disable_coalescing(job_ring);
146 if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL) {
147 ret = caam_jr_disable_irqs(job_ring->irq_fd);
148 SEC_ASSERT(ret == 0, ret,
149 "Failed to disable irqs for job ring %p",
157 * @brief Release the resources used by the SEC user space driver.
159 * Reset and release SEC's job rings indicated by the User Application at
160 * init_job_ring() and free any memory allocated internally.
161 * Call once during application tear down.
163 * @note In case there are any descriptors in-flight (descriptors received by
164 * SEC driver for processing and for which no response was yet provided to UA),
165 * the descriptors are discarded without any notifications to User Application.
167 * @retval ::0 is returned for a successful execution
168 * @retval ::-1 is returned if SEC driver release is in progress
171 caam_jr_dev_uninit(struct rte_cryptodev *dev)
173 struct sec_job_ring_t *internals;
175 PMD_INIT_FUNC_TRACE();
179 internals = dev->data->dev_private;
180 rte_free(dev->security_ctx);
182 /* If any descriptors in flight , poll and wait
183 * until all descriptors are received and silently discarded.
186 shutdown_job_ring(internals);
187 close_job_ring(internals);
188 rte_mempool_free(internals->ctx_pool);
191 CAAM_JR_INFO("Closing crypto device %s", dev->data->name);
193 /* last caam jr instance) */
194 if (g_job_rings_no == 0)
195 g_driver_state = SEC_DRIVER_STATE_IDLE;
200 /* @brief Initialize the software and hardware resources tied to a job ring.
201 * @param [in] jr_mode; Model to be used by SEC Driver to receive
202 * notifications from SEC. Can be either
203 * of the three: #SEC_NOTIFICATION_TYPE_NAPI
204 * #SEC_NOTIFICATION_TYPE_IRQ or
205 * #SEC_NOTIFICATION_TYPE_POLL
206 * @param [in] NAPI_mode The NAPI work mode to configure a job ring at
207 * startup. Used only when #SEC_NOTIFICATION_TYPE
208 * is set to #SEC_NOTIFICATION_TYPE_NAPI.
209 * @param [in] irq_coalescing_timer This value determines the maximum
210 * amount of time after processing a
211 * descriptor before raising an interrupt.
212 * @param [in] irq_coalescing_count This value determines how many
213 * descriptors are completed before
214 * raising an interrupt.
215 * @param [in] reg_base_addr, The job ring base address register
216 * @param [in] irq_id The job ring interrupt identification number.
217 * @retval job_ring_handle for successful job ring configuration
218 * @retval NULL on error
222 init_job_ring(void *reg_base_addr, uint32_t irq_id)
224 struct sec_job_ring_t *job_ring = NULL;
226 int jr_mode = SEC_NOTIFICATION_TYPE_POLL;
228 int irq_coalescing_timer = 0;
229 int irq_coalescing_count = 0;
231 for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
232 if (g_job_rings[i].irq_fd == 0) {
233 job_ring = &g_job_rings[i];
238 if (job_ring == NULL) {
239 CAAM_JR_ERR("No free job ring\n");
243 job_ring->register_base_addr = reg_base_addr;
244 job_ring->jr_mode = jr_mode;
245 job_ring->napi_mode = 0;
246 job_ring->irq_fd = irq_id;
248 /* Allocate mem for input and output ring */
250 /* Allocate memory for input ring */
251 job_ring->input_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
252 SEC_DMA_MEM_INPUT_RING_SIZE);
253 memset(job_ring->input_ring, 0, SEC_DMA_MEM_INPUT_RING_SIZE);
255 /* Allocate memory for output ring */
256 job_ring->output_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
257 SEC_DMA_MEM_OUTPUT_RING_SIZE);
258 memset(job_ring->output_ring, 0, SEC_DMA_MEM_OUTPUT_RING_SIZE);
260 /* Reset job ring in SEC hw and configure job ring registers */
261 ret = hw_reset_job_ring(job_ring);
263 CAAM_JR_ERR("Failed to reset hardware job ring");
267 if (jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
268 /* When SEC US driver works in NAPI mode, the UA can select
269 * if the driver starts with IRQs on or off.
271 if (napi_mode == SEC_STARTUP_INTERRUPT_MODE) {
272 CAAM_JR_INFO("Enabling DONE IRQ generationon job ring - %p",
274 ret = caam_jr_enable_irqs(job_ring->irq_fd);
276 CAAM_JR_ERR("Failed to enable irqs for job ring");
280 } else if (jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
281 /* When SEC US driver works in pure interrupt mode,
282 * IRQ's are always enabled.
284 CAAM_JR_INFO("Enabling DONE IRQ generation on job ring - %p",
286 ret = caam_jr_enable_irqs(job_ring->irq_fd);
288 CAAM_JR_ERR("Failed to enable irqs for job ring");
292 if (irq_coalescing_timer || irq_coalescing_count) {
293 hw_job_ring_set_coalescing_param(job_ring,
294 irq_coalescing_timer,
295 irq_coalescing_count);
297 hw_job_ring_enable_coalescing(job_ring);
298 job_ring->coalescing_en = 1;
301 job_ring->jr_state = SEC_JOB_RING_STATE_STARTED;
302 job_ring->max_nb_queue_pairs = RTE_CAAM_MAX_NB_SEC_QPS;
303 job_ring->max_nb_sessions = RTE_CAAM_JR_PMD_MAX_NB_SESSIONS;
307 caam_jr_dma_free(job_ring->output_ring);
308 caam_jr_dma_free(job_ring->input_ring);
314 caam_jr_dev_init(const char *name,
315 struct rte_vdev_device *vdev,
316 struct rte_cryptodev_pmd_init_params *init_params)
318 struct rte_cryptodev *dev;
319 struct uio_job_ring *job_ring;
320 char str[RTE_CRYPTODEV_NAME_MAX_LEN];
322 PMD_INIT_FUNC_TRACE();
324 /* Validate driver state */
325 if (g_driver_state == SEC_DRIVER_STATE_IDLE) {
326 g_job_rings_max = sec_configure();
327 if (!g_job_rings_max) {
328 CAAM_JR_ERR("No job ring detected on UIO !!!!");
331 /* Update driver state */
332 g_driver_state = SEC_DRIVER_STATE_STARTED;
335 if (g_job_rings_no >= g_job_rings_max) {
336 CAAM_JR_ERR("No more job rings available max=%d!!!!",
341 job_ring = config_job_ring();
342 if (job_ring == NULL) {
343 CAAM_JR_ERR("failed to create job ring");
347 snprintf(str, sizeof(str), "caam_jr%d", job_ring->jr_id);
349 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
351 CAAM_JR_ERR("failed to create cryptodev vdev");
354 /*TODO free it during teardown*/
355 dev->data->dev_private = init_job_ring(job_ring->register_base_addr,
358 if (!dev->data->dev_private) {
359 CAAM_JR_ERR("Ring memory allocation failed\n");
363 dev->driver_id = cryptodev_driver_id;
366 /* For secondary processes, we don't initialise any further as primary
367 * has already done this work. Only check we don't need a different
370 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
371 CAAM_JR_WARN("Device already init by primary process");
375 RTE_LOG(INFO, PMD, "%s cryptodev init\n", dev->data->name);
380 caam_jr_dev_uninit(dev);
381 rte_cryptodev_pmd_release_device(dev);
383 free_job_ring(job_ring->uio_fd);
385 CAAM_JR_ERR("driver %s: cryptodev_caam_jr_create failed",
391 /** Initialise CAAM JR crypto device */
393 cryptodev_caam_jr_probe(struct rte_vdev_device *vdev)
395 struct rte_cryptodev_pmd_init_params init_params = {
397 sizeof(struct sec_job_ring_t),
399 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
402 const char *input_args;
404 name = rte_vdev_device_name(vdev);
408 input_args = rte_vdev_device_args(vdev);
409 rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
411 /* if sec device version is not configured */
412 if (!rta_get_sec_era()) {
413 const struct device_node *caam_node;
415 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
416 const uint32_t *prop = of_get_property(caam_node,
421 INTL_SEC_ERA(cpu_to_caam32(*prop)));
426 #ifdef RTE_LIBRTE_PMD_CAAM_JR_BE
427 if (rta_get_sec_era() > RTA_SEC_ERA_8) {
429 "CAAM is compiled in BE mode for device with sec era > 8???\n");
434 return caam_jr_dev_init(name, vdev, &init_params);
437 /** Uninitialise CAAM JR crypto device */
439 cryptodev_caam_jr_remove(struct rte_vdev_device *vdev)
441 struct rte_cryptodev *cryptodev;
444 name = rte_vdev_device_name(vdev);
448 cryptodev = rte_cryptodev_pmd_get_named_dev(name);
449 if (cryptodev == NULL)
452 caam_jr_dev_uninit(cryptodev);
454 return rte_cryptodev_pmd_destroy(cryptodev);
457 static struct rte_vdev_driver cryptodev_caam_jr_drv = {
458 .probe = cryptodev_caam_jr_probe,
459 .remove = cryptodev_caam_jr_remove
462 static struct cryptodev_driver caam_jr_crypto_drv;
464 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CAAM_JR_PMD, cryptodev_caam_jr_drv);
465 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CAAM_JR_PMD,
466 "max_nb_queue_pairs=<int>"
468 RTE_PMD_REGISTER_CRYPTO_DRIVER(caam_jr_crypto_drv, cryptodev_caam_jr_drv.driver,
469 cryptodev_driver_id);
471 RTE_INIT(caam_jr_init_log)
473 caam_jr_logtype = rte_log_register("pmd.crypto.caam");
474 if (caam_jr_logtype >= 0)
475 rte_log_set_level(caam_jr_logtype, RTE_LOG_NOTICE);