drivers/crypto/caam_jr/caam_jr.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017-2018 NXP
   3  */
   4
   5 #include <fcntl.h>
   6 #include <unistd.h>
   7 #include <sched.h>
   8 #include <net/if.h>
   9
  10 #include <rte_byteorder.h>
  11 #include <rte_common.h>
  12 #include <rte_cryptodev_pmd.h>
  13 #include <rte_crypto.h>
  14 #include <rte_cryptodev.h>
  15 #include <rte_bus_vdev.h>
  16 #include <rte_malloc.h>
  17 #include <rte_security_driver.h>
  18 #include <rte_hexdump.h>
  19
  20 #include <caam_jr_config.h>
  21 #include <caam_jr_hw_specific.h>
  22 #include <caam_jr_pvt.h>
  23 #include <caam_jr_log.h>
  24
  25 /* RTA header files */
  26 #include <hw/desc/common.h>
  27 #include <hw/desc/algo.h>
  28 #include <hw/desc/ipsec.h>
  29 #include <of.h>
  30
  31 #define CAAM_JR_DBG     0
  32 #define CRYPTODEV_NAME_CAAM_JR_PMD      crypto_caam_jr
  33 static uint8_t cryptodev_driver_id;
  34 int caam_jr_logtype;
  35
  36 enum rta_sec_era rta_sec_era;
  37
  38 /* Lists the states possible for the SEC user space driver. */
  39 enum sec_driver_state_e {
  40         SEC_DRIVER_STATE_IDLE,          /* Driver not initialized */
  41         SEC_DRIVER_STATE_STARTED,       /* Driver initialized and can be used*/
  42         SEC_DRIVER_STATE_RELEASE,       /* Driver release is in progress */
  43 };
  44
  45 /* Job rings used for communication with SEC HW */
  46 static struct sec_job_ring_t g_job_rings[MAX_SEC_JOB_RINGS];
  47
  48 /* The current state of SEC user space driver */
  49 static enum sec_driver_state_e g_driver_state = SEC_DRIVER_STATE_IDLE;
  50
  51 /* The number of job rings used by SEC user space driver */
  52 static int g_job_rings_no;
  53 static int g_job_rings_max;
  54
  55 /* @brief Poll the HW for already processed jobs in the JR
  56  * and silently discard the available jobs or notify them to UA
  57  * with indicated error code.
  58  *
  59  * @param [in,out]  job_ring        The job ring to poll.
  60  * @param [in]  do_notify           Can be #TRUE or #FALSE. Indicates if
  61  *                                  descriptors are to be discarded
  62  *                                  or notified to UA with given error_code.
  63  * @param [out] notified_descs    Number of notified descriptors. Can be NULL
  64  *                                      if do_notify is #FALSE
  65  */
  66 static void
  67 hw_flush_job_ring(struct sec_job_ring_t *job_ring,
  68                   uint32_t do_notify,
  69                   uint32_t *notified_descs)
  70 {
  71         int32_t jobs_no_to_discard = 0;
  72         int32_t discarded_descs_no = 0;
  73
  74         PMD_INIT_FUNC_TRACE();
  75         CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Flushing jr notify desc=[%d]",
  76                 job_ring, job_ring->pidx, job_ring->cidx, do_notify);
  77
  78         jobs_no_to_discard = hw_get_no_finished_jobs(job_ring);
  79
  80         /* Discard all jobs */
  81         CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Discarding %d descs",
  82                   job_ring, job_ring->pidx, job_ring->cidx,
  83                   jobs_no_to_discard);
  84
  85         while (jobs_no_to_discard > discarded_descs_no) {
  86                 discarded_descs_no++;
  87                 /* Now increment the consumer index for the current job ring,
  88                  * AFTER saving job in temporary location!
  89                  * Increment the consumer index for the current job ring
  90                  */
  91                 job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
  92                                          SEC_JOB_RING_SIZE);
  93
  94                 hw_remove_entries(job_ring, 1);
  95         }
  96
  97         if (do_notify == true) {
  98                 ASSERT(notified_descs != NULL);
  99                 *notified_descs = discarded_descs_no;
 100         }
 101 }
 102
 103 /* Release queue pair */
 104 static int
 105 caam_jr_queue_pair_release(struct rte_cryptodev *dev,
 106                            uint16_t qp_id)
 107 {
 108         struct sec_job_ring_t *internals;
 109         struct caam_jr_qp *qp = NULL;
 110
 111         PMD_INIT_FUNC_TRACE();
 112         CAAM_JR_DEBUG("dev =%p, queue =%d", dev, qp_id);
 113
 114         internals = dev->data->dev_private;
 115         if (qp_id >= internals->max_nb_queue_pairs) {
 116                 CAAM_JR_ERR("Max supported qpid %d",
 117                              internals->max_nb_queue_pairs);
 118                 return -EINVAL;
 119         }
 120
 121         qp = &internals->qps[qp_id];
 122         qp->ring = NULL;
 123         dev->data->queue_pairs[qp_id] = NULL;
 124
 125         return 0;
 126 }
 127
 128 /* Setup a queue pair */
 129 static int
 130 caam_jr_queue_pair_setup(
 131                 struct rte_cryptodev *dev, uint16_t qp_id,
 132                 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
 133                 __rte_unused int socket_id,
 134                 __rte_unused struct rte_mempool *session_pool)
 135 {
 136         struct sec_job_ring_t *internals;
 137         struct caam_jr_qp *qp = NULL;
 138
 139         PMD_INIT_FUNC_TRACE();
 140         CAAM_JR_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);
 141
 142         internals = dev->data->dev_private;
 143         if (qp_id >= internals->max_nb_queue_pairs) {
 144                 CAAM_JR_ERR("Max supported qpid %d",
 145                              internals->max_nb_queue_pairs);
 146                 return -EINVAL;
 147         }
 148
 149         qp = &internals->qps[qp_id];
 150         qp->ring = internals;
 151         dev->data->queue_pairs[qp_id] = qp;
 152
 153         return 0;
 154 }
 155
 156 /* Return the number of allocated queue pairs */
 157 static uint32_t
 158 caam_jr_queue_pair_count(struct rte_cryptodev *dev)
 159 {
 160         PMD_INIT_FUNC_TRACE();
 161
 162         return dev->data->nb_queue_pairs;
 163 }
 164
 165 /* Returns the size of the aesni gcm session structure */
 166 static unsigned int
 167 caam_jr_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
 168 {
 169         PMD_INIT_FUNC_TRACE();
 170
 171         return sizeof(struct caam_jr_session);
 172 }
 173
 174 static int
 175 caam_jr_cipher_init(struct rte_cryptodev *dev __rte_unused,
 176                     struct rte_crypto_sym_xform *xform,
 177                     struct caam_jr_session *session)
 178 {
 179         PMD_INIT_FUNC_TRACE();
 180         session->cipher_alg = xform->cipher.algo;
 181         session->iv.length = xform->cipher.iv.length;
 182         session->iv.offset = xform->cipher.iv.offset;
 183         session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
 184                                                RTE_CACHE_LINE_SIZE);
 185         if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
 186                 CAAM_JR_ERR("No Memory for cipher key\n");
 187                 return -ENOMEM;
 188         }
 189         session->cipher_key.length = xform->cipher.key.length;
 190
 191         memcpy(session->cipher_key.data, xform->cipher.key.data,
 192                xform->cipher.key.length);
 193         session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
 194                         DIR_ENC : DIR_DEC;
 195
 196         return 0;
 197 }
 198
 199 static int
 200 caam_jr_auth_init(struct rte_cryptodev *dev __rte_unused,
 201                   struct rte_crypto_sym_xform *xform,
 202                   struct caam_jr_session *session)
 203 {
 204         PMD_INIT_FUNC_TRACE();
 205         session->auth_alg = xform->auth.algo;
 206         session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
 207                                              RTE_CACHE_LINE_SIZE);
 208         if (session->auth_key.data == NULL && xform->auth.key.length > 0) {
 209                 CAAM_JR_ERR("No Memory for auth key\n");
 210                 return -ENOMEM;
 211         }
 212         session->auth_key.length = xform->auth.key.length;
 213         session->digest_length = xform->auth.digest_length;
 214
 215         memcpy(session->auth_key.data, xform->auth.key.data,
 216                xform->auth.key.length);
 217         session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
 218                         DIR_ENC : DIR_DEC;
 219
 220         return 0;
 221 }
 222
 223 static int
 224 caam_jr_aead_init(struct rte_cryptodev *dev __rte_unused,
 225                   struct rte_crypto_sym_xform *xform,
 226                   struct caam_jr_session *session)
 227 {
 228         PMD_INIT_FUNC_TRACE();
 229         session->aead_alg = xform->aead.algo;
 230         session->iv.length = xform->aead.iv.length;
 231         session->iv.offset = xform->aead.iv.offset;
 232         session->auth_only_len = xform->aead.aad_length;
 233         session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length,
 234                                              RTE_CACHE_LINE_SIZE);
 235         if (session->aead_key.data == NULL && xform->aead.key.length > 0) {
 236                 CAAM_JR_ERR("No Memory for aead key\n");
 237                 return -ENOMEM;
 238         }
 239         session->aead_key.length = xform->aead.key.length;
 240         session->digest_length = xform->aead.digest_length;
 241
 242         memcpy(session->aead_key.data, xform->aead.key.data,
 243                xform->aead.key.length);
 244         session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
 245                         DIR_ENC : DIR_DEC;
 246
 247         return 0;
 248 }
 249
 250 static int
 251 caam_jr_set_session_parameters(struct rte_cryptodev *dev,
 252                                struct rte_crypto_sym_xform *xform, void *sess)
 253 {
 254         struct sec_job_ring_t *internals = dev->data->dev_private;
 255         struct caam_jr_session *session = sess;
 256
 257         PMD_INIT_FUNC_TRACE();
 258
 259         if (unlikely(sess == NULL)) {
 260                 CAAM_JR_ERR("invalid session struct");
 261                 return -EINVAL;
 262         }
 263
 264         /* Default IV length = 0 */
 265         session->iv.length = 0;
 266
 267         /* Cipher Only */
 268         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
 269                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
 270                 caam_jr_cipher_init(dev, xform, session);
 271
 272         /* Authentication Only */
 273         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 274                    xform->next == NULL) {
 275                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
 276                 caam_jr_auth_init(dev, xform, session);
 277
 278         /* Cipher then Authenticate */
 279         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 280                    xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 281                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 282                         caam_jr_cipher_init(dev, xform, session);
 283                         caam_jr_auth_init(dev, xform->next, session);
 284                 } else {
 285                         CAAM_JR_ERR("Not supported: Auth then Cipher");
 286                         goto err1;
 287                 }
 288
 289         /* Authenticate then Cipher */
 290         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 291                    xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 292                 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 293                         caam_jr_auth_init(dev, xform, session);
 294                         caam_jr_cipher_init(dev, xform->next, session);
 295                 } else {
 296                         CAAM_JR_ERR("Not supported: Auth then Cipher");
 297                         goto err1;
 298                 }
 299
 300         /* AEAD operation for AES-GCM kind of Algorithms */
 301         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
 302                    xform->next == NULL) {
 303                 caam_jr_aead_init(dev, xform, session);
 304
 305         } else {
 306                 CAAM_JR_ERR("Invalid crypto type");
 307                 return -EINVAL;
 308         }
 309         session->ctx_pool = internals->ctx_pool;
 310
 311         return 0;
 312
 313 err1:
 314         rte_free(session->cipher_key.data);
 315         rte_free(session->auth_key.data);
 316         memset(session, 0, sizeof(struct caam_jr_session));
 317
 318         return -EINVAL;
 319 }
 320
 321 static int
 322 caam_jr_sym_session_configure(struct rte_cryptodev *dev,
 323                               struct rte_crypto_sym_xform *xform,
 324                               struct rte_cryptodev_sym_session *sess,
 325                               struct rte_mempool *mempool)
 326 {
 327         void *sess_private_data;
 328         int ret;
 329
 330         PMD_INIT_FUNC_TRACE();
 331
 332         if (rte_mempool_get(mempool, &sess_private_data)) {
 333                 CAAM_JR_ERR("Couldn't get object from session mempool");
 334                 return -ENOMEM;
 335         }
 336
 337         memset(sess_private_data, 0, sizeof(struct caam_jr_session));
 338         ret = caam_jr_set_session_parameters(dev, xform, sess_private_data);
 339         if (ret != 0) {
 340                 CAAM_JR_ERR("failed to configure session parameters");
 341                 /* Return session to mempool */
 342                 rte_mempool_put(mempool, sess_private_data);
 343                 return ret;
 344         }
 345
 346         set_sym_session_private_data(sess, dev->driver_id, sess_private_data);
 347
 348         return 0;
 349 }
 350
 351 /* Clear the memory of session so it doesn't leave key material behind */
 352 static void
 353 caam_jr_sym_session_clear(struct rte_cryptodev *dev,
 354                 struct rte_cryptodev_sym_session *sess)
 355 {
 356         uint8_t index = dev->driver_id;
 357         void *sess_priv = get_sym_session_private_data(sess, index);
 358         struct caam_jr_session *s = (struct caam_jr_session *)sess_priv;
 359
 360         PMD_INIT_FUNC_TRACE();
 361
 362         if (sess_priv) {
 363                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
 364
 365                 rte_free(s->cipher_key.data);
 366                 rte_free(s->auth_key.data);
 367                 memset(s, 0, sizeof(struct caam_jr_session));
 368                 set_sym_session_private_data(sess, index, NULL);
 369                 rte_mempool_put(sess_mp, sess_priv);
 370         }
 371 }
 372
 373 static int
 374 caam_jr_dev_configure(struct rte_cryptodev *dev,
 375                        struct rte_cryptodev_config *config __rte_unused)
 376 {
 377         char str[20];
 378         struct sec_job_ring_t *internals;
 379
 380         PMD_INIT_FUNC_TRACE();
 381
 382         internals = dev->data->dev_private;
 383         sprintf(str, "ctx_pool_%d", dev->data->dev_id);
 384         if (!internals->ctx_pool) {
 385                 internals->ctx_pool = rte_mempool_create((const char *)str,
 386                                                 CTX_POOL_NUM_BUFS,
 387                                                 sizeof(struct caam_jr_op_ctx),
 388                                                 CTX_POOL_CACHE_SIZE, 0,
 389                                                 NULL, NULL, NULL, NULL,
 390                                                 SOCKET_ID_ANY, 0);
 391                 if (!internals->ctx_pool) {
 392                         CAAM_JR_ERR("%s create failed\n", str);
 393                         return -ENOMEM;
 394                 }
 395         } else
 396                 CAAM_JR_INFO("mempool already created for dev_id : %d",
 397                                 dev->data->dev_id);
 398
 399         return 0;
 400 }
 401
 402 static int
 403 caam_jr_dev_start(struct rte_cryptodev *dev __rte_unused)
 404 {
 405         PMD_INIT_FUNC_TRACE();
 406         return 0;
 407 }
 408
 409 static void
 410 caam_jr_dev_stop(struct rte_cryptodev *dev __rte_unused)
 411 {
 412         PMD_INIT_FUNC_TRACE();
 413 }
 414
 415 static int
 416 caam_jr_dev_close(struct rte_cryptodev *dev)
 417 {
 418         struct sec_job_ring_t *internals;
 419
 420         PMD_INIT_FUNC_TRACE();
 421
 422         if (dev == NULL)
 423                 return -ENOMEM;
 424
 425         internals = dev->data->dev_private;
 426         rte_mempool_free(internals->ctx_pool);
 427         internals->ctx_pool = NULL;
 428
 429         return 0;
 430 }
 431
 432 static void
 433 caam_jr_dev_infos_get(struct rte_cryptodev *dev,
 434                        struct rte_cryptodev_info *info)
 435 {
 436         struct sec_job_ring_t *internals = dev->data->dev_private;
 437
 438         PMD_INIT_FUNC_TRACE();
 439         if (info != NULL) {
 440                 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
 441                 info->feature_flags = dev->feature_flags;
 442                 info->sym.max_nb_sessions = internals->max_nb_sessions;
 443                 info->driver_id = cryptodev_driver_id;
 444         }
 445 }
 446
 447 static struct rte_cryptodev_ops caam_jr_ops = {
 448         .dev_configure        = caam_jr_dev_configure,
 449         .dev_start            = caam_jr_dev_start,
 450         .dev_stop             = caam_jr_dev_stop,
 451         .dev_close            = caam_jr_dev_close,
 452         .dev_infos_get        = caam_jr_dev_infos_get,
 453         .queue_pair_setup     = caam_jr_queue_pair_setup,
 454         .queue_pair_release   = caam_jr_queue_pair_release,
 455         .queue_pair_count     = caam_jr_queue_pair_count,
 456         .sym_session_get_size = caam_jr_sym_session_get_size,
 457         .sym_session_configure = caam_jr_sym_session_configure,
 458         .sym_session_clear    = caam_jr_sym_session_clear
 459 };
 460
 461
 462 /* @brief Flush job rings of any processed descs.
 463  * The processed descs are silently dropped,
 464  * WITHOUT being notified to UA.
 465  */
 466 static void
 467 close_job_ring(struct sec_job_ring_t *job_ring)
 468 {
 469         PMD_INIT_FUNC_TRACE();
 470         if (job_ring->irq_fd) {
 471                 /* Producer index is frozen. If consumer index is not equal
 472                  * with producer index, then we have descs to flush.
 473                  */
 474                 while (job_ring->pidx != job_ring->cidx)
 475                         hw_flush_job_ring(job_ring, false, NULL);
 476
 477                 /* free the uio job ring */
 478                 free_job_ring(job_ring->irq_fd);
 479                 job_ring->irq_fd = 0;
 480                 caam_jr_dma_free(job_ring->input_ring);
 481                 caam_jr_dma_free(job_ring->output_ring);
 482                 g_job_rings_no--;
 483         }
 484 }
 485
 486 /** @brief Release the software and hardware resources tied to a job ring.
 487  * @param [in] job_ring The job ring
 488  *
 489  * @retval  0 for success
 490  * @retval  -1 for error
 491  */
 492 static int
 493 shutdown_job_ring(struct sec_job_ring_t *job_ring)
 494 {
 495         int ret = 0;
 496
 497         PMD_INIT_FUNC_TRACE();
 498         ASSERT(job_ring != NULL);
 499         ret = hw_shutdown_job_ring(job_ring);
 500         SEC_ASSERT(ret == 0, ret,
 501                 "Failed to shutdown hardware job ring %p",
 502                 job_ring);
 503
 504         if (job_ring->coalescing_en)
 505                 hw_job_ring_disable_coalescing(job_ring);
 506
 507         if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL) {
 508                 ret = caam_jr_disable_irqs(job_ring->irq_fd);
 509                 SEC_ASSERT(ret == 0, ret,
 510                 "Failed to disable irqs for job ring %p",
 511                 job_ring);
 512         }
 513
 514         return ret;
 515 }
 516
 517 /*
 518  * @brief Release the resources used by the SEC user space driver.
 519  *
 520  * Reset and release SEC's job rings indicated by the User Application at
 521  * init_job_ring() and free any memory allocated internally.
 522  * Call once during application tear down.
 523  *
 524  * @note In case there are any descriptors in-flight (descriptors received by
 525  * SEC driver for processing and for which no response was yet provided to UA),
 526  * the descriptors are discarded without any notifications to User Application.
 527  *
 528  * @retval ::0                  is returned for a successful execution
 529  * @retval ::-1         is returned if SEC driver release is in progress
 530  */
 531 static int
 532 caam_jr_dev_uninit(struct rte_cryptodev *dev)
 533 {
 534         struct sec_job_ring_t *internals;
 535
 536         PMD_INIT_FUNC_TRACE();
 537         if (dev == NULL)
 538                 return -ENODEV;
 539
 540         internals = dev->data->dev_private;
 541         rte_free(dev->security_ctx);
 542
 543         /* If any descriptors in flight , poll and wait
 544          * until all descriptors are received and silently discarded.
 545          */
 546         if (internals) {
 547                 shutdown_job_ring(internals);
 548                 close_job_ring(internals);
 549                 rte_mempool_free(internals->ctx_pool);
 550         }
 551
 552         CAAM_JR_INFO("Closing crypto device %s", dev->data->name);
 553
 554         /* last caam jr instance) */
 555         if (g_job_rings_no == 0)
 556                 g_driver_state = SEC_DRIVER_STATE_IDLE;
 557
 558         return SEC_SUCCESS;
 559 }
 560
 561 /* @brief Initialize the software and hardware resources tied to a job ring.
 562  * @param [in] jr_mode;         Model to be used by SEC Driver to receive
 563  *                              notifications from SEC.  Can be either
 564  *                              of the three: #SEC_NOTIFICATION_TYPE_NAPI
 565  *                              #SEC_NOTIFICATION_TYPE_IRQ or
 566  *                              #SEC_NOTIFICATION_TYPE_POLL
 567  * @param [in] NAPI_mode        The NAPI work mode to configure a job ring at
 568  *                              startup. Used only when #SEC_NOTIFICATION_TYPE
 569  *                              is set to #SEC_NOTIFICATION_TYPE_NAPI.
 570  * @param [in] irq_coalescing_timer This value determines the maximum
 571  *                                      amount of time after processing a
 572  *                                      descriptor before raising an interrupt.
 573  * @param [in] irq_coalescing_count This value determines how many
 574  *                                      descriptors are completed before
 575  *                                      raising an interrupt.
 576  * @param [in] reg_base_addr,   The job ring base address register
 577  * @param [in] irq_id           The job ring interrupt identification number.
 578  * @retval  job_ring_handle for successful job ring configuration
 579  * @retval  NULL on error
 580  *
 581  */
 582 static void *
 583 init_job_ring(void *reg_base_addr, uint32_t irq_id)
 584 {
 585         struct sec_job_ring_t *job_ring = NULL;
 586         int i, ret = 0;
 587         int jr_mode = SEC_NOTIFICATION_TYPE_POLL;
 588         int napi_mode = 0;
 589         int irq_coalescing_timer = 0;
 590         int irq_coalescing_count = 0;
 591
 592         for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
 593                 if (g_job_rings[i].irq_fd == 0) {
 594                         job_ring = &g_job_rings[i];
 595                         g_job_rings_no++;
 596                         break;
 597                 }
 598         }
 599         if (job_ring == NULL) {
 600                 CAAM_JR_ERR("No free job ring\n");
 601                 return NULL;
 602         }
 603
 604         job_ring->register_base_addr = reg_base_addr;
 605         job_ring->jr_mode = jr_mode;
 606         job_ring->napi_mode = 0;
 607         job_ring->irq_fd = irq_id;
 608
 609         /* Allocate mem for input and output ring */
 610
 611         /* Allocate memory for input ring */
 612         job_ring->input_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
 613                                 SEC_DMA_MEM_INPUT_RING_SIZE);
 614         memset(job_ring->input_ring, 0, SEC_DMA_MEM_INPUT_RING_SIZE);
 615
 616         /* Allocate memory for output ring */
 617         job_ring->output_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
 618                                 SEC_DMA_MEM_OUTPUT_RING_SIZE);
 619         memset(job_ring->output_ring, 0, SEC_DMA_MEM_OUTPUT_RING_SIZE);
 620
 621         /* Reset job ring in SEC hw and configure job ring registers */
 622         ret = hw_reset_job_ring(job_ring);
 623         if (ret != 0) {
 624                 CAAM_JR_ERR("Failed to reset hardware job ring");
 625                 goto cleanup;
 626         }
 627
 628         if (jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
 629         /* When SEC US driver works in NAPI mode, the UA can select
 630          * if the driver starts with IRQs on or off.
 631          */
 632                 if (napi_mode == SEC_STARTUP_INTERRUPT_MODE) {
 633                         CAAM_JR_INFO("Enabling DONE IRQ generationon job ring - %p",
 634                                 job_ring);
 635                         ret = caam_jr_enable_irqs(job_ring->irq_fd);
 636                         if (ret != 0) {
 637                                 CAAM_JR_ERR("Failed to enable irqs for job ring");
 638                                 goto cleanup;
 639                         }
 640                 }
 641         } else if (jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
 642         /* When SEC US driver works in pure interrupt mode,
 643          * IRQ's are always enabled.
 644          */
 645                 CAAM_JR_INFO("Enabling DONE IRQ generation on job ring - %p",
 646                          job_ring);
 647                 ret = caam_jr_enable_irqs(job_ring->irq_fd);
 648                 if (ret != 0) {
 649                         CAAM_JR_ERR("Failed to enable irqs for job ring");
 650                         goto cleanup;
 651                 }
 652         }
 653         if (irq_coalescing_timer || irq_coalescing_count) {
 654                 hw_job_ring_set_coalescing_param(job_ring,
 655                          irq_coalescing_timer,
 656                          irq_coalescing_count);
 657
 658                 hw_job_ring_enable_coalescing(job_ring);
 659                 job_ring->coalescing_en = 1;
 660         }
 661
 662         job_ring->jr_state = SEC_JOB_RING_STATE_STARTED;
 663         job_ring->max_nb_queue_pairs = RTE_CAAM_MAX_NB_SEC_QPS;
 664         job_ring->max_nb_sessions = RTE_CAAM_JR_PMD_MAX_NB_SESSIONS;
 665
 666         return job_ring;
 667 cleanup:
 668         caam_jr_dma_free(job_ring->output_ring);
 669         caam_jr_dma_free(job_ring->input_ring);
 670         return NULL;
 671 }
 672
 673
 674 static int
 675 caam_jr_dev_init(const char *name,
 676                  struct rte_vdev_device *vdev,
 677                  struct rte_cryptodev_pmd_init_params *init_params)
 678 {
 679         struct rte_cryptodev *dev;
 680         struct uio_job_ring *job_ring;
 681         char str[RTE_CRYPTODEV_NAME_MAX_LEN];
 682
 683         PMD_INIT_FUNC_TRACE();
 684
 685         /* Validate driver state */
 686         if (g_driver_state == SEC_DRIVER_STATE_IDLE) {
 687                 g_job_rings_max = sec_configure();
 688                 if (!g_job_rings_max) {
 689                         CAAM_JR_ERR("No job ring detected on UIO !!!!");
 690                         return -1;
 691                 }
 692                 /* Update driver state */
 693                 g_driver_state = SEC_DRIVER_STATE_STARTED;
 694         }
 695
 696         if (g_job_rings_no >= g_job_rings_max) {
 697                 CAAM_JR_ERR("No more job rings available max=%d!!!!",
 698                                 g_job_rings_max);
 699                 return -1;
 700         }
 701
 702         job_ring = config_job_ring();
 703         if (job_ring == NULL) {
 704                 CAAM_JR_ERR("failed to create job ring");
 705                 goto init_error;
 706         }
 707
 708         snprintf(str, sizeof(str), "caam_jr%d", job_ring->jr_id);
 709
 710         dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
 711         if (dev == NULL) {
 712                 CAAM_JR_ERR("failed to create cryptodev vdev");
 713                 goto cleanup;
 714         }
 715         /*TODO free it during teardown*/
 716         dev->data->dev_private = init_job_ring(job_ring->register_base_addr,
 717                                                 job_ring->uio_fd);
 718
 719         if (!dev->data->dev_private) {
 720                 CAAM_JR_ERR("Ring memory allocation failed\n");
 721                 goto cleanup2;
 722         }
 723
 724         dev->driver_id = cryptodev_driver_id;
 725         dev->dev_ops = &caam_jr_ops;
 726
 727         /* register rx/tx burst functions for data path */
 728         dev->dequeue_burst = NULL;
 729         dev->enqueue_burst = NULL;
 730         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 731                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
 732                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 733                         RTE_CRYPTODEV_FF_SECURITY |
 734                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
 735                         RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
 736                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
 737                         RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
 738                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
 739
 740         /* For secondary processes, we don't initialise any further as primary
 741          * has already done this work. Only check we don't need a different
 742          * RX function
 743          */
 744         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 745                 CAAM_JR_WARN("Device already init by primary process");
 746                 return 0;
 747         }
 748
 749         RTE_LOG(INFO, PMD, "%s cryptodev init\n", dev->data->name);
 750
 751         return 0;
 752
 753 cleanup2:
 754         caam_jr_dev_uninit(dev);
 755         rte_cryptodev_pmd_release_device(dev);
 756 cleanup:
 757         free_job_ring(job_ring->uio_fd);
 758 init_error:
 759         CAAM_JR_ERR("driver %s: cryptodev_caam_jr_create failed",
 760                         init_params->name);
 761
 762         return -ENXIO;
 763 }
 764
 765 /** Initialise CAAM JR crypto device */
 766 static int
 767 cryptodev_caam_jr_probe(struct rte_vdev_device *vdev)
 768 {
 769         struct rte_cryptodev_pmd_init_params init_params = {
 770                 "",
 771                 sizeof(struct sec_job_ring_t),
 772                 rte_socket_id(),
 773                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
 774         };
 775         const char *name;
 776         const char *input_args;
 777
 778         name = rte_vdev_device_name(vdev);
 779         if (name == NULL)
 780                 return -EINVAL;
 781
 782         input_args = rte_vdev_device_args(vdev);
 783         rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
 784
 785         /* if sec device version is not configured */
 786         if (!rta_get_sec_era()) {
 787                 const struct device_node *caam_node;
 788
 789                 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
 790                         const uint32_t *prop = of_get_property(caam_node,
 791                                         "fsl,sec-era",
 792                                         NULL);
 793                         if (prop) {
 794                                 rta_set_sec_era(
 795                                         INTL_SEC_ERA(cpu_to_caam32(*prop)));
 796                                 break;
 797                         }
 798                 }
 799         }
 800 #ifdef RTE_LIBRTE_PMD_CAAM_JR_BE
 801         if (rta_get_sec_era() > RTA_SEC_ERA_8) {
 802                 RTE_LOG(ERR, PMD,
 803                 "CAAM is compiled in BE mode for device with sec era > 8???\n");
 804                 return -EINVAL;
 805         }
 806 #endif
 807
 808         return caam_jr_dev_init(name, vdev, &init_params);
 809 }
 810
 811 /** Uninitialise CAAM JR crypto device */
 812 static int
 813 cryptodev_caam_jr_remove(struct rte_vdev_device *vdev)
 814 {
 815         struct rte_cryptodev *cryptodev;
 816         const char *name;
 817
 818         name = rte_vdev_device_name(vdev);
 819         if (name == NULL)
 820                 return -EINVAL;
 821
 822         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
 823         if (cryptodev == NULL)
 824                 return -ENODEV;
 825
 826         caam_jr_dev_uninit(cryptodev);
 827
 828         return rte_cryptodev_pmd_destroy(cryptodev);
 829 }
 830
 831 static struct rte_vdev_driver cryptodev_caam_jr_drv = {
 832         .probe = cryptodev_caam_jr_probe,
 833         .remove = cryptodev_caam_jr_remove
 834 };
 835
 836 static struct cryptodev_driver caam_jr_crypto_drv;
 837
 838 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CAAM_JR_PMD, cryptodev_caam_jr_drv);
 839 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CAAM_JR_PMD,
 840         "max_nb_queue_pairs=<int>"
 841         "socket_id=<int>");
 842 RTE_PMD_REGISTER_CRYPTO_DRIVER(caam_jr_crypto_drv, cryptodev_caam_jr_drv.driver,
 843                 cryptodev_driver_id);
 844
 845 RTE_INIT(caam_jr_init_log)
 846 {
 847         caam_jr_logtype = rte_log_register("pmd.crypto.caam");
 848         if (caam_jr_logtype >= 0)
 849                 rte_log_set_level(caam_jr_logtype, RTE_LOG_NOTICE);
 850 }