drivers/crypto/caam_jr/caam_jr.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017-2019 NXP
   3  */
   4
   5 #include <fcntl.h>
   6 #include <unistd.h>
   7 #include <sched.h>
   8 #include <net/if.h>
   9
  10 #include <rte_byteorder.h>
  11 #include <rte_common.h>
  12 #include <rte_cryptodev_pmd.h>
  13 #include <rte_crypto.h>
  14 #include <rte_cryptodev.h>
  15 #include <rte_bus_vdev.h>
  16 #include <rte_malloc.h>
  17 #include <rte_security_driver.h>
  18 #include <rte_hexdump.h>
  19
  20 #include <caam_jr_capabilities.h>
  21 #include <caam_jr_config.h>
  22 #include <caam_jr_hw_specific.h>
  23 #include <caam_jr_pvt.h>
  24 #include <caam_jr_desc.h>
  25 #include <caam_jr_log.h>
  26
  27 /* RTA header files */
  28 #include <desc/common.h>
  29 #include <desc/algo.h>
  30 #include <dpaa_of.h>
  31 #ifdef RTE_LIBRTE_PMD_CAAM_JR_DEBUG
  32 #define CAAM_JR_DBG    1
  33 #else
  34 #define CAAM_JR_DBG     0
  35 #endif
  36 #define CRYPTODEV_NAME_CAAM_JR_PMD      crypto_caam_jr
  37 static uint8_t cryptodev_driver_id;
  38
  39 /* Lists the states possible for the SEC user space driver. */
  40 enum sec_driver_state_e {
  41         SEC_DRIVER_STATE_IDLE,          /* Driver not initialized */
  42         SEC_DRIVER_STATE_STARTED,       /* Driver initialized and can be used*/
  43         SEC_DRIVER_STATE_RELEASE,       /* Driver release is in progress */
  44 };
  45
  46 /* Job rings used for communication with SEC HW */
  47 static struct sec_job_ring_t g_job_rings[MAX_SEC_JOB_RINGS];
  48
  49 /* The current state of SEC user space driver */
  50 static enum sec_driver_state_e g_driver_state = SEC_DRIVER_STATE_IDLE;
  51
  52 /* The number of job rings used by SEC user space driver */
  53 static int g_job_rings_no;
  54 static int g_job_rings_max;
  55
  56 struct sec_outring_entry {
  57         phys_addr_t desc;       /* Pointer to completed descriptor */
  58         uint32_t status;        /* Status for completed descriptor */
  59 } __rte_packed;
  60
  61 /* virtual address conversin when mempool support is available for ctx */
  62 static inline phys_addr_t
  63 caam_jr_vtop_ctx(struct caam_jr_op_ctx *ctx, void *vaddr)
  64 {
  65         return (size_t)vaddr - ctx->vtop_offset;
  66 }
  67
  68 static inline void
  69 caam_jr_op_ending(struct caam_jr_op_ctx *ctx)
  70 {
  71         /* report op status to sym->op and then free the ctx memory  */
  72         rte_mempool_put(ctx->ctx_pool, (void *)ctx);
  73 }
  74
  75 static inline struct caam_jr_op_ctx *
  76 caam_jr_alloc_ctx(struct caam_jr_session *ses)
  77 {
  78         struct caam_jr_op_ctx *ctx;
  79         int ret;
  80
  81         ret = rte_mempool_get(ses->ctx_pool, (void **)(&ctx));
  82         if (!ctx || ret) {
  83                 CAAM_JR_DP_WARN("Alloc sec descriptor failed!");
  84                 return NULL;
  85         }
  86         /*
  87          * Clear SG memory. There are 16 SG entries of 16 Bytes each.
  88          * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
  89          * to clear all the SG entries. caam_jr_alloc_ctx() is called for
  90          * each packet, memset is costlier than dcbz_64().
  91          */
  92         dcbz_64(&ctx->sg[SG_CACHELINE_0]);
  93         dcbz_64(&ctx->sg[SG_CACHELINE_1]);
  94         dcbz_64(&ctx->sg[SG_CACHELINE_2]);
  95         dcbz_64(&ctx->sg[SG_CACHELINE_3]);
  96
  97         ctx->ctx_pool = ses->ctx_pool;
  98         ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx);
  99
 100         return ctx;
 101 }
 102
 103 static
 104 void caam_jr_stats_get(struct rte_cryptodev *dev,
 105                         struct rte_cryptodev_stats *stats)
 106 {
 107         struct caam_jr_qp **qp = (struct caam_jr_qp **)
 108                                         dev->data->queue_pairs;
 109         int i;
 110
 111         PMD_INIT_FUNC_TRACE();
 112         if (stats == NULL) {
 113                 CAAM_JR_ERR("Invalid stats ptr NULL");
 114                 return;
 115         }
 116         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 117                 if (qp[i] == NULL) {
 118                         CAAM_JR_WARN("Uninitialised queue pair");
 119                         continue;
 120                 }
 121
 122                 stats->enqueued_count += qp[i]->tx_pkts;
 123                 stats->dequeued_count += qp[i]->rx_pkts;
 124                 stats->enqueue_err_count += qp[i]->tx_errs;
 125                 stats->dequeue_err_count += qp[i]->rx_errs;
 126                 CAAM_JR_INFO("extra stats:\n\tRX Poll ERR = %" PRIu64
 127                              "\n\tTX Ring Full = %" PRIu64,
 128                              qp[i]->rx_poll_err,
 129                              qp[i]->tx_ring_full);
 130         }
 131 }
 132
 133 static
 134 void caam_jr_stats_reset(struct rte_cryptodev *dev)
 135 {
 136         int i;
 137         struct caam_jr_qp **qp = (struct caam_jr_qp **)
 138                                    (dev->data->queue_pairs);
 139
 140         PMD_INIT_FUNC_TRACE();
 141         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 142                 if (qp[i] == NULL) {
 143                         CAAM_JR_WARN("Uninitialised queue pair");
 144                         continue;
 145                 }
 146                 qp[i]->rx_pkts = 0;
 147                 qp[i]->rx_errs = 0;
 148                 qp[i]->rx_poll_err = 0;
 149                 qp[i]->tx_pkts = 0;
 150                 qp[i]->tx_errs = 0;
 151                 qp[i]->tx_ring_full = 0;
 152         }
 153 }
 154
 155 static inline int
 156 is_cipher_only(struct caam_jr_session *ses)
 157 {
 158         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 159                 (ses->auth_alg == RTE_CRYPTO_AUTH_NULL));
 160 }
 161
 162 static inline int
 163 is_auth_only(struct caam_jr_session *ses)
 164 {
 165         return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) &&
 166                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL));
 167 }
 168
 169 static inline int
 170 is_aead(struct caam_jr_session *ses)
 171 {
 172         return ((ses->cipher_alg == 0) &&
 173                 (ses->auth_alg == 0) &&
 174                 (ses->aead_alg != 0));
 175 }
 176
 177 static inline int
 178 is_auth_cipher(struct caam_jr_session *ses)
 179 {
 180         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 181                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) &&
 182                 (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC));
 183 }
 184
 185 static inline int
 186 is_proto_ipsec(struct caam_jr_session *ses)
 187 {
 188         return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC);
 189 }
 190
 191 static inline int
 192 is_encode(struct caam_jr_session *ses)
 193 {
 194         return ses->dir == DIR_ENC;
 195 }
 196
 197 static inline int
 198 is_decode(struct caam_jr_session *ses)
 199 {
 200         return ses->dir == DIR_DEC;
 201 }
 202
 203 static inline void
 204 caam_auth_alg(struct caam_jr_session *ses, struct alginfo *alginfo_a)
 205 {
 206         switch (ses->auth_alg) {
 207         case RTE_CRYPTO_AUTH_NULL:
 208                 ses->digest_length = 0;
 209                 break;
 210         case RTE_CRYPTO_AUTH_MD5_HMAC:
 211                 alginfo_a->algtype =
 212                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 213                         OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5;
 214                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 215                 break;
 216         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 217                 alginfo_a->algtype =
 218                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 219                         OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1;
 220                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 221                 break;
 222         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 223                 alginfo_a->algtype =
 224                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 225                         OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224;
 226                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 227                 break;
 228         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 229                 alginfo_a->algtype =
 230                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 231                         OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256;
 232                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 233                 break;
 234         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 235                 alginfo_a->algtype =
 236                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 237                         OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384;
 238                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 239                 break;
 240         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 241                 alginfo_a->algtype =
 242                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 243                         OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512;
 244                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 245                 break;
 246         default:
 247                 CAAM_JR_DEBUG("unsupported auth alg %u", ses->auth_alg);
 248         }
 249 }
 250
 251 static inline void
 252 caam_cipher_alg(struct caam_jr_session *ses, struct alginfo *alginfo_c)
 253 {
 254         switch (ses->cipher_alg) {
 255         case RTE_CRYPTO_CIPHER_NULL:
 256                 break;
 257         case RTE_CRYPTO_CIPHER_AES_CBC:
 258                 alginfo_c->algtype =
 259                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 260                         OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES;
 261                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 262                 break;
 263         case RTE_CRYPTO_CIPHER_3DES_CBC:
 264                 alginfo_c->algtype =
 265                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 266                         OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES;
 267                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 268                 break;
 269         case RTE_CRYPTO_CIPHER_AES_CTR:
 270                 alginfo_c->algtype =
 271                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 272                         OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES;
 273                 alginfo_c->algmode = OP_ALG_AAI_CTR;
 274                 break;
 275         default:
 276                 CAAM_JR_DEBUG("unsupported cipher alg %d", ses->cipher_alg);
 277         }
 278 }
 279
 280 static inline void
 281 caam_aead_alg(struct caam_jr_session *ses, struct alginfo *alginfo)
 282 {
 283         switch (ses->aead_alg) {
 284         case RTE_CRYPTO_AEAD_AES_GCM:
 285                 alginfo->algtype = OP_ALG_ALGSEL_AES;
 286                 alginfo->algmode = OP_ALG_AAI_GCM;
 287                 break;
 288         default:
 289                 CAAM_JR_DEBUG("unsupported AEAD alg %d", ses->aead_alg);
 290         }
 291 }
 292
 293 /* prepare command block of the session */
 294 static int
 295 caam_jr_prep_cdb(struct caam_jr_session *ses)
 296 {
 297         struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0};
 298         int32_t shared_desc_len = 0;
 299         struct sec_cdb *cdb;
 300         int err;
 301 #if CAAM_BYTE_ORDER == CORE_BYTE_ORDER
 302         int swap = false;
 303 #else
 304         int swap = true;
 305 #endif
 306
 307         if (ses->cdb)
 308                 caam_jr_dma_free(ses->cdb);
 309
 310         cdb = caam_jr_dma_mem_alloc(L1_CACHE_BYTES, sizeof(struct sec_cdb));
 311         if (!cdb) {
 312                 CAAM_JR_ERR("failed to allocate memory for cdb\n");
 313                 return -1;
 314         }
 315
 316         ses->cdb = cdb;
 317
 318         memset(cdb, 0, sizeof(struct sec_cdb));
 319
 320         if (is_cipher_only(ses)) {
 321                 caam_cipher_alg(ses, &alginfo_c);
 322                 if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 323                         CAAM_JR_ERR("not supported cipher alg");
 324                         rte_free(cdb);
 325                         return -ENOTSUP;
 326                 }
 327
 328                 alginfo_c.key = (size_t)ses->cipher_key.data;
 329                 alginfo_c.keylen = ses->cipher_key.length;
 330                 alginfo_c.key_enc_flags = 0;
 331                 alginfo_c.key_type = RTA_DATA_IMM;
 332
 333                 shared_desc_len = cnstr_shdsc_blkcipher(
 334                                                 cdb->sh_desc, true,
 335                                                 swap, SHR_NEVER, &alginfo_c,
 336                                                 ses->iv.length,
 337                                                 ses->dir);
 338         } else if (is_auth_only(ses)) {
 339                 caam_auth_alg(ses, &alginfo_a);
 340                 if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 341                         CAAM_JR_ERR("not supported auth alg");
 342                         rte_free(cdb);
 343                         return -ENOTSUP;
 344                 }
 345
 346                 alginfo_a.key = (size_t)ses->auth_key.data;
 347                 alginfo_a.keylen = ses->auth_key.length;
 348                 alginfo_a.key_enc_flags = 0;
 349                 alginfo_a.key_type = RTA_DATA_IMM;
 350
 351                 shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true,
 352                                                    swap, SHR_NEVER, &alginfo_a,
 353                                                    !ses->dir,
 354                                                    ses->digest_length);
 355         } else if (is_aead(ses)) {
 356                 caam_aead_alg(ses, &alginfo);
 357                 if (alginfo.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 358                         CAAM_JR_ERR("not supported aead alg");
 359                         rte_free(cdb);
 360                         return -ENOTSUP;
 361                 }
 362                 alginfo.key = (size_t)ses->aead_key.data;
 363                 alginfo.keylen = ses->aead_key.length;
 364                 alginfo.key_enc_flags = 0;
 365                 alginfo.key_type = RTA_DATA_IMM;
 366
 367                 if (ses->dir == DIR_ENC)
 368                         shared_desc_len = cnstr_shdsc_gcm_encap(
 369                                         cdb->sh_desc, true, swap,
 370                                         SHR_NEVER, &alginfo,
 371                                         ses->iv.length,
 372                                         ses->digest_length);
 373                 else
 374                         shared_desc_len = cnstr_shdsc_gcm_decap(
 375                                         cdb->sh_desc, true, swap,
 376                                         SHR_NEVER, &alginfo,
 377                                         ses->iv.length,
 378                                         ses->digest_length);
 379         } else {
 380                 caam_cipher_alg(ses, &alginfo_c);
 381                 if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 382                         CAAM_JR_ERR("not supported cipher alg");
 383                         rte_free(cdb);
 384                         return -ENOTSUP;
 385                 }
 386
 387                 alginfo_c.key = (size_t)ses->cipher_key.data;
 388                 alginfo_c.keylen = ses->cipher_key.length;
 389                 alginfo_c.key_enc_flags = 0;
 390                 alginfo_c.key_type = RTA_DATA_IMM;
 391
 392                 caam_auth_alg(ses, &alginfo_a);
 393                 if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 394                         CAAM_JR_ERR("not supported auth alg");
 395                         rte_free(cdb);
 396                         return -ENOTSUP;
 397                 }
 398
 399                 alginfo_a.key = (size_t)ses->auth_key.data;
 400                 alginfo_a.keylen = ses->auth_key.length;
 401                 alginfo_a.key_enc_flags = 0;
 402                 alginfo_a.key_type = RTA_DATA_IMM;
 403
 404                 cdb->sh_desc[0] = alginfo_c.keylen;
 405                 cdb->sh_desc[1] = alginfo_a.keylen;
 406                 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
 407                                        MIN_JOB_DESC_SIZE,
 408                                        (unsigned int *)cdb->sh_desc,
 409                                        &cdb->sh_desc[2], 2);
 410
 411                 if (err < 0) {
 412                         CAAM_JR_ERR("Crypto: Incorrect key lengths");
 413                         rte_free(cdb);
 414                         return err;
 415                 }
 416                 if (cdb->sh_desc[2] & 1)
 417                         alginfo_c.key_type = RTA_DATA_IMM;
 418                 else {
 419                         alginfo_c.key = (size_t)caam_jr_mem_vtop(
 420                                                 (void *)(size_t)alginfo_c.key);
 421                         alginfo_c.key_type = RTA_DATA_PTR;
 422                 }
 423                 if (cdb->sh_desc[2] & (1<<1))
 424                         alginfo_a.key_type = RTA_DATA_IMM;
 425                 else {
 426                         alginfo_a.key = (size_t)caam_jr_mem_vtop(
 427                                                 (void *)(size_t)alginfo_a.key);
 428                         alginfo_a.key_type = RTA_DATA_PTR;
 429                 }
 430                 cdb->sh_desc[0] = 0;
 431                 cdb->sh_desc[1] = 0;
 432                 cdb->sh_desc[2] = 0;
 433                 if (is_proto_ipsec(ses)) {
 434                         if (ses->dir == DIR_ENC) {
 435                                 shared_desc_len = cnstr_shdsc_ipsec_new_encap(
 436                                                 cdb->sh_desc,
 437                                                 true, swap, SHR_SERIAL,
 438                                                 &ses->encap_pdb,
 439                                                 (uint8_t *)&ses->ip4_hdr,
 440                                                 &alginfo_c, &alginfo_a);
 441                         } else if (ses->dir == DIR_DEC) {
 442                                 shared_desc_len = cnstr_shdsc_ipsec_new_decap(
 443                                                 cdb->sh_desc,
 444                                                 true, swap, SHR_SERIAL,
 445                                                 &ses->decap_pdb,
 446                                                 &alginfo_c, &alginfo_a);
 447                         }
 448                 } else {
 449                         /* Auth_only_len is overwritten in fd for each job */
 450                         shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
 451                                         true, swap, SHR_SERIAL,
 452                                         &alginfo_c, &alginfo_a,
 453                                         ses->iv.length,
 454                                         ses->digest_length, ses->dir);
 455                 }
 456         }
 457
 458         if (shared_desc_len < 0) {
 459                 CAAM_JR_ERR("error in preparing command block");
 460                 return shared_desc_len;
 461         }
 462
 463 #if CAAM_JR_DBG
 464         SEC_DUMP_DESC(cdb->sh_desc);
 465 #endif
 466
 467         cdb->sh_hdr.hi.field.idlen = shared_desc_len;
 468
 469         return 0;
 470 }
 471
 472 /* @brief Poll the HW for already processed jobs in the JR
 473  * and silently discard the available jobs or notify them to UA
 474  * with indicated error code.
 475  *
 476  * @param [in,out]  job_ring        The job ring to poll.
 477  * @param [in]  do_notify           Can be #TRUE or #FALSE. Indicates if
 478  *                                  descriptors are to be discarded
 479  *                                  or notified to UA with given error_code.
 480  * @param [out] notified_descs    Number of notified descriptors. Can be NULL
 481  *                                      if do_notify is #FALSE
 482  */
 483 static void
 484 hw_flush_job_ring(struct sec_job_ring_t *job_ring,
 485                   uint32_t do_notify,
 486                   uint32_t *notified_descs)
 487 {
 488         int32_t jobs_no_to_discard = 0;
 489         int32_t discarded_descs_no = 0;
 490
 491         CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Flushing jr notify desc=[%d]",
 492                 job_ring, job_ring->pidx, job_ring->cidx, do_notify);
 493
 494         jobs_no_to_discard = hw_get_no_finished_jobs(job_ring);
 495
 496         /* Discard all jobs */
 497         CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Discarding %d descs",
 498                   job_ring, job_ring->pidx, job_ring->cidx,
 499                   jobs_no_to_discard);
 500
 501         while (jobs_no_to_discard > discarded_descs_no) {
 502                 discarded_descs_no++;
 503                 /* Now increment the consumer index for the current job ring,
 504                  * AFTER saving job in temporary location!
 505                  * Increment the consumer index for the current job ring
 506                  */
 507                 job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
 508                                          SEC_JOB_RING_SIZE);
 509
 510                 hw_remove_entries(job_ring, 1);
 511         }
 512
 513         if (do_notify == true) {
 514                 ASSERT(notified_descs != NULL);
 515                 *notified_descs = discarded_descs_no;
 516         }
 517 }
 518
 519 /* @brief Poll the HW for already processed jobs in the JR
 520  * and notify the available jobs to UA.
 521  *
 522  * @param [in]  job_ring        The job ring to poll.
 523  * @param [in]  limit           The maximum number of jobs to notify.
 524  *                              If set to negative value, all available jobs are
 525  *                              notified.
 526  *
 527  * @retval >=0 for No of jobs notified to UA.
 528  * @retval -1 for error
 529  */
 530 static int
 531 hw_poll_job_ring(struct sec_job_ring_t *job_ring,
 532                  struct rte_crypto_op **ops, int32_t limit,
 533                  struct caam_jr_qp *jr_qp)
 534 {
 535         int32_t jobs_no_to_notify = 0; /* the number of done jobs to notify*/
 536         int32_t number_of_jobs_available = 0;
 537         int32_t notified_descs_no = 0;
 538         uint32_t sec_error_code = 0;
 539         struct job_descriptor *current_desc;
 540         phys_addr_t current_desc_addr;
 541         phys_addr_t *temp_addr;
 542         struct caam_jr_op_ctx *ctx;
 543
 544         /* TODO check for ops have memory*/
 545         /* check here if any JR error that cannot be written
 546          * in the output status word has occurred
 547          */
 548         if (JR_REG_JRINT_JRE_EXTRACT(GET_JR_REG(JRINT, job_ring))) {
 549                 CAAM_JR_INFO("err received");
 550                 sec_error_code = JR_REG_JRINT_ERR_TYPE_EXTRACT(
 551                                         GET_JR_REG(JRINT, job_ring));
 552                 if (unlikely(sec_error_code)) {
 553                         hw_job_ring_error_print(job_ring, sec_error_code);
 554                         return -1;
 555                 }
 556         }
 557         /* compute the number of jobs available in the job ring based on the
 558          * producer and consumer index values.
 559          */
 560         number_of_jobs_available = hw_get_no_finished_jobs(job_ring);
 561         /* Compute the number of notifications that need to be raised to UA
 562          * If limit > total number of done jobs -> notify all done jobs
 563          * If limit = 0 -> error
 564          * If limit < total number of done jobs -> notify a number
 565          * of done jobs equal with limit
 566          */
 567         jobs_no_to_notify = (limit > number_of_jobs_available) ?
 568                                 number_of_jobs_available : limit;
 569         CAAM_JR_DP_DEBUG(
 570                 "Jr[%p] pi[%d] ci[%d].limit =%d Available=%d.Jobs to notify=%d",
 571                 job_ring, job_ring->pidx, job_ring->cidx,
 572                 limit, number_of_jobs_available, jobs_no_to_notify);
 573
 574         rte_smp_rmb();
 575
 576         while (jobs_no_to_notify > notified_descs_no) {
 577                 static uint64_t false_alarm;
 578                 static uint64_t real_poll;
 579
 580                 /* Get job status here */
 581                 sec_error_code = job_ring->output_ring[job_ring->cidx].status;
 582                 /* Get completed descriptor */
 583                 temp_addr = &(job_ring->output_ring[job_ring->cidx].desc);
 584                 current_desc_addr = (phys_addr_t)sec_read_addr(temp_addr);
 585
 586                 real_poll++;
 587                 /* todo check if it is false alarm no desc present */
 588                 if (!current_desc_addr) {
 589                         false_alarm++;
 590                         printf("false alarm %" PRIu64 "real %" PRIu64
 591                                 " sec_err =0x%x cidx Index =0%d\n",
 592                                 false_alarm, real_poll,
 593                                 sec_error_code, job_ring->cidx);
 594                         rte_panic("CAAM JR descriptor NULL");
 595                         return notified_descs_no;
 596                 }
 597                 current_desc = (struct job_descriptor *)
 598                                 caam_jr_dma_ptov(current_desc_addr);
 599                 /* now increment the consumer index for the current job ring,
 600                  * AFTER saving job in temporary location!
 601                  */
 602                 job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
 603                                  SEC_JOB_RING_SIZE);
 604                 /* Signal that the job has been processed and the slot is free*/
 605                 hw_remove_entries(job_ring, 1);
 606                 /*TODO for multiple ops, packets*/
 607                 ctx = container_of(current_desc, struct caam_jr_op_ctx, jobdes);
 608                 if (unlikely(sec_error_code)) {
 609                         CAAM_JR_ERR("desc at cidx %d generated error 0x%x\n",
 610                                 job_ring->cidx, sec_error_code);
 611                         hw_handle_job_ring_error(job_ring, sec_error_code);
 612                         //todo improve with exact errors
 613                         ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 614                         jr_qp->rx_errs++;
 615                 } else {
 616                         ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 617 #if CAAM_JR_DBG
 618                         if (ctx->op->sym->m_dst) {
 619                                 rte_hexdump(stdout, "PROCESSED",
 620                                 rte_pktmbuf_mtod(ctx->op->sym->m_dst, void *),
 621                                 rte_pktmbuf_data_len(ctx->op->sym->m_dst));
 622                         } else {
 623                                 rte_hexdump(stdout, "PROCESSED",
 624                                 rte_pktmbuf_mtod(ctx->op->sym->m_src, void *),
 625                                 rte_pktmbuf_data_len(ctx->op->sym->m_src));
 626                         }
 627 #endif
 628                 }
 629                 if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 630                         struct ip *ip4_hdr;
 631
 632                         if (ctx->op->sym->m_dst) {
 633                                 /*TODO check for ip header or other*/
 634                                 ip4_hdr = (struct ip *)
 635                                 rte_pktmbuf_mtod(ctx->op->sym->m_dst, char*);
 636                                 ctx->op->sym->m_dst->pkt_len =
 637                                         rte_be_to_cpu_16(ip4_hdr->ip_len);
 638                                 ctx->op->sym->m_dst->data_len =
 639                                         rte_be_to_cpu_16(ip4_hdr->ip_len);
 640                         } else {
 641                                 ip4_hdr = (struct ip *)
 642                                 rte_pktmbuf_mtod(ctx->op->sym->m_src, char*);
 643                                 ctx->op->sym->m_src->pkt_len =
 644                                         rte_be_to_cpu_16(ip4_hdr->ip_len);
 645                                 ctx->op->sym->m_src->data_len =
 646                                         rte_be_to_cpu_16(ip4_hdr->ip_len);
 647                         }
 648                 }
 649                 *ops = ctx->op;
 650                 caam_jr_op_ending(ctx);
 651                 ops++;
 652                 notified_descs_no++;
 653         }
 654         return notified_descs_no;
 655 }
 656
 657 static uint16_t
 658 caam_jr_dequeue_burst(void *qp, struct rte_crypto_op **ops,
 659                        uint16_t nb_ops)
 660 {
 661         struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp;
 662         struct sec_job_ring_t *ring = jr_qp->ring;
 663         int num_rx;
 664         int ret;
 665
 666         CAAM_JR_DP_DEBUG("Jr[%p]Polling. limit[%d]", ring, nb_ops);
 667
 668         /* Poll job ring
 669          * If nb_ops < 0 -> poll JR until no more notifications are available.
 670          * If nb_ops > 0 -> poll JR until limit is reached.
 671          */
 672
 673         /* Run hw poll job ring */
 674         num_rx = hw_poll_job_ring(ring, ops, nb_ops, jr_qp);
 675         if (num_rx < 0) {
 676                 CAAM_JR_ERR("Error polling SEC engine (%d)", num_rx);
 677                 return 0;
 678         }
 679
 680         CAAM_JR_DP_DEBUG("Jr[%p].Jobs notified[%d]. ", ring, num_rx);
 681
 682         if (ring->jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
 683                 if (num_rx < nb_ops) {
 684                         ret = caam_jr_enable_irqs(ring->irq_fd);
 685                         SEC_ASSERT(ret == 0, ret,
 686                         "Failed to enable irqs for job ring %p", ring);
 687                 }
 688         } else if (ring->jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
 689
 690                 /* Always enable IRQ generation when in pure IRQ mode */
 691                 ret = caam_jr_enable_irqs(ring->irq_fd);
 692                 SEC_ASSERT(ret == 0, ret,
 693                         "Failed to enable irqs for job ring %p", ring);
 694         }
 695
 696         jr_qp->rx_pkts += num_rx;
 697
 698         return num_rx;
 699 }
 700
 701 /**
 702  * packet looks like:
 703  *              |<----data_len------->|
 704  *    |ip_header|ah_header|icv|payload|
 705  *              ^
 706  *              |
 707  *         mbuf->pkt.data
 708  */
 709 static inline struct caam_jr_op_ctx *
 710 build_auth_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
 711 {
 712         struct rte_crypto_sym_op *sym = op->sym;
 713         struct rte_mbuf *mbuf = sym->m_src;
 714         struct caam_jr_op_ctx *ctx;
 715         struct sec4_sg_entry *sg;
 716         int     length;
 717         struct sec_cdb *cdb;
 718         uint64_t sdesc_offset;
 719         struct sec_job_descriptor_t *jobdescr;
 720         uint8_t extra_segs;
 721
 722         if (is_decode(ses))
 723                 extra_segs = 2;
 724         else
 725                 extra_segs = 1;
 726
 727         if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) {
 728                 CAAM_JR_DP_ERR("Auth: Max sec segs supported is %d",
 729                                 MAX_SG_ENTRIES);
 730                 return NULL;
 731         }
 732
 733         ctx = caam_jr_alloc_ctx(ses);
 734         if (!ctx)
 735                 return NULL;
 736
 737         ctx->op = op;
 738
 739         cdb = ses->cdb;
 740         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
 741
 742         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
 743
 744         SEC_JD_INIT(jobdescr);
 745         SEC_JD_SET_SD(jobdescr,
 746                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
 747                 cdb->sh_hdr.hi.field.idlen);
 748
 749         /* output */
 750         SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr,
 751                         0, ses->digest_length);
 752
 753         /*input */
 754         sg = &ctx->sg[0];
 755         length = sym->auth.data.length;
 756         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) + sym->auth.data.offset);
 757         sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
 758
 759         /* Successive segs */
 760         mbuf = mbuf->next;
 761         while (mbuf) {
 762                 sg++;
 763                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
 764                 sg->len = cpu_to_caam32(mbuf->data_len);
 765                 mbuf = mbuf->next;
 766         }
 767
 768         if (is_decode(ses)) {
 769                 /* digest verification case */
 770                 sg++;
 771                 /* hash result or digest, save digest first */
 772                 rte_memcpy(ctx->digest, sym->auth.digest.data,
 773                            ses->digest_length);
 774 #if CAAM_JR_DBG
 775                 rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length);
 776 #endif
 777                 sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest));
 778                 sg->len = cpu_to_caam32(ses->digest_length);
 779                 length += ses->digest_length;
 780         } else {
 781                 sg->len -= ses->digest_length;
 782         }
 783
 784         /* last element*/
 785         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
 786
 787         SEC_JD_SET_IN_PTR(jobdescr,
 788                 (uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0, length);
 789         /* enabling sg list */
 790         (jobdescr)->seq_in.command.word  |= 0x01000000;
 791
 792         return ctx;
 793 }
 794
 795 static inline struct caam_jr_op_ctx *
 796 build_auth_only(struct rte_crypto_op *op, struct caam_jr_session *ses)
 797 {
 798         struct rte_crypto_sym_op *sym = op->sym;
 799         struct caam_jr_op_ctx *ctx;
 800         struct sec4_sg_entry *sg;
 801         rte_iova_t start_addr;
 802         struct sec_cdb *cdb;
 803         uint64_t sdesc_offset;
 804         struct sec_job_descriptor_t *jobdescr;
 805
 806         ctx = caam_jr_alloc_ctx(ses);
 807         if (!ctx)
 808                 return NULL;
 809
 810         ctx->op = op;
 811
 812         cdb = ses->cdb;
 813         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
 814
 815         start_addr = rte_pktmbuf_iova(sym->m_src);
 816
 817         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
 818
 819         SEC_JD_INIT(jobdescr);
 820         SEC_JD_SET_SD(jobdescr,
 821                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
 822                 cdb->sh_hdr.hi.field.idlen);
 823
 824         /* output */
 825         SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr,
 826                         0, ses->digest_length);
 827
 828         /*input */
 829         if (is_decode(ses)) {
 830                 sg = &ctx->sg[0];
 831                 SEC_JD_SET_IN_PTR(jobdescr,
 832                         (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0,
 833                         (sym->auth.data.length + ses->digest_length));
 834                 /* enabling sg list */
 835                 (jobdescr)->seq_in.command.word  |= 0x01000000;
 836
 837                 /* hash result or digest, save digest first */
 838                 rte_memcpy(ctx->digest, sym->auth.digest.data,
 839                            ses->digest_length);
 840                 sg->ptr = cpu_to_caam64(start_addr + sym->auth.data.offset);
 841                 sg->len = cpu_to_caam32(sym->auth.data.length);
 842
 843 #if CAAM_JR_DBG
 844                 rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length);
 845 #endif
 846                 /* let's check digest by hw */
 847                 sg++;
 848                 sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest));
 849                 sg->len = cpu_to_caam32(ses->digest_length);
 850                 /* last element*/
 851                 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
 852         } else {
 853                 SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)start_addr,
 854                         sym->auth.data.offset, sym->auth.data.length);
 855         }
 856         return ctx;
 857 }
 858
 859 static inline struct caam_jr_op_ctx *
 860 build_cipher_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
 861 {
 862         struct rte_crypto_sym_op *sym = op->sym;
 863         struct rte_mbuf *mbuf = sym->m_src;
 864         struct caam_jr_op_ctx *ctx;
 865         struct sec4_sg_entry *sg, *in_sg;
 866         int length;
 867         struct sec_cdb *cdb;
 868         uint64_t sdesc_offset;
 869         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 870                         ses->iv.offset);
 871         struct sec_job_descriptor_t *jobdescr;
 872         uint8_t reg_segs;
 873
 874         if (sym->m_dst) {
 875                 mbuf = sym->m_dst;
 876                 reg_segs = mbuf->nb_segs + sym->m_src->nb_segs + 2;
 877         } else {
 878                 mbuf = sym->m_src;
 879                 reg_segs = mbuf->nb_segs * 2 + 2;
 880         }
 881
 882         if (reg_segs > MAX_SG_ENTRIES) {
 883                 CAAM_JR_DP_ERR("Cipher: Max sec segs supported is %d",
 884                                 MAX_SG_ENTRIES);
 885                 return NULL;
 886         }
 887
 888         ctx = caam_jr_alloc_ctx(ses);
 889         if (!ctx)
 890                 return NULL;
 891
 892         ctx->op = op;
 893         cdb = ses->cdb;
 894         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
 895
 896         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
 897
 898         SEC_JD_INIT(jobdescr);
 899         SEC_JD_SET_SD(jobdescr,
 900                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
 901                 cdb->sh_hdr.hi.field.idlen);
 902
 903 #if CAAM_JR_DBG
 904         CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d",
 905                         sym->m_src->data_off, sym->cipher.data.offset,
 906                         sym->cipher.data.length, ses->iv.length);
 907 #endif
 908         /* output */
 909         if (sym->m_dst)
 910                 mbuf = sym->m_dst;
 911         else
 912                 mbuf = sym->m_src;
 913
 914         sg = &ctx->sg[0];
 915         length = sym->cipher.data.length;
 916
 917         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
 918                 + sym->cipher.data.offset);
 919         sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset);
 920
 921         /* Successive segs */
 922         mbuf = mbuf->next;
 923         while (mbuf) {
 924                 sg++;
 925                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
 926                 sg->len = cpu_to_caam32(mbuf->data_len);
 927                 mbuf = mbuf->next;
 928         }
 929         /* last element*/
 930         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
 931
 932         SEC_JD_SET_OUT_PTR(jobdescr,
 933                         (uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0,
 934                         length);
 935         /*enabling sg bit */
 936         (jobdescr)->seq_out.command.word  |= 0x01000000;
 937
 938         /*input */
 939         sg++;
 940         mbuf = sym->m_src;
 941         in_sg = sg;
 942
 943         length = sym->cipher.data.length + ses->iv.length;
 944
 945         /* IV */
 946         sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
 947         sg->len = cpu_to_caam32(ses->iv.length);
 948
 949         /* 1st seg */
 950         sg++;
 951         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
 952                                 + sym->cipher.data.offset);
 953         sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset);
 954
 955         /* Successive segs */
 956         mbuf = mbuf->next;
 957         while (mbuf) {
 958                 sg++;
 959                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
 960                 sg->len = cpu_to_caam32(mbuf->data_len);
 961                 mbuf = mbuf->next;
 962         }
 963         /* last element*/
 964         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
 965
 966
 967         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, in_sg), 0,
 968                                 length);
 969         /*enabling sg bit */
 970         (jobdescr)->seq_in.command.word  |= 0x01000000;
 971
 972         return ctx;
 973 }
 974
 975 static inline struct caam_jr_op_ctx *
 976 build_cipher_only(struct rte_crypto_op *op, struct caam_jr_session *ses)
 977 {
 978         struct rte_crypto_sym_op *sym = op->sym;
 979         struct caam_jr_op_ctx *ctx;
 980         struct sec4_sg_entry *sg;
 981         rte_iova_t src_start_addr, dst_start_addr;
 982         struct sec_cdb *cdb;
 983         uint64_t sdesc_offset;
 984         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 985                         ses->iv.offset);
 986         struct sec_job_descriptor_t *jobdescr;
 987
 988         ctx = caam_jr_alloc_ctx(ses);
 989         if (!ctx)
 990                 return NULL;
 991
 992         ctx->op = op;
 993         cdb = ses->cdb;
 994         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
 995
 996         src_start_addr = rte_pktmbuf_iova(sym->m_src);
 997         if (sym->m_dst)
 998                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
 999         else
1000                 dst_start_addr = src_start_addr;
1001
1002         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1003
1004         SEC_JD_INIT(jobdescr);
1005         SEC_JD_SET_SD(jobdescr,
1006                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1007                 cdb->sh_hdr.hi.field.idlen);
1008
1009 #if CAAM_JR_DBG
1010         CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d",
1011                         sym->m_src->data_off, sym->cipher.data.offset,
1012                         sym->cipher.data.length, ses->iv.length);
1013 #endif
1014         /* output */
1015         SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr,
1016                         sym->cipher.data.offset,
1017                         sym->cipher.data.length + ses->iv.length);
1018
1019         /*input */
1020         sg = &ctx->sg[0];
1021         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0,
1022                                 sym->cipher.data.length + ses->iv.length);
1023         /*enabling sg bit */
1024         (jobdescr)->seq_in.command.word  |= 0x01000000;
1025
1026         sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1027         sg->len = cpu_to_caam32(ses->iv.length);
1028
1029         sg = &ctx->sg[1];
1030         sg->ptr = cpu_to_caam64(src_start_addr + sym->cipher.data.offset);
1031         sg->len = cpu_to_caam32(sym->cipher.data.length);
1032         /* last element*/
1033         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1034
1035         return ctx;
1036 }
1037
1038 /* For decapsulation:
1039  *     Input:
1040  * +----+----------------+--------------------------------+-----+
1041  * | IV | Auth-only data | Authenticated & Encrypted data | ICV |
1042  * +----+----------------+--------------------------------+-----+
1043  *     Output:
1044  * +----+--------------------------+
1045  * | Decrypted & authenticated data |
1046  * +----+--------------------------+
1047  */
1048
1049 static inline struct caam_jr_op_ctx *
1050 build_cipher_auth_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
1051 {
1052         struct rte_crypto_sym_op *sym = op->sym;
1053         struct caam_jr_op_ctx *ctx;
1054         struct sec4_sg_entry *sg, *out_sg, *in_sg;
1055         struct rte_mbuf *mbuf;
1056         uint32_t length = 0;
1057         struct sec_cdb *cdb;
1058         uint64_t sdesc_offset;
1059         uint8_t req_segs;
1060         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1061                         ses->iv.offset);
1062         struct sec_job_descriptor_t *jobdescr;
1063         uint16_t auth_hdr_len = sym->cipher.data.offset -
1064                         sym->auth.data.offset;
1065         uint16_t auth_tail_len = sym->auth.data.length -
1066                         sym->cipher.data.length - auth_hdr_len;
1067         uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
1068
1069         if (sym->m_dst) {
1070                 mbuf = sym->m_dst;
1071                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3;
1072         } else {
1073                 mbuf = sym->m_src;
1074                 req_segs = mbuf->nb_segs * 2 + 3;
1075         }
1076
1077         if (req_segs > MAX_SG_ENTRIES) {
1078                 CAAM_JR_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
1079                                 MAX_SG_ENTRIES);
1080                 return NULL;
1081         }
1082
1083         ctx = caam_jr_alloc_ctx(ses);
1084         if (!ctx)
1085                 return NULL;
1086
1087         ctx->op = op;
1088         cdb = ses->cdb;
1089         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
1090
1091         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1092
1093         SEC_JD_INIT(jobdescr);
1094         SEC_JD_SET_SD(jobdescr,
1095                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1096                 cdb->sh_hdr.hi.field.idlen);
1097
1098         /* output */
1099         if (sym->m_dst)
1100                 mbuf = sym->m_dst;
1101         else
1102                 mbuf = sym->m_src;
1103
1104         out_sg = &ctx->sg[0];
1105         if (is_encode(ses))
1106                 length = sym->auth.data.length + ses->digest_length;
1107         else
1108                 length = sym->auth.data.length;
1109
1110         sg = &ctx->sg[0];
1111
1112         /* 1st seg */
1113         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
1114                 + sym->auth.data.offset);
1115         sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
1116
1117         /* Successive segs */
1118         mbuf = mbuf->next;
1119         while (mbuf) {
1120                 sg++;
1121                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
1122                 sg->len = cpu_to_caam32(mbuf->data_len);
1123                 mbuf = mbuf->next;
1124         }
1125
1126         if (is_encode(ses)) {
1127                 /* set auth output */
1128                 sg++;
1129                 sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr);
1130                 sg->len = cpu_to_caam32(ses->digest_length);
1131         }
1132         /* last element*/
1133         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1134
1135         SEC_JD_SET_OUT_PTR(jobdescr,
1136                            (uint64_t)caam_jr_dma_vtop(out_sg), 0, length);
1137         /* set sg bit */
1138         (jobdescr)->seq_out.command.word  |= 0x01000000;
1139
1140         /* input */
1141         sg++;
1142         mbuf = sym->m_src;
1143         in_sg = sg;
1144         if (is_encode(ses))
1145                 length = ses->iv.length + sym->auth.data.length;
1146         else
1147                 length = ses->iv.length + sym->auth.data.length
1148                                                 + ses->digest_length;
1149
1150         sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1151         sg->len = cpu_to_caam32(ses->iv.length);
1152
1153         sg++;
1154         /* 1st seg */
1155         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
1156                 + sym->auth.data.offset);
1157         sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
1158
1159         /* Successive segs */
1160         mbuf = mbuf->next;
1161         while (mbuf) {
1162                 sg++;
1163                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
1164                 sg->len = cpu_to_caam32(mbuf->data_len);
1165                 mbuf = mbuf->next;
1166         }
1167
1168         if (is_decode(ses)) {
1169                 sg++;
1170                 rte_memcpy(ctx->digest, sym->auth.digest.data,
1171                        ses->digest_length);
1172                 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest));
1173                 sg->len = cpu_to_caam32(ses->digest_length);
1174         }
1175         /* last element*/
1176         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1177
1178         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(in_sg), 0,
1179                                 length);
1180         /* set sg bit */
1181         (jobdescr)->seq_in.command.word  |= 0x01000000;
1182         /* Auth_only_len is set as 0 in descriptor and it is
1183          * overwritten here in the jd which will update
1184          * the DPOVRD reg.
1185          */
1186         if (auth_only_len)
1187                 /* set sg bit */
1188                 (jobdescr)->dpovrd = 0x80000000 | auth_only_len;
1189
1190         return ctx;
1191 }
1192
1193 static inline struct caam_jr_op_ctx *
1194 build_cipher_auth(struct rte_crypto_op *op, struct caam_jr_session *ses)
1195 {
1196         struct rte_crypto_sym_op *sym = op->sym;
1197         struct caam_jr_op_ctx *ctx;
1198         struct sec4_sg_entry *sg;
1199         rte_iova_t src_start_addr, dst_start_addr;
1200         uint32_t length = 0;
1201         struct sec_cdb *cdb;
1202         uint64_t sdesc_offset;
1203         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1204                         ses->iv.offset);
1205         struct sec_job_descriptor_t *jobdescr;
1206         uint16_t auth_hdr_len = sym->cipher.data.offset -
1207                         sym->auth.data.offset;
1208         uint16_t auth_tail_len = sym->auth.data.length -
1209                         sym->cipher.data.length - auth_hdr_len;
1210         uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
1211
1212         src_start_addr = rte_pktmbuf_iova(sym->m_src);
1213         if (sym->m_dst)
1214                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
1215         else
1216                 dst_start_addr = src_start_addr;
1217
1218         ctx = caam_jr_alloc_ctx(ses);
1219         if (!ctx)
1220                 return NULL;
1221
1222         ctx->op = op;
1223         cdb = ses->cdb;
1224         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
1225
1226         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1227
1228         SEC_JD_INIT(jobdescr);
1229         SEC_JD_SET_SD(jobdescr,
1230                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1231                 cdb->sh_hdr.hi.field.idlen);
1232
1233         /* input */
1234         sg = &ctx->sg[0];
1235         if (is_encode(ses)) {
1236                 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1237                 sg->len = cpu_to_caam32(ses->iv.length);
1238                 length += ses->iv.length;
1239
1240                 sg++;
1241                 sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset);
1242                 sg->len = cpu_to_caam32(sym->auth.data.length);
1243                 length += sym->auth.data.length;
1244                 /* last element*/
1245                 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1246         } else {
1247                 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1248                 sg->len = cpu_to_caam32(ses->iv.length);
1249                 length += ses->iv.length;
1250
1251                 sg++;
1252                 sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset);
1253                 sg->len = cpu_to_caam32(sym->auth.data.length);
1254                 length += sym->auth.data.length;
1255
1256                 rte_memcpy(ctx->digest, sym->auth.digest.data,
1257                        ses->digest_length);
1258                 sg++;
1259                 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest));
1260                 sg->len = cpu_to_caam32(ses->digest_length);
1261                 length += ses->digest_length;
1262                 /* last element*/
1263                 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1264         }
1265
1266         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(&ctx->sg[0]), 0,
1267                                 length);
1268         /* set sg bit */
1269         (jobdescr)->seq_in.command.word  |= 0x01000000;
1270
1271         /* output */
1272         sg = &ctx->sg[6];
1273
1274         sg->ptr = cpu_to_caam64(dst_start_addr + sym->cipher.data.offset);
1275         sg->len = cpu_to_caam32(sym->cipher.data.length);
1276         length = sym->cipher.data.length;
1277
1278         if (is_encode(ses)) {
1279                 /* set auth output */
1280                 sg++;
1281                 sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr);
1282                 sg->len = cpu_to_caam32(ses->digest_length);
1283                 length += ses->digest_length;
1284         }
1285         /* last element*/
1286         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1287
1288         SEC_JD_SET_OUT_PTR(jobdescr,
1289                            (uint64_t)caam_jr_dma_vtop(&ctx->sg[6]), 0, length);
1290         /* set sg bit */
1291         (jobdescr)->seq_out.command.word  |= 0x01000000;
1292
1293         /* Auth_only_len is set as 0 in descriptor and it is
1294          * overwritten here in the jd which will update
1295          * the DPOVRD reg.
1296          */
1297         if (auth_only_len)
1298                 /* set sg bit */
1299                 (jobdescr)->dpovrd = 0x80000000 | auth_only_len;
1300
1301         return ctx;
1302 }
1303
1304 static inline struct caam_jr_op_ctx *
1305 build_proto(struct rte_crypto_op *op, struct caam_jr_session *ses)
1306 {
1307         struct rte_crypto_sym_op *sym = op->sym;
1308         struct caam_jr_op_ctx *ctx = NULL;
1309         phys_addr_t src_start_addr, dst_start_addr;
1310         struct sec_cdb *cdb;
1311         uint64_t sdesc_offset;
1312         struct sec_job_descriptor_t *jobdescr;
1313
1314         ctx = caam_jr_alloc_ctx(ses);
1315         if (!ctx)
1316                 return NULL;
1317         ctx->op = op;
1318
1319         src_start_addr = rte_pktmbuf_iova(sym->m_src);
1320         if (sym->m_dst)
1321                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
1322         else
1323                 dst_start_addr = src_start_addr;
1324
1325         cdb = ses->cdb;
1326         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
1327
1328         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1329
1330         SEC_JD_INIT(jobdescr);
1331         SEC_JD_SET_SD(jobdescr,
1332                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1333                         cdb->sh_hdr.hi.field.idlen);
1334
1335         /* output */
1336         SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr, 0,
1337                         sym->m_src->buf_len - sym->m_src->data_off);
1338         /* input */
1339         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)src_start_addr, 0,
1340                         sym->m_src->pkt_len);
1341         sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
1342
1343         return ctx;
1344 }
1345
1346 static int
1347 caam_jr_enqueue_op(struct rte_crypto_op *op, struct caam_jr_qp *qp)
1348 {
1349         struct sec_job_ring_t *ring = qp->ring;
1350         struct caam_jr_session *ses;
1351         struct caam_jr_op_ctx *ctx = NULL;
1352         struct sec_job_descriptor_t *jobdescr __rte_unused;
1353 #if CAAM_JR_DBG
1354         int i;
1355 #endif
1356
1357         switch (op->sess_type) {
1358         case RTE_CRYPTO_OP_WITH_SESSION:
1359                 ses = (struct caam_jr_session *)
1360                 get_sym_session_private_data(op->sym->session,
1361                                         cryptodev_driver_id);
1362                 break;
1363         case RTE_CRYPTO_OP_SECURITY_SESSION:
1364                 ses = (struct caam_jr_session *)
1365                         get_sec_session_private_data(
1366                                         op->sym->sec_session);
1367                 break;
1368         default:
1369                 CAAM_JR_DP_ERR("sessionless crypto op not supported");
1370                 qp->tx_errs++;
1371                 return -1;
1372         }
1373
1374         if (unlikely(!ses->qp || ses->qp != qp)) {
1375                 CAAM_JR_DP_DEBUG("Old:sess->qp=%p New qp = %p\n", ses->qp, qp);
1376                 ses->qp = qp;
1377                 caam_jr_prep_cdb(ses);
1378         }
1379
1380         if (rte_pktmbuf_is_contiguous(op->sym->m_src)) {
1381                 if (is_auth_cipher(ses))
1382                         ctx = build_cipher_auth(op, ses);
1383                 else if (is_aead(ses))
1384                         goto err1;
1385                 else if (is_auth_only(ses))
1386                         ctx = build_auth_only(op, ses);
1387                 else if (is_cipher_only(ses))
1388                         ctx = build_cipher_only(op, ses);
1389                 else if (is_proto_ipsec(ses))
1390                         ctx = build_proto(op, ses);
1391         } else {
1392                 if (is_auth_cipher(ses))
1393                         ctx = build_cipher_auth_sg(op, ses);
1394                 else if (is_aead(ses))
1395                         goto err1;
1396                 else if (is_auth_only(ses))
1397                         ctx = build_auth_only_sg(op, ses);
1398                 else if (is_cipher_only(ses))
1399                         ctx = build_cipher_only_sg(op, ses);
1400         }
1401 err1:
1402         if (unlikely(!ctx)) {
1403                 qp->tx_errs++;
1404                 CAAM_JR_ERR("not supported sec op");
1405                 return -1;
1406         }
1407 #if CAAM_JR_DBG
1408         if (is_decode(ses))
1409                 rte_hexdump(stdout, "DECODE",
1410                         rte_pktmbuf_mtod(op->sym->m_src, void *),
1411                         rte_pktmbuf_data_len(op->sym->m_src));
1412         else
1413                 rte_hexdump(stdout, "ENCODE",
1414                         rte_pktmbuf_mtod(op->sym->m_src, void *),
1415                         rte_pktmbuf_data_len(op->sym->m_src));
1416
1417         printf("\n JD before conversion\n");
1418         for (i = 0; i < 12; i++)
1419                 printf("\n 0x%08x", ctx->jobdes.desc[i]);
1420 #endif
1421
1422         CAAM_JR_DP_DEBUG("Jr[%p] pi[%d] ci[%d].Before sending desc",
1423                       ring, ring->pidx, ring->cidx);
1424
1425         /* todo - do we want to retry */
1426         if (SEC_JOB_RING_IS_FULL(ring->pidx, ring->cidx,
1427                          SEC_JOB_RING_SIZE, SEC_JOB_RING_SIZE)) {
1428                 CAAM_JR_DP_DEBUG("Ring FULL Jr[%p] pi[%d] ci[%d].Size = %d",
1429                               ring, ring->pidx, ring->cidx, SEC_JOB_RING_SIZE);
1430                 caam_jr_op_ending(ctx);
1431                 qp->tx_ring_full++;
1432                 return -EBUSY;
1433         }
1434
1435 #if CORE_BYTE_ORDER != CAAM_BYTE_ORDER
1436         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1437
1438         jobdescr->deschdr.command.word =
1439                 cpu_to_caam32(jobdescr->deschdr.command.word);
1440         jobdescr->sd_ptr = cpu_to_caam64(jobdescr->sd_ptr);
1441         jobdescr->seq_out.command.word =
1442                 cpu_to_caam32(jobdescr->seq_out.command.word);
1443         jobdescr->seq_out_ptr = cpu_to_caam64(jobdescr->seq_out_ptr);
1444         jobdescr->out_ext_length = cpu_to_caam32(jobdescr->out_ext_length);
1445         jobdescr->seq_in.command.word =
1446                 cpu_to_caam32(jobdescr->seq_in.command.word);
1447         jobdescr->seq_in_ptr = cpu_to_caam64(jobdescr->seq_in_ptr);
1448         jobdescr->in_ext_length = cpu_to_caam32(jobdescr->in_ext_length);
1449         jobdescr->load_dpovrd.command.word =
1450                 cpu_to_caam32(jobdescr->load_dpovrd.command.word);
1451         jobdescr->dpovrd = cpu_to_caam32(jobdescr->dpovrd);
1452 #endif
1453
1454         /* Set ptr in input ring to current descriptor  */
1455         sec_write_addr(&ring->input_ring[ring->pidx],
1456                         (phys_addr_t)caam_jr_vtop_ctx(ctx, ctx->jobdes.desc));
1457         rte_smp_wmb();
1458
1459         /* Notify HW that a new job is enqueued */
1460         hw_enqueue_desc_on_job_ring(ring);
1461
1462         /* increment the producer index for the current job ring */
1463         ring->pidx = SEC_CIRCULAR_COUNTER(ring->pidx, SEC_JOB_RING_SIZE);
1464
1465         return 0;
1466 }
1467
1468 static uint16_t
1469 caam_jr_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1470                        uint16_t nb_ops)
1471 {
1472         /* Function to transmit the frames to given device and queuepair */
1473         uint32_t loop;
1474         int32_t ret;
1475         struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp;
1476         uint16_t num_tx = 0;
1477         /*Prepare each packet which is to be sent*/
1478         for (loop = 0; loop < nb_ops; loop++) {
1479                 ret = caam_jr_enqueue_op(ops[loop], jr_qp);
1480                 if (!ret)
1481                         num_tx++;
1482         }
1483
1484         jr_qp->tx_pkts += num_tx;
1485
1486         return num_tx;
1487 }
1488
1489 /* Release queue pair */
1490 static int
1491 caam_jr_queue_pair_release(struct rte_cryptodev *dev,
1492                            uint16_t qp_id)
1493 {
1494         struct sec_job_ring_t *internals;
1495         struct caam_jr_qp *qp = NULL;
1496
1497         PMD_INIT_FUNC_TRACE();
1498         CAAM_JR_DEBUG("dev =%p, queue =%d", dev, qp_id);
1499
1500         internals = dev->data->dev_private;
1501         if (qp_id >= internals->max_nb_queue_pairs) {
1502                 CAAM_JR_ERR("Max supported qpid %d",
1503                              internals->max_nb_queue_pairs);
1504                 return -EINVAL;
1505         }
1506
1507         qp = &internals->qps[qp_id];
1508         qp->ring = NULL;
1509         dev->data->queue_pairs[qp_id] = NULL;
1510
1511         return 0;
1512 }
1513
1514 /* Setup a queue pair */
1515 static int
1516 caam_jr_queue_pair_setup(
1517                 struct rte_cryptodev *dev, uint16_t qp_id,
1518                 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
1519                 __rte_unused int socket_id)
1520 {
1521         struct sec_job_ring_t *internals;
1522         struct caam_jr_qp *qp = NULL;
1523
1524         PMD_INIT_FUNC_TRACE();
1525         CAAM_JR_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);
1526
1527         internals = dev->data->dev_private;
1528         if (qp_id >= internals->max_nb_queue_pairs) {
1529                 CAAM_JR_ERR("Max supported qpid %d",
1530                              internals->max_nb_queue_pairs);
1531                 return -EINVAL;
1532         }
1533
1534         qp = &internals->qps[qp_id];
1535         qp->ring = internals;
1536         dev->data->queue_pairs[qp_id] = qp;
1537
1538         return 0;
1539 }
1540
1541 /* Returns the size of the aesni gcm session structure */
1542 static unsigned int
1543 caam_jr_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
1544 {
1545         PMD_INIT_FUNC_TRACE();
1546
1547         return sizeof(struct caam_jr_session);
1548 }
1549
1550 static int
1551 caam_jr_cipher_init(struct rte_cryptodev *dev __rte_unused,
1552                     struct rte_crypto_sym_xform *xform,
1553                     struct caam_jr_session *session)
1554 {
1555         session->cipher_alg = xform->cipher.algo;
1556         session->iv.length = xform->cipher.iv.length;
1557         session->iv.offset = xform->cipher.iv.offset;
1558         session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
1559                                                RTE_CACHE_LINE_SIZE);
1560         if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
1561                 CAAM_JR_ERR("No Memory for cipher key\n");
1562                 return -ENOMEM;
1563         }
1564         session->cipher_key.length = xform->cipher.key.length;
1565
1566         memcpy(session->cipher_key.data, xform->cipher.key.data,
1567                xform->cipher.key.length);
1568         session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
1569                         DIR_ENC : DIR_DEC;
1570
1571         return 0;
1572 }
1573
1574 static int
1575 caam_jr_auth_init(struct rte_cryptodev *dev __rte_unused,
1576                   struct rte_crypto_sym_xform *xform,
1577                   struct caam_jr_session *session)
1578 {
1579         session->auth_alg = xform->auth.algo;
1580         session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
1581                                              RTE_CACHE_LINE_SIZE);
1582         if (session->auth_key.data == NULL && xform->auth.key.length > 0) {
1583                 CAAM_JR_ERR("No Memory for auth key\n");
1584                 return -ENOMEM;
1585         }
1586         session->auth_key.length = xform->auth.key.length;
1587         session->digest_length = xform->auth.digest_length;
1588
1589         memcpy(session->auth_key.data, xform->auth.key.data,
1590                xform->auth.key.length);
1591         session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
1592                         DIR_ENC : DIR_DEC;
1593
1594         return 0;
1595 }
1596
1597 static int
1598 caam_jr_aead_init(struct rte_cryptodev *dev __rte_unused,
1599                   struct rte_crypto_sym_xform *xform,
1600                   struct caam_jr_session *session)
1601 {
1602         session->aead_alg = xform->aead.algo;
1603         session->iv.length = xform->aead.iv.length;
1604         session->iv.offset = xform->aead.iv.offset;
1605         session->auth_only_len = xform->aead.aad_length;
1606         session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length,
1607                                              RTE_CACHE_LINE_SIZE);
1608         if (session->aead_key.data == NULL && xform->aead.key.length > 0) {
1609                 CAAM_JR_ERR("No Memory for aead key\n");
1610                 return -ENOMEM;
1611         }
1612         session->aead_key.length = xform->aead.key.length;
1613         session->digest_length = xform->aead.digest_length;
1614
1615         memcpy(session->aead_key.data, xform->aead.key.data,
1616                xform->aead.key.length);
1617         session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
1618                         DIR_ENC : DIR_DEC;
1619
1620         return 0;
1621 }
1622
1623 static int
1624 caam_jr_set_session_parameters(struct rte_cryptodev *dev,
1625                                struct rte_crypto_sym_xform *xform, void *sess)
1626 {
1627         struct sec_job_ring_t *internals = dev->data->dev_private;
1628         struct caam_jr_session *session = sess;
1629
1630         PMD_INIT_FUNC_TRACE();
1631
1632         if (unlikely(sess == NULL)) {
1633                 CAAM_JR_ERR("invalid session struct");
1634                 return -EINVAL;
1635         }
1636
1637         /* Default IV length = 0 */
1638         session->iv.length = 0;
1639
1640         /* Cipher Only */
1641         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
1642                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1643                 caam_jr_cipher_init(dev, xform, session);
1644
1645         /* Authentication Only */
1646         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1647                    xform->next == NULL) {
1648                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
1649                 caam_jr_auth_init(dev, xform, session);
1650
1651         /* Cipher then Authenticate */
1652         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1653                    xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
1654                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1655                         caam_jr_cipher_init(dev, xform, session);
1656                         caam_jr_auth_init(dev, xform->next, session);
1657                 } else {
1658                         CAAM_JR_ERR("Not supported: Auth then Cipher");
1659                         goto err1;
1660                 }
1661
1662         /* Authenticate then Cipher */
1663         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1664                    xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
1665                 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
1666                         caam_jr_auth_init(dev, xform, session);
1667                         caam_jr_cipher_init(dev, xform->next, session);
1668                 } else {
1669                         CAAM_JR_ERR("Not supported: Auth then Cipher");
1670                         goto err1;
1671                 }
1672
1673         /* AEAD operation for AES-GCM kind of Algorithms */
1674         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
1675                    xform->next == NULL) {
1676                 caam_jr_aead_init(dev, xform, session);
1677
1678         } else {
1679                 CAAM_JR_ERR("Invalid crypto type");
1680                 return -EINVAL;
1681         }
1682         session->ctx_pool = internals->ctx_pool;
1683
1684         return 0;
1685
1686 err1:
1687         rte_free(session->cipher_key.data);
1688         rte_free(session->auth_key.data);
1689         memset(session, 0, sizeof(struct caam_jr_session));
1690
1691         return -EINVAL;
1692 }
1693
1694 static int
1695 caam_jr_sym_session_configure(struct rte_cryptodev *dev,
1696                               struct rte_crypto_sym_xform *xform,
1697                               struct rte_cryptodev_sym_session *sess,
1698                               struct rte_mempool *mempool)
1699 {
1700         void *sess_private_data;
1701         int ret;
1702
1703         PMD_INIT_FUNC_TRACE();
1704
1705         if (rte_mempool_get(mempool, &sess_private_data)) {
1706                 CAAM_JR_ERR("Couldn't get object from session mempool");
1707                 return -ENOMEM;
1708         }
1709
1710         memset(sess_private_data, 0, sizeof(struct caam_jr_session));
1711         ret = caam_jr_set_session_parameters(dev, xform, sess_private_data);
1712         if (ret != 0) {
1713                 CAAM_JR_ERR("failed to configure session parameters");
1714                 /* Return session to mempool */
1715                 rte_mempool_put(mempool, sess_private_data);
1716                 return ret;
1717         }
1718
1719         set_sym_session_private_data(sess, dev->driver_id, sess_private_data);
1720
1721         return 0;
1722 }
1723
1724 /* Clear the memory of session so it doesn't leave key material behind */
1725 static void
1726 caam_jr_sym_session_clear(struct rte_cryptodev *dev,
1727                 struct rte_cryptodev_sym_session *sess)
1728 {
1729         uint8_t index = dev->driver_id;
1730         void *sess_priv = get_sym_session_private_data(sess, index);
1731         struct caam_jr_session *s = (struct caam_jr_session *)sess_priv;
1732
1733         PMD_INIT_FUNC_TRACE();
1734
1735         if (sess_priv) {
1736                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1737
1738                 rte_free(s->cipher_key.data);
1739                 rte_free(s->auth_key.data);
1740                 memset(s, 0, sizeof(struct caam_jr_session));
1741                 set_sym_session_private_data(sess, index, NULL);
1742                 rte_mempool_put(sess_mp, sess_priv);
1743         }
1744 }
1745
1746 static int
1747 caam_jr_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
1748                           struct rte_security_session_conf *conf,
1749                           void *sess)
1750 {
1751         struct sec_job_ring_t *internals = dev->data->dev_private;
1752         struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
1753         struct rte_crypto_auth_xform *auth_xform;
1754         struct rte_crypto_cipher_xform *cipher_xform;
1755         struct caam_jr_session *session = (struct caam_jr_session *)sess;
1756
1757         PMD_INIT_FUNC_TRACE();
1758
1759         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
1760                 cipher_xform = &conf->crypto_xform->cipher;
1761                 auth_xform = &conf->crypto_xform->next->auth;
1762         } else {
1763                 auth_xform = &conf->crypto_xform->auth;
1764                 cipher_xform = &conf->crypto_xform->next->cipher;
1765         }
1766         session->proto_alg = conf->protocol;
1767         session->cipher_key.data = rte_zmalloc(NULL,
1768                                                cipher_xform->key.length,
1769                                                RTE_CACHE_LINE_SIZE);
1770         if (session->cipher_key.data == NULL &&
1771                         cipher_xform->key.length > 0) {
1772                 CAAM_JR_ERR("No Memory for cipher key\n");
1773                 return -ENOMEM;
1774         }
1775
1776         session->cipher_key.length = cipher_xform->key.length;
1777         session->auth_key.data = rte_zmalloc(NULL,
1778                                         auth_xform->key.length,
1779                                         RTE_CACHE_LINE_SIZE);
1780         if (session->auth_key.data == NULL &&
1781                         auth_xform->key.length > 0) {
1782                 CAAM_JR_ERR("No Memory for auth key\n");
1783                 rte_free(session->cipher_key.data);
1784                 return -ENOMEM;
1785         }
1786         session->auth_key.length = auth_xform->key.length;
1787         memcpy(session->cipher_key.data, cipher_xform->key.data,
1788                         cipher_xform->key.length);
1789         memcpy(session->auth_key.data, auth_xform->key.data,
1790                         auth_xform->key.length);
1791
1792         switch (auth_xform->algo) {
1793         case RTE_CRYPTO_AUTH_SHA1_HMAC:
1794                 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
1795                 break;
1796         case RTE_CRYPTO_AUTH_MD5_HMAC:
1797                 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
1798                 break;
1799         case RTE_CRYPTO_AUTH_SHA256_HMAC:
1800                 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
1801                 break;
1802         case RTE_CRYPTO_AUTH_SHA384_HMAC:
1803                 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
1804                 break;
1805         case RTE_CRYPTO_AUTH_SHA512_HMAC:
1806                 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
1807                 break;
1808         case RTE_CRYPTO_AUTH_AES_CMAC:
1809                 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
1810                 break;
1811         case RTE_CRYPTO_AUTH_NULL:
1812                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1813                 break;
1814         case RTE_CRYPTO_AUTH_SHA224_HMAC:
1815         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
1816         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
1817         case RTE_CRYPTO_AUTH_SHA1:
1818         case RTE_CRYPTO_AUTH_SHA256:
1819         case RTE_CRYPTO_AUTH_SHA512:
1820         case RTE_CRYPTO_AUTH_SHA224:
1821         case RTE_CRYPTO_AUTH_SHA384:
1822         case RTE_CRYPTO_AUTH_MD5:
1823         case RTE_CRYPTO_AUTH_AES_GMAC:
1824         case RTE_CRYPTO_AUTH_KASUMI_F9:
1825         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
1826         case RTE_CRYPTO_AUTH_ZUC_EIA3:
1827                 CAAM_JR_ERR("Crypto: Unsupported auth alg %u\n",
1828                         auth_xform->algo);
1829                 goto out;
1830         default:
1831                 CAAM_JR_ERR("Crypto: Undefined Auth specified %u\n",
1832                         auth_xform->algo);
1833                 goto out;
1834         }
1835
1836         switch (cipher_xform->algo) {
1837         case RTE_CRYPTO_CIPHER_AES_CBC:
1838                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
1839                 break;
1840         case RTE_CRYPTO_CIPHER_3DES_CBC:
1841                 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
1842                 break;
1843         case RTE_CRYPTO_CIPHER_AES_CTR:
1844                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
1845                 break;
1846         case RTE_CRYPTO_CIPHER_NULL:
1847         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
1848         case RTE_CRYPTO_CIPHER_3DES_ECB:
1849         case RTE_CRYPTO_CIPHER_AES_ECB:
1850         case RTE_CRYPTO_CIPHER_KASUMI_F8:
1851                 CAAM_JR_ERR("Crypto: Unsupported Cipher alg %u\n",
1852                         cipher_xform->algo);
1853                 goto out;
1854         default:
1855                 CAAM_JR_ERR("Crypto: Undefined Cipher specified %u\n",
1856                         cipher_xform->algo);
1857                 goto out;
1858         }
1859
1860         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
1861                 memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) +
1862                                 sizeof(session->ip4_hdr));
1863                 session->ip4_hdr.ip_v = IPVERSION;
1864                 session->ip4_hdr.ip_hl = 5;
1865                 session->ip4_hdr.ip_len = rte_cpu_to_be_16(
1866                                                 sizeof(session->ip4_hdr));
1867                 session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
1868                 session->ip4_hdr.ip_id = 0;
1869                 session->ip4_hdr.ip_off = 0;
1870                 session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
1871                 session->ip4_hdr.ip_p = (ipsec_xform->proto ==
1872                                 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP
1873                                 : IPPROTO_AH;
1874                 session->ip4_hdr.ip_sum = 0;
1875                 session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
1876                 session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
1877                 session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
1878                                                 (void *)&session->ip4_hdr,
1879                                                 sizeof(struct ip));
1880
1881                 session->encap_pdb.options =
1882                         (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
1883                         PDBOPTS_ESP_OIHI_PDB_INL |
1884                         PDBOPTS_ESP_IVSRC |
1885                         PDBHMO_ESP_ENCAP_DTTL;
1886                 if (ipsec_xform->options.esn)
1887                         session->encap_pdb.options |= PDBOPTS_ESP_ESN;
1888                 session->encap_pdb.spi = ipsec_xform->spi;
1889                 session->encap_pdb.ip_hdr_len = sizeof(struct ip);
1890
1891                 session->dir = DIR_ENC;
1892         } else if (ipsec_xform->direction ==
1893                         RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
1894                 memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
1895                 session->decap_pdb.options = sizeof(struct ip) << 16;
1896                 if (ipsec_xform->options.esn)
1897                         session->decap_pdb.options |= PDBOPTS_ESP_ESN;
1898                 session->dir = DIR_DEC;
1899         } else
1900                 goto out;
1901         session->ctx_pool = internals->ctx_pool;
1902
1903         return 0;
1904 out:
1905         rte_free(session->auth_key.data);
1906         rte_free(session->cipher_key.data);
1907         memset(session, 0, sizeof(struct caam_jr_session));
1908         return -1;
1909 }
1910
1911 static int
1912 caam_jr_security_session_create(void *dev,
1913                                 struct rte_security_session_conf *conf,
1914                                 struct rte_security_session *sess,
1915                                 struct rte_mempool *mempool)
1916 {
1917         void *sess_private_data;
1918         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
1919         int ret;
1920
1921         if (rte_mempool_get(mempool, &sess_private_data)) {
1922                 CAAM_JR_ERR("Couldn't get object from session mempool");
1923                 return -ENOMEM;
1924         }
1925
1926         switch (conf->protocol) {
1927         case RTE_SECURITY_PROTOCOL_IPSEC:
1928                 ret = caam_jr_set_ipsec_session(cdev, conf,
1929                                 sess_private_data);
1930                 break;
1931         case RTE_SECURITY_PROTOCOL_MACSEC:
1932                 return -ENOTSUP;
1933         default:
1934                 return -EINVAL;
1935         }
1936         if (ret != 0) {
1937                 CAAM_JR_ERR("failed to configure session parameters");
1938                 /* Return session to mempool */
1939                 rte_mempool_put(mempool, sess_private_data);
1940                 return ret;
1941         }
1942
1943         set_sec_session_private_data(sess, sess_private_data);
1944
1945         return ret;
1946 }
1947
1948 /* Clear the memory of session so it doesn't leave key material behind */
1949 static int
1950 caam_jr_security_session_destroy(void *dev __rte_unused,
1951                                  struct rte_security_session *sess)
1952 {
1953         PMD_INIT_FUNC_TRACE();
1954         void *sess_priv = get_sec_session_private_data(sess);
1955
1956         struct caam_jr_session *s = (struct caam_jr_session *)sess_priv;
1957
1958         if (sess_priv) {
1959                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1960
1961                 rte_free(s->cipher_key.data);
1962                 rte_free(s->auth_key.data);
1963                 memset(sess, 0, sizeof(struct caam_jr_session));
1964                 set_sec_session_private_data(sess, NULL);
1965                 rte_mempool_put(sess_mp, sess_priv);
1966         }
1967         return 0;
1968 }
1969
1970
1971 static int
1972 caam_jr_dev_configure(struct rte_cryptodev *dev,
1973                        struct rte_cryptodev_config *config __rte_unused)
1974 {
1975         char str[20];
1976         struct sec_job_ring_t *internals;
1977
1978         PMD_INIT_FUNC_TRACE();
1979
1980         internals = dev->data->dev_private;
1981         snprintf(str, sizeof(str), "ctx_pool_%d", dev->data->dev_id);
1982         if (!internals->ctx_pool) {
1983                 internals->ctx_pool = rte_mempool_create((const char *)str,
1984                                                 CTX_POOL_NUM_BUFS,
1985                                                 sizeof(struct caam_jr_op_ctx),
1986                                                 CTX_POOL_CACHE_SIZE, 0,
1987                                                 NULL, NULL, NULL, NULL,
1988                                                 SOCKET_ID_ANY, 0);
1989                 if (!internals->ctx_pool) {
1990                         CAAM_JR_ERR("%s create failed\n", str);
1991                         return -ENOMEM;
1992                 }
1993         } else
1994                 CAAM_JR_INFO("mempool already created for dev_id : %d",
1995                                 dev->data->dev_id);
1996
1997         return 0;
1998 }
1999
2000 static int
2001 caam_jr_dev_start(struct rte_cryptodev *dev __rte_unused)
2002 {
2003         PMD_INIT_FUNC_TRACE();
2004         return 0;
2005 }
2006
2007 static void
2008 caam_jr_dev_stop(struct rte_cryptodev *dev __rte_unused)
2009 {
2010         PMD_INIT_FUNC_TRACE();
2011 }
2012
2013 static int
2014 caam_jr_dev_close(struct rte_cryptodev *dev)
2015 {
2016         struct sec_job_ring_t *internals;
2017
2018         PMD_INIT_FUNC_TRACE();
2019
2020         if (dev == NULL)
2021                 return -ENOMEM;
2022
2023         internals = dev->data->dev_private;
2024         rte_mempool_free(internals->ctx_pool);
2025         internals->ctx_pool = NULL;
2026
2027         return 0;
2028 }
2029
2030 static void
2031 caam_jr_dev_infos_get(struct rte_cryptodev *dev,
2032                        struct rte_cryptodev_info *info)
2033 {
2034         struct sec_job_ring_t *internals = dev->data->dev_private;
2035
2036         PMD_INIT_FUNC_TRACE();
2037         if (info != NULL) {
2038                 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
2039                 info->feature_flags = dev->feature_flags;
2040                 info->capabilities = caam_jr_get_cryptodev_capabilities();
2041                 info->sym.max_nb_sessions = internals->max_nb_sessions;
2042                 info->driver_id = cryptodev_driver_id;
2043         }
2044 }
2045
2046 static struct rte_cryptodev_ops caam_jr_ops = {
2047         .dev_configure        = caam_jr_dev_configure,
2048         .dev_start            = caam_jr_dev_start,
2049         .dev_stop             = caam_jr_dev_stop,
2050         .dev_close            = caam_jr_dev_close,
2051         .dev_infos_get        = caam_jr_dev_infos_get,
2052         .stats_get            = caam_jr_stats_get,
2053         .stats_reset          = caam_jr_stats_reset,
2054         .queue_pair_setup     = caam_jr_queue_pair_setup,
2055         .queue_pair_release   = caam_jr_queue_pair_release,
2056         .sym_session_get_size = caam_jr_sym_session_get_size,
2057         .sym_session_configure = caam_jr_sym_session_configure,
2058         .sym_session_clear    = caam_jr_sym_session_clear
2059 };
2060
2061 static struct rte_security_ops caam_jr_security_ops = {
2062         .session_create = caam_jr_security_session_create,
2063         .session_update = NULL,
2064         .session_stats_get = NULL,
2065         .session_destroy = caam_jr_security_session_destroy,
2066         .set_pkt_metadata = NULL,
2067         .capabilities_get = caam_jr_get_security_capabilities
2068 };
2069
2070 /* @brief Flush job rings of any processed descs.
2071  * The processed descs are silently dropped,
2072  * WITHOUT being notified to UA.
2073  */
2074 static void
2075 close_job_ring(struct sec_job_ring_t *job_ring)
2076 {
2077         if (job_ring->irq_fd != -1) {
2078                 /* Producer index is frozen. If consumer index is not equal
2079                  * with producer index, then we have descs to flush.
2080                  */
2081                 while (job_ring->pidx != job_ring->cidx)
2082                         hw_flush_job_ring(job_ring, false, NULL);
2083
2084                 /* free the uio job ring */
2085                 free_job_ring(job_ring->irq_fd);
2086                 job_ring->irq_fd = -1;
2087                 caam_jr_dma_free(job_ring->input_ring);
2088                 caam_jr_dma_free(job_ring->output_ring);
2089                 g_job_rings_no--;
2090         }
2091 }
2092
2093 /** @brief Release the software and hardware resources tied to a job ring.
2094  * @param [in] job_ring The job ring
2095  *
2096  * @retval  0 for success
2097  * @retval  -1 for error
2098  */
2099 static int
2100 shutdown_job_ring(struct sec_job_ring_t *job_ring)
2101 {
2102         int ret = 0;
2103
2104         PMD_INIT_FUNC_TRACE();
2105         ASSERT(job_ring != NULL);
2106         ret = hw_shutdown_job_ring(job_ring);
2107         SEC_ASSERT(ret == 0, ret,
2108                 "Failed to shutdown hardware job ring %p",
2109                 job_ring);
2110
2111         if (job_ring->coalescing_en)
2112                 hw_job_ring_disable_coalescing(job_ring);
2113
2114         if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL) {
2115                 ret = caam_jr_disable_irqs(job_ring->irq_fd);
2116                 SEC_ASSERT(ret == 0, ret,
2117                 "Failed to disable irqs for job ring %p",
2118                 job_ring);
2119         }
2120
2121         return ret;
2122 }
2123
2124 /*
2125  * @brief Release the resources used by the SEC user space driver.
2126  *
2127  * Reset and release SEC's job rings indicated by the User Application at
2128  * init_job_ring() and free any memory allocated internally.
2129  * Call once during application tear down.
2130  *
2131  * @note In case there are any descriptors in-flight (descriptors received by
2132  * SEC driver for processing and for which no response was yet provided to UA),
2133  * the descriptors are discarded without any notifications to User Application.
2134  *
2135  * @retval ::0                  is returned for a successful execution
2136  * @retval ::-1         is returned if SEC driver release is in progress
2137  */
2138 static int
2139 caam_jr_dev_uninit(struct rte_cryptodev *dev)
2140 {
2141         struct sec_job_ring_t *internals;
2142
2143         PMD_INIT_FUNC_TRACE();
2144         if (dev == NULL)
2145                 return -ENODEV;
2146
2147         internals = dev->data->dev_private;
2148         rte_free(dev->security_ctx);
2149
2150         /* If any descriptors in flight , poll and wait
2151          * until all descriptors are received and silently discarded.
2152          */
2153         if (internals) {
2154                 shutdown_job_ring(internals);
2155                 close_job_ring(internals);
2156                 rte_mempool_free(internals->ctx_pool);
2157         }
2158
2159         CAAM_JR_INFO("Closing crypto device %s", dev->data->name);
2160
2161         /* last caam jr instance) */
2162         if (g_job_rings_no == 0)
2163                 g_driver_state = SEC_DRIVER_STATE_IDLE;
2164
2165         return SEC_SUCCESS;
2166 }
2167
2168 /* @brief Initialize the software and hardware resources tied to a job ring.
2169  * @param [in] jr_mode;         Model to be used by SEC Driver to receive
2170  *                              notifications from SEC.  Can be either
2171  *                              of the three: #SEC_NOTIFICATION_TYPE_NAPI
2172  *                              #SEC_NOTIFICATION_TYPE_IRQ or
2173  *                              #SEC_NOTIFICATION_TYPE_POLL
2174  * @param [in] NAPI_mode        The NAPI work mode to configure a job ring at
2175  *                              startup. Used only when #SEC_NOTIFICATION_TYPE
2176  *                              is set to #SEC_NOTIFICATION_TYPE_NAPI.
2177  * @param [in] irq_coalescing_timer This value determines the maximum
2178  *                                      amount of time after processing a
2179  *                                      descriptor before raising an interrupt.
2180  * @param [in] irq_coalescing_count This value determines how many
2181  *                                      descriptors are completed before
2182  *                                      raising an interrupt.
2183  * @param [in] reg_base_addr,   The job ring base address register
2184  * @param [in] irq_id           The job ring interrupt identification number.
2185  * @retval  job_ring_handle for successful job ring configuration
2186  * @retval  NULL on error
2187  *
2188  */
2189 static void *
2190 init_job_ring(void *reg_base_addr, int irq_id)
2191 {
2192         struct sec_job_ring_t *job_ring = NULL;
2193         int i, ret = 0;
2194         int jr_mode = SEC_NOTIFICATION_TYPE_POLL;
2195         int napi_mode = 0;
2196         int irq_coalescing_timer = 0;
2197         int irq_coalescing_count = 0;
2198
2199         for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
2200                 if (g_job_rings[i].irq_fd == -1) {
2201                         job_ring = &g_job_rings[i];
2202                         g_job_rings_no++;
2203                         break;
2204                 }
2205         }
2206         if (job_ring == NULL) {
2207                 CAAM_JR_ERR("No free job ring\n");
2208                 return NULL;
2209         }
2210
2211         job_ring->register_base_addr = reg_base_addr;
2212         job_ring->jr_mode = jr_mode;
2213         job_ring->napi_mode = 0;
2214         job_ring->irq_fd = irq_id;
2215
2216         /* Allocate mem for input and output ring */
2217
2218         /* Allocate memory for input ring */
2219         job_ring->input_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
2220                                 SEC_DMA_MEM_INPUT_RING_SIZE);
2221         memset(job_ring->input_ring, 0, SEC_DMA_MEM_INPUT_RING_SIZE);
2222
2223         /* Allocate memory for output ring */
2224         job_ring->output_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
2225                                 SEC_DMA_MEM_OUTPUT_RING_SIZE);
2226         memset(job_ring->output_ring, 0, SEC_DMA_MEM_OUTPUT_RING_SIZE);
2227
2228         /* Reset job ring in SEC hw and configure job ring registers */
2229         ret = hw_reset_job_ring(job_ring);
2230         if (ret != 0) {
2231                 CAAM_JR_ERR("Failed to reset hardware job ring");
2232                 goto cleanup;
2233         }
2234
2235         if (jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
2236         /* When SEC US driver works in NAPI mode, the UA can select
2237          * if the driver starts with IRQs on or off.
2238          */
2239                 if (napi_mode == SEC_STARTUP_INTERRUPT_MODE) {
2240                         CAAM_JR_INFO("Enabling DONE IRQ generationon job ring - %p",
2241                                 job_ring);
2242                         ret = caam_jr_enable_irqs(job_ring->irq_fd);
2243                         if (ret != 0) {
2244                                 CAAM_JR_ERR("Failed to enable irqs for job ring");
2245                                 goto cleanup;
2246                         }
2247                 }
2248         } else if (jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
2249         /* When SEC US driver works in pure interrupt mode,
2250          * IRQ's are always enabled.
2251          */
2252                 CAAM_JR_INFO("Enabling DONE IRQ generation on job ring - %p",
2253                          job_ring);
2254                 ret = caam_jr_enable_irqs(job_ring->irq_fd);
2255                 if (ret != 0) {
2256                         CAAM_JR_ERR("Failed to enable irqs for job ring");
2257                         goto cleanup;
2258                 }
2259         }
2260         if (irq_coalescing_timer || irq_coalescing_count) {
2261                 hw_job_ring_set_coalescing_param(job_ring,
2262                          irq_coalescing_timer,
2263                          irq_coalescing_count);
2264
2265                 hw_job_ring_enable_coalescing(job_ring);
2266                 job_ring->coalescing_en = 1;
2267         }
2268
2269         job_ring->jr_state = SEC_JOB_RING_STATE_STARTED;
2270         job_ring->max_nb_queue_pairs = RTE_CAAM_MAX_NB_SEC_QPS;
2271         job_ring->max_nb_sessions = RTE_CAAM_JR_PMD_MAX_NB_SESSIONS;
2272
2273         return job_ring;
2274 cleanup:
2275         caam_jr_dma_free(job_ring->output_ring);
2276         caam_jr_dma_free(job_ring->input_ring);
2277         return NULL;
2278 }
2279
2280
2281 static int
2282 caam_jr_dev_init(const char *name,
2283                  struct rte_vdev_device *vdev,
2284                  struct rte_cryptodev_pmd_init_params *init_params)
2285 {
2286         struct rte_cryptodev *dev;
2287         struct rte_security_ctx *security_instance;
2288         struct uio_job_ring *job_ring;
2289         char str[RTE_CRYPTODEV_NAME_MAX_LEN];
2290
2291         PMD_INIT_FUNC_TRACE();
2292
2293         /* Validate driver state */
2294         if (g_driver_state == SEC_DRIVER_STATE_IDLE) {
2295                 g_job_rings_max = sec_configure();
2296                 if (!g_job_rings_max) {
2297                         CAAM_JR_ERR("No job ring detected on UIO !!!!");
2298                         return -1;
2299                 }
2300                 /* Update driver state */
2301                 g_driver_state = SEC_DRIVER_STATE_STARTED;
2302         }
2303
2304         if (g_job_rings_no >= g_job_rings_max) {
2305                 CAAM_JR_ERR("No more job rings available max=%d!!!!",
2306                                 g_job_rings_max);
2307                 return -1;
2308         }
2309
2310         job_ring = config_job_ring();
2311         if (job_ring == NULL) {
2312                 CAAM_JR_ERR("failed to create job ring");
2313                 goto init_error;
2314         }
2315
2316         snprintf(str, sizeof(str), "caam_jr%d", job_ring->jr_id);
2317
2318         dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
2319         if (dev == NULL) {
2320                 CAAM_JR_ERR("failed to create cryptodev vdev");
2321                 goto cleanup;
2322         }
2323         /*TODO free it during teardown*/
2324         dev->data->dev_private = init_job_ring(job_ring->register_base_addr,
2325                                                 job_ring->uio_fd);
2326
2327         if (!dev->data->dev_private) {
2328                 CAAM_JR_ERR("Ring memory allocation failed\n");
2329                 goto cleanup2;
2330         }
2331
2332         dev->driver_id = cryptodev_driver_id;
2333         dev->dev_ops = &caam_jr_ops;
2334
2335         /* register rx/tx burst functions for data path */
2336         dev->dequeue_burst = caam_jr_dequeue_burst;
2337         dev->enqueue_burst = caam_jr_enqueue_burst;
2338         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2339                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
2340                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2341                         RTE_CRYPTODEV_FF_SECURITY |
2342                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
2343                         RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
2344                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
2345                         RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
2346                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
2347
2348         /* For secondary processes, we don't initialise any further as primary
2349          * has already done this work. Only check we don't need a different
2350          * RX function
2351          */
2352         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2353                 CAAM_JR_WARN("Device already init by primary process");
2354                 return 0;
2355         }
2356
2357         /*TODO free it during teardown*/
2358         security_instance = rte_malloc("caam_jr",
2359                                 sizeof(struct rte_security_ctx), 0);
2360         if (security_instance == NULL) {
2361                 CAAM_JR_ERR("memory allocation failed\n");
2362                 //todo error handling.
2363                 goto cleanup2;
2364         }
2365
2366         security_instance->device = (void *)dev;
2367         security_instance->ops = &caam_jr_security_ops;
2368         security_instance->sess_cnt = 0;
2369         dev->security_ctx = security_instance;
2370
2371         RTE_LOG(INFO, PMD, "%s cryptodev init\n", dev->data->name);
2372
2373         return 0;
2374
2375 cleanup2:
2376         caam_jr_dev_uninit(dev);
2377         rte_cryptodev_pmd_release_device(dev);
2378 cleanup:
2379         free_job_ring(job_ring->uio_fd);
2380 init_error:
2381         CAAM_JR_ERR("driver %s: cryptodev_caam_jr_create failed",
2382                         init_params->name);
2383
2384         return -ENXIO;
2385 }
2386
2387 /** Initialise CAAM JR crypto device */
2388 static int
2389 cryptodev_caam_jr_probe(struct rte_vdev_device *vdev)
2390 {
2391         struct rte_cryptodev_pmd_init_params init_params = {
2392                 "",
2393                 sizeof(struct sec_job_ring_t),
2394                 rte_socket_id(),
2395                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
2396         };
2397         const char *name;
2398         const char *input_args;
2399
2400         name = rte_vdev_device_name(vdev);
2401         if (name == NULL)
2402                 return -EINVAL;
2403
2404         input_args = rte_vdev_device_args(vdev);
2405         rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
2406
2407         /* if sec device version is not configured */
2408         if (!rta_get_sec_era()) {
2409                 const struct device_node *caam_node;
2410
2411                 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
2412                         const uint32_t *prop = of_get_property(caam_node,
2413                                         "fsl,sec-era",
2414                                         NULL);
2415                         if (prop) {
2416                                 rta_set_sec_era(
2417                                         INTL_SEC_ERA(cpu_to_caam32(*prop)));
2418                                 break;
2419                         }
2420                 }
2421         }
2422 #ifdef RTE_LIBRTE_PMD_CAAM_JR_BE
2423         if (rta_get_sec_era() > RTA_SEC_ERA_8) {
2424                 RTE_LOG(ERR, PMD,
2425                 "CAAM is compiled in BE mode for device with sec era > 8???\n");
2426                 return -EINVAL;
2427         }
2428 #endif
2429
2430         return caam_jr_dev_init(name, vdev, &init_params);
2431 }
2432
2433 /** Uninitialise CAAM JR crypto device */
2434 static int
2435 cryptodev_caam_jr_remove(struct rte_vdev_device *vdev)
2436 {
2437         struct rte_cryptodev *cryptodev;
2438         const char *name;
2439
2440         name = rte_vdev_device_name(vdev);
2441         if (name == NULL)
2442                 return -EINVAL;
2443
2444         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
2445         if (cryptodev == NULL)
2446                 return -ENODEV;
2447
2448         caam_jr_dev_uninit(cryptodev);
2449
2450         return rte_cryptodev_pmd_destroy(cryptodev);
2451 }
2452
2453 static void
2454 sec_job_rings_init(void)
2455 {
2456         int i;
2457
2458         for (i = 0; i < MAX_SEC_JOB_RINGS; i++)
2459                 g_job_rings[i].irq_fd = -1;
2460 }
2461
2462 static struct rte_vdev_driver cryptodev_caam_jr_drv = {
2463         .probe = cryptodev_caam_jr_probe,
2464         .remove = cryptodev_caam_jr_remove
2465 };
2466
2467 static struct cryptodev_driver caam_jr_crypto_drv;
2468
2469 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CAAM_JR_PMD, cryptodev_caam_jr_drv);
2470 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CAAM_JR_PMD,
2471         "max_nb_queue_pairs=<int>"
2472         "socket_id=<int>");
2473 RTE_PMD_REGISTER_CRYPTO_DRIVER(caam_jr_crypto_drv, cryptodev_caam_jr_drv.driver,
2474                 cryptodev_driver_id);
2475
2476 RTE_INIT(caam_jr_init)
2477 {
2478         sec_uio_job_rings_init();
2479         sec_job_rings_init();
2480 }
2481
2482 RTE_LOG_REGISTER(caam_jr_logtype, pmd.crypto.caam, NOTICE);