drivers/crypto/caam_jr/caam_jr.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017-2019 NXP
   3  */
   4
   5 #include <fcntl.h>
   6 #include <unistd.h>
   7 #include <sched.h>
   8 #include <net/if.h>
   9
  10 #include <rte_byteorder.h>
  11 #include <rte_common.h>
  12 #include <rte_cryptodev_pmd.h>
  13 #include <rte_crypto.h>
  14 #include <rte_cryptodev.h>
  15 #include <rte_bus_vdev.h>
  16 #include <rte_malloc.h>
  17 #include <rte_security_driver.h>
  18 #include <rte_hexdump.h>
  19
  20 #include <caam_jr_capabilities.h>
  21 #include <caam_jr_config.h>
  22 #include <caam_jr_hw_specific.h>
  23 #include <caam_jr_pvt.h>
  24 #include <caam_jr_desc.h>
  25 #include <caam_jr_log.h>
  26
  27 /* RTA header files */
  28 #include <desc/common.h>
  29 #include <desc/algo.h>
  30 #include <dpaa_of.h>
  31 #ifdef RTE_LIBRTE_PMD_CAAM_JR_DEBUG
  32 #define CAAM_JR_DBG    1
  33 #else
  34 #define CAAM_JR_DBG     0
  35 #endif
  36 #define CRYPTODEV_NAME_CAAM_JR_PMD      crypto_caam_jr
  37 static uint8_t cryptodev_driver_id;
  38 int caam_jr_logtype;
  39
  40 enum rta_sec_era rta_sec_era;
  41
  42 /* Lists the states possible for the SEC user space driver. */
  43 enum sec_driver_state_e {
  44         SEC_DRIVER_STATE_IDLE,          /* Driver not initialized */
  45         SEC_DRIVER_STATE_STARTED,       /* Driver initialized and can be used*/
  46         SEC_DRIVER_STATE_RELEASE,       /* Driver release is in progress */
  47 };
  48
  49 /* Job rings used for communication with SEC HW */
  50 static struct sec_job_ring_t g_job_rings[MAX_SEC_JOB_RINGS];
  51
  52 /* The current state of SEC user space driver */
  53 static enum sec_driver_state_e g_driver_state = SEC_DRIVER_STATE_IDLE;
  54
  55 /* The number of job rings used by SEC user space driver */
  56 static int g_job_rings_no;
  57 static int g_job_rings_max;
  58
  59 struct sec_outring_entry {
  60         phys_addr_t desc;       /* Pointer to completed descriptor */
  61         uint32_t status;        /* Status for completed descriptor */
  62 } __rte_packed;
  63
  64 /* virtual address conversin when mempool support is available for ctx */
  65 static inline phys_addr_t
  66 caam_jr_vtop_ctx(struct caam_jr_op_ctx *ctx, void *vaddr)
  67 {
  68         return (size_t)vaddr - ctx->vtop_offset;
  69 }
  70
  71 static inline void
  72 caam_jr_op_ending(struct caam_jr_op_ctx *ctx)
  73 {
  74         /* report op status to sym->op and then free the ctx memory  */
  75         rte_mempool_put(ctx->ctx_pool, (void *)ctx);
  76 }
  77
  78 static inline struct caam_jr_op_ctx *
  79 caam_jr_alloc_ctx(struct caam_jr_session *ses)
  80 {
  81         struct caam_jr_op_ctx *ctx;
  82         int ret;
  83
  84         ret = rte_mempool_get(ses->ctx_pool, (void **)(&ctx));
  85         if (!ctx || ret) {
  86                 CAAM_JR_DP_WARN("Alloc sec descriptor failed!");
  87                 return NULL;
  88         }
  89         /*
  90          * Clear SG memory. There are 16 SG entries of 16 Bytes each.
  91          * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
  92          * to clear all the SG entries. caam_jr_alloc_ctx() is called for
  93          * each packet, memset is costlier than dcbz_64().
  94          */
  95         dcbz_64(&ctx->sg[SG_CACHELINE_0]);
  96         dcbz_64(&ctx->sg[SG_CACHELINE_1]);
  97         dcbz_64(&ctx->sg[SG_CACHELINE_2]);
  98         dcbz_64(&ctx->sg[SG_CACHELINE_3]);
  99
 100         ctx->ctx_pool = ses->ctx_pool;
 101         ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx);
 102
 103         return ctx;
 104 }
 105
 106 static
 107 void caam_jr_stats_get(struct rte_cryptodev *dev,
 108                         struct rte_cryptodev_stats *stats)
 109 {
 110         struct caam_jr_qp **qp = (struct caam_jr_qp **)
 111                                         dev->data->queue_pairs;
 112         int i;
 113
 114         PMD_INIT_FUNC_TRACE();
 115         if (stats == NULL) {
 116                 CAAM_JR_ERR("Invalid stats ptr NULL");
 117                 return;
 118         }
 119         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 120                 if (qp[i] == NULL) {
 121                         CAAM_JR_WARN("Uninitialised queue pair");
 122                         continue;
 123                 }
 124
 125                 stats->enqueued_count += qp[i]->tx_pkts;
 126                 stats->dequeued_count += qp[i]->rx_pkts;
 127                 stats->enqueue_err_count += qp[i]->tx_errs;
 128                 stats->dequeue_err_count += qp[i]->rx_errs;
 129                 CAAM_JR_INFO("extra stats:\n\tRX Poll ERR = %" PRIu64
 130                              "\n\tTX Ring Full = %" PRIu64,
 131                              qp[i]->rx_poll_err,
 132                              qp[i]->tx_ring_full);
 133         }
 134 }
 135
 136 static
 137 void caam_jr_stats_reset(struct rte_cryptodev *dev)
 138 {
 139         int i;
 140         struct caam_jr_qp **qp = (struct caam_jr_qp **)
 141                                    (dev->data->queue_pairs);
 142
 143         PMD_INIT_FUNC_TRACE();
 144         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 145                 if (qp[i] == NULL) {
 146                         CAAM_JR_WARN("Uninitialised queue pair");
 147                         continue;
 148                 }
 149                 qp[i]->rx_pkts = 0;
 150                 qp[i]->rx_errs = 0;
 151                 qp[i]->rx_poll_err = 0;
 152                 qp[i]->tx_pkts = 0;
 153                 qp[i]->tx_errs = 0;
 154                 qp[i]->tx_ring_full = 0;
 155         }
 156 }
 157
 158 static inline int
 159 is_cipher_only(struct caam_jr_session *ses)
 160 {
 161         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 162                 (ses->auth_alg == RTE_CRYPTO_AUTH_NULL));
 163 }
 164
 165 static inline int
 166 is_auth_only(struct caam_jr_session *ses)
 167 {
 168         return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) &&
 169                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL));
 170 }
 171
 172 static inline int
 173 is_aead(struct caam_jr_session *ses)
 174 {
 175         return ((ses->cipher_alg == 0) &&
 176                 (ses->auth_alg == 0) &&
 177                 (ses->aead_alg != 0));
 178 }
 179
 180 static inline int
 181 is_auth_cipher(struct caam_jr_session *ses)
 182 {
 183         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 184                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) &&
 185                 (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC));
 186 }
 187
 188 static inline int
 189 is_proto_ipsec(struct caam_jr_session *ses)
 190 {
 191         return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC);
 192 }
 193
 194 static inline int
 195 is_encode(struct caam_jr_session *ses)
 196 {
 197         return ses->dir == DIR_ENC;
 198 }
 199
 200 static inline int
 201 is_decode(struct caam_jr_session *ses)
 202 {
 203         return ses->dir == DIR_DEC;
 204 }
 205
 206 static inline void
 207 caam_auth_alg(struct caam_jr_session *ses, struct alginfo *alginfo_a)
 208 {
 209         switch (ses->auth_alg) {
 210         case RTE_CRYPTO_AUTH_NULL:
 211                 ses->digest_length = 0;
 212                 break;
 213         case RTE_CRYPTO_AUTH_MD5_HMAC:
 214                 alginfo_a->algtype =
 215                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 216                         OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5;
 217                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 218                 break;
 219         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 220                 alginfo_a->algtype =
 221                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 222                         OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1;
 223                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 224                 break;
 225         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 226                 alginfo_a->algtype =
 227                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 228                         OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224;
 229                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 230                 break;
 231         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 232                 alginfo_a->algtype =
 233                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 234                         OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256;
 235                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 236                 break;
 237         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 238                 alginfo_a->algtype =
 239                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 240                         OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384;
 241                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 242                 break;
 243         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 244                 alginfo_a->algtype =
 245                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 246                         OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512;
 247                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 248                 break;
 249         default:
 250                 CAAM_JR_DEBUG("unsupported auth alg %u", ses->auth_alg);
 251         }
 252 }
 253
 254 static inline void
 255 caam_cipher_alg(struct caam_jr_session *ses, struct alginfo *alginfo_c)
 256 {
 257         switch (ses->cipher_alg) {
 258         case RTE_CRYPTO_CIPHER_NULL:
 259                 break;
 260         case RTE_CRYPTO_CIPHER_AES_CBC:
 261                 alginfo_c->algtype =
 262                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 263                         OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES;
 264                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 265                 break;
 266         case RTE_CRYPTO_CIPHER_3DES_CBC:
 267                 alginfo_c->algtype =
 268                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 269                         OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES;
 270                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 271                 break;
 272         case RTE_CRYPTO_CIPHER_AES_CTR:
 273                 alginfo_c->algtype =
 274                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 275                         OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES;
 276                 alginfo_c->algmode = OP_ALG_AAI_CTR;
 277                 break;
 278         default:
 279                 CAAM_JR_DEBUG("unsupported cipher alg %d", ses->cipher_alg);
 280         }
 281 }
 282
 283 static inline void
 284 caam_aead_alg(struct caam_jr_session *ses, struct alginfo *alginfo)
 285 {
 286         switch (ses->aead_alg) {
 287         case RTE_CRYPTO_AEAD_AES_GCM:
 288                 alginfo->algtype = OP_ALG_ALGSEL_AES;
 289                 alginfo->algmode = OP_ALG_AAI_GCM;
 290                 break;
 291         default:
 292                 CAAM_JR_DEBUG("unsupported AEAD alg %d", ses->aead_alg);
 293         }
 294 }
 295
 296 /* prepare command block of the session */
 297 static int
 298 caam_jr_prep_cdb(struct caam_jr_session *ses)
 299 {
 300         struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0};
 301         int32_t shared_desc_len = 0;
 302         struct sec_cdb *cdb;
 303         int err;
 304 #if CAAM_BYTE_ORDER == CORE_BYTE_ORDER
 305         int swap = false;
 306 #else
 307         int swap = true;
 308 #endif
 309
 310         if (ses->cdb)
 311                 caam_jr_dma_free(ses->cdb);
 312
 313         cdb = caam_jr_dma_mem_alloc(L1_CACHE_BYTES, sizeof(struct sec_cdb));
 314         if (!cdb) {
 315                 CAAM_JR_ERR("failed to allocate memory for cdb\n");
 316                 return -1;
 317         }
 318
 319         ses->cdb = cdb;
 320
 321         memset(cdb, 0, sizeof(struct sec_cdb));
 322
 323         if (is_cipher_only(ses)) {
 324                 caam_cipher_alg(ses, &alginfo_c);
 325                 if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 326                         CAAM_JR_ERR("not supported cipher alg");
 327                         rte_free(cdb);
 328                         return -ENOTSUP;
 329                 }
 330
 331                 alginfo_c.key = (size_t)ses->cipher_key.data;
 332                 alginfo_c.keylen = ses->cipher_key.length;
 333                 alginfo_c.key_enc_flags = 0;
 334                 alginfo_c.key_type = RTA_DATA_IMM;
 335
 336                 shared_desc_len = cnstr_shdsc_blkcipher(
 337                                                 cdb->sh_desc, true,
 338                                                 swap, SHR_NEVER, &alginfo_c,
 339                                                 ses->iv.length,
 340                                                 ses->dir);
 341         } else if (is_auth_only(ses)) {
 342                 caam_auth_alg(ses, &alginfo_a);
 343                 if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 344                         CAAM_JR_ERR("not supported auth alg");
 345                         rte_free(cdb);
 346                         return -ENOTSUP;
 347                 }
 348
 349                 alginfo_a.key = (size_t)ses->auth_key.data;
 350                 alginfo_a.keylen = ses->auth_key.length;
 351                 alginfo_a.key_enc_flags = 0;
 352                 alginfo_a.key_type = RTA_DATA_IMM;
 353
 354                 shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true,
 355                                                    swap, SHR_NEVER, &alginfo_a,
 356                                                    !ses->dir,
 357                                                    ses->digest_length);
 358         } else if (is_aead(ses)) {
 359                 caam_aead_alg(ses, &alginfo);
 360                 if (alginfo.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 361                         CAAM_JR_ERR("not supported aead alg");
 362                         rte_free(cdb);
 363                         return -ENOTSUP;
 364                 }
 365                 alginfo.key = (size_t)ses->aead_key.data;
 366                 alginfo.keylen = ses->aead_key.length;
 367                 alginfo.key_enc_flags = 0;
 368                 alginfo.key_type = RTA_DATA_IMM;
 369
 370                 if (ses->dir == DIR_ENC)
 371                         shared_desc_len = cnstr_shdsc_gcm_encap(
 372                                         cdb->sh_desc, true, swap,
 373                                         SHR_NEVER, &alginfo,
 374                                         ses->iv.length,
 375                                         ses->digest_length);
 376                 else
 377                         shared_desc_len = cnstr_shdsc_gcm_decap(
 378                                         cdb->sh_desc, true, swap,
 379                                         SHR_NEVER, &alginfo,
 380                                         ses->iv.length,
 381                                         ses->digest_length);
 382         } else {
 383                 caam_cipher_alg(ses, &alginfo_c);
 384                 if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 385                         CAAM_JR_ERR("not supported cipher alg");
 386                         rte_free(cdb);
 387                         return -ENOTSUP;
 388                 }
 389
 390                 alginfo_c.key = (size_t)ses->cipher_key.data;
 391                 alginfo_c.keylen = ses->cipher_key.length;
 392                 alginfo_c.key_enc_flags = 0;
 393                 alginfo_c.key_type = RTA_DATA_IMM;
 394
 395                 caam_auth_alg(ses, &alginfo_a);
 396                 if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 397                         CAAM_JR_ERR("not supported auth alg");
 398                         rte_free(cdb);
 399                         return -ENOTSUP;
 400                 }
 401
 402                 alginfo_a.key = (size_t)ses->auth_key.data;
 403                 alginfo_a.keylen = ses->auth_key.length;
 404                 alginfo_a.key_enc_flags = 0;
 405                 alginfo_a.key_type = RTA_DATA_IMM;
 406
 407                 cdb->sh_desc[0] = alginfo_c.keylen;
 408                 cdb->sh_desc[1] = alginfo_a.keylen;
 409                 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
 410                                        MIN_JOB_DESC_SIZE,
 411                                        (unsigned int *)cdb->sh_desc,
 412                                        &cdb->sh_desc[2], 2);
 413
 414                 if (err < 0) {
 415                         CAAM_JR_ERR("Crypto: Incorrect key lengths");
 416                         rte_free(cdb);
 417                         return err;
 418                 }
 419                 if (cdb->sh_desc[2] & 1)
 420                         alginfo_c.key_type = RTA_DATA_IMM;
 421                 else {
 422                         alginfo_c.key = (size_t)caam_jr_mem_vtop(
 423                                                 (void *)(size_t)alginfo_c.key);
 424                         alginfo_c.key_type = RTA_DATA_PTR;
 425                 }
 426                 if (cdb->sh_desc[2] & (1<<1))
 427                         alginfo_a.key_type = RTA_DATA_IMM;
 428                 else {
 429                         alginfo_a.key = (size_t)caam_jr_mem_vtop(
 430                                                 (void *)(size_t)alginfo_a.key);
 431                         alginfo_a.key_type = RTA_DATA_PTR;
 432                 }
 433                 cdb->sh_desc[0] = 0;
 434                 cdb->sh_desc[1] = 0;
 435                 cdb->sh_desc[2] = 0;
 436                 if (is_proto_ipsec(ses)) {
 437                         if (ses->dir == DIR_ENC) {
 438                                 shared_desc_len = cnstr_shdsc_ipsec_new_encap(
 439                                                 cdb->sh_desc,
 440                                                 true, swap, SHR_SERIAL,
 441                                                 &ses->encap_pdb,
 442                                                 (uint8_t *)&ses->ip4_hdr,
 443                                                 &alginfo_c, &alginfo_a);
 444                         } else if (ses->dir == DIR_DEC) {
 445                                 shared_desc_len = cnstr_shdsc_ipsec_new_decap(
 446                                                 cdb->sh_desc,
 447                                                 true, swap, SHR_SERIAL,
 448                                                 &ses->decap_pdb,
 449                                                 &alginfo_c, &alginfo_a);
 450                         }
 451                 } else {
 452                         /* Auth_only_len is overwritten in fd for each job */
 453                         shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
 454                                         true, swap, SHR_SERIAL,
 455                                         &alginfo_c, &alginfo_a,
 456                                         ses->iv.length,
 457                                         ses->digest_length, ses->dir);
 458                 }
 459         }
 460
 461         if (shared_desc_len < 0) {
 462                 CAAM_JR_ERR("error in preparing command block");
 463                 return shared_desc_len;
 464         }
 465
 466 #if CAAM_JR_DBG
 467         SEC_DUMP_DESC(cdb->sh_desc);
 468 #endif
 469
 470         cdb->sh_hdr.hi.field.idlen = shared_desc_len;
 471
 472         return 0;
 473 }
 474
 475 /* @brief Poll the HW for already processed jobs in the JR
 476  * and silently discard the available jobs or notify them to UA
 477  * with indicated error code.
 478  *
 479  * @param [in,out]  job_ring        The job ring to poll.
 480  * @param [in]  do_notify           Can be #TRUE or #FALSE. Indicates if
 481  *                                  descriptors are to be discarded
 482  *                                  or notified to UA with given error_code.
 483  * @param [out] notified_descs    Number of notified descriptors. Can be NULL
 484  *                                      if do_notify is #FALSE
 485  */
 486 static void
 487 hw_flush_job_ring(struct sec_job_ring_t *job_ring,
 488                   uint32_t do_notify,
 489                   uint32_t *notified_descs)
 490 {
 491         int32_t jobs_no_to_discard = 0;
 492         int32_t discarded_descs_no = 0;
 493
 494         CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Flushing jr notify desc=[%d]",
 495                 job_ring, job_ring->pidx, job_ring->cidx, do_notify);
 496
 497         jobs_no_to_discard = hw_get_no_finished_jobs(job_ring);
 498
 499         /* Discard all jobs */
 500         CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Discarding %d descs",
 501                   job_ring, job_ring->pidx, job_ring->cidx,
 502                   jobs_no_to_discard);
 503
 504         while (jobs_no_to_discard > discarded_descs_no) {
 505                 discarded_descs_no++;
 506                 /* Now increment the consumer index for the current job ring,
 507                  * AFTER saving job in temporary location!
 508                  * Increment the consumer index for the current job ring
 509                  */
 510                 job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
 511                                          SEC_JOB_RING_SIZE);
 512
 513                 hw_remove_entries(job_ring, 1);
 514         }
 515
 516         if (do_notify == true) {
 517                 ASSERT(notified_descs != NULL);
 518                 *notified_descs = discarded_descs_no;
 519         }
 520 }
 521
 522 /* @brief Poll the HW for already processed jobs in the JR
 523  * and notify the available jobs to UA.
 524  *
 525  * @param [in]  job_ring        The job ring to poll.
 526  * @param [in]  limit           The maximum number of jobs to notify.
 527  *                              If set to negative value, all available jobs are
 528  *                              notified.
 529  *
 530  * @retval >=0 for No of jobs notified to UA.
 531  * @retval -1 for error
 532  */
 533 static int
 534 hw_poll_job_ring(struct sec_job_ring_t *job_ring,
 535                  struct rte_crypto_op **ops, int32_t limit,
 536                  struct caam_jr_qp *jr_qp)
 537 {
 538         int32_t jobs_no_to_notify = 0; /* the number of done jobs to notify*/
 539         int32_t number_of_jobs_available = 0;
 540         int32_t notified_descs_no = 0;
 541         uint32_t sec_error_code = 0;
 542         struct job_descriptor *current_desc;
 543         phys_addr_t current_desc_addr;
 544         phys_addr_t *temp_addr;
 545         struct caam_jr_op_ctx *ctx;
 546
 547         /* TODO check for ops have memory*/
 548         /* check here if any JR error that cannot be written
 549          * in the output status word has occurred
 550          */
 551         if (JR_REG_JRINT_JRE_EXTRACT(GET_JR_REG(JRINT, job_ring))) {
 552                 CAAM_JR_INFO("err received");
 553                 sec_error_code = JR_REG_JRINT_ERR_TYPE_EXTRACT(
 554                                         GET_JR_REG(JRINT, job_ring));
 555                 if (unlikely(sec_error_code)) {
 556                         hw_job_ring_error_print(job_ring, sec_error_code);
 557                         return -1;
 558                 }
 559         }
 560         /* compute the number of jobs available in the job ring based on the
 561          * producer and consumer index values.
 562          */
 563         number_of_jobs_available = hw_get_no_finished_jobs(job_ring);
 564         /* Compute the number of notifications that need to be raised to UA
 565          * If limit > total number of done jobs -> notify all done jobs
 566          * If limit = 0 -> error
 567          * If limit < total number of done jobs -> notify a number
 568          * of done jobs equal with limit
 569          */
 570         jobs_no_to_notify = (limit > number_of_jobs_available) ?
 571                                 number_of_jobs_available : limit;
 572         CAAM_JR_DP_DEBUG(
 573                 "Jr[%p] pi[%d] ci[%d].limit =%d Available=%d.Jobs to notify=%d",
 574                 job_ring, job_ring->pidx, job_ring->cidx,
 575                 limit, number_of_jobs_available, jobs_no_to_notify);
 576
 577         rte_smp_rmb();
 578
 579         while (jobs_no_to_notify > notified_descs_no) {
 580                 static uint64_t false_alarm;
 581                 static uint64_t real_poll;
 582
 583                 /* Get job status here */
 584                 sec_error_code = job_ring->output_ring[job_ring->cidx].status;
 585                 /* Get completed descriptor */
 586                 temp_addr = &(job_ring->output_ring[job_ring->cidx].desc);
 587                 current_desc_addr = (phys_addr_t)sec_read_addr(temp_addr);
 588
 589                 real_poll++;
 590                 /* todo check if it is false alarm no desc present */
 591                 if (!current_desc_addr) {
 592                         false_alarm++;
 593                         printf("false alarm %" PRIu64 "real %" PRIu64
 594                                 " sec_err =0x%x cidx Index =0%d\n",
 595                                 false_alarm, real_poll,
 596                                 sec_error_code, job_ring->cidx);
 597                         rte_panic("CAAM JR descriptor NULL");
 598                         return notified_descs_no;
 599                 }
 600                 current_desc = (struct job_descriptor *)
 601                                 caam_jr_dma_ptov(current_desc_addr);
 602                 /* now increment the consumer index for the current job ring,
 603                  * AFTER saving job in temporary location!
 604                  */
 605                 job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
 606                                  SEC_JOB_RING_SIZE);
 607                 /* Signal that the job has been processed and the slot is free*/
 608                 hw_remove_entries(job_ring, 1);
 609                 /*TODO for multiple ops, packets*/
 610                 ctx = container_of(current_desc, struct caam_jr_op_ctx, jobdes);
 611                 if (unlikely(sec_error_code)) {
 612                         CAAM_JR_ERR("desc at cidx %d generated error 0x%x\n",
 613                                 job_ring->cidx, sec_error_code);
 614                         hw_handle_job_ring_error(job_ring, sec_error_code);
 615                         //todo improve with exact errors
 616                         ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 617                         jr_qp->rx_errs++;
 618                 } else {
 619                         ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 620 #if CAAM_JR_DBG
 621                         if (ctx->op->sym->m_dst) {
 622                                 rte_hexdump(stdout, "PROCESSED",
 623                                 rte_pktmbuf_mtod(ctx->op->sym->m_dst, void *),
 624                                 rte_pktmbuf_data_len(ctx->op->sym->m_dst));
 625                         } else {
 626                                 rte_hexdump(stdout, "PROCESSED",
 627                                 rte_pktmbuf_mtod(ctx->op->sym->m_src, void *),
 628                                 rte_pktmbuf_data_len(ctx->op->sym->m_src));
 629                         }
 630 #endif
 631                 }
 632                 if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 633                         struct ip *ip4_hdr;
 634
 635                         if (ctx->op->sym->m_dst) {
 636                                 /*TODO check for ip header or other*/
 637                                 ip4_hdr = (struct ip *)
 638                                 rte_pktmbuf_mtod(ctx->op->sym->m_dst, char*);
 639                                 ctx->op->sym->m_dst->pkt_len =
 640                                         rte_be_to_cpu_16(ip4_hdr->ip_len);
 641                                 ctx->op->sym->m_dst->data_len =
 642                                         rte_be_to_cpu_16(ip4_hdr->ip_len);
 643                         } else {
 644                                 ip4_hdr = (struct ip *)
 645                                 rte_pktmbuf_mtod(ctx->op->sym->m_src, char*);
 646                                 ctx->op->sym->m_src->pkt_len =
 647                                         rte_be_to_cpu_16(ip4_hdr->ip_len);
 648                                 ctx->op->sym->m_src->data_len =
 649                                         rte_be_to_cpu_16(ip4_hdr->ip_len);
 650                         }
 651                 }
 652                 *ops = ctx->op;
 653                 caam_jr_op_ending(ctx);
 654                 ops++;
 655                 notified_descs_no++;
 656         }
 657         return notified_descs_no;
 658 }
 659
 660 static uint16_t
 661 caam_jr_dequeue_burst(void *qp, struct rte_crypto_op **ops,
 662                        uint16_t nb_ops)
 663 {
 664         struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp;
 665         struct sec_job_ring_t *ring = jr_qp->ring;
 666         int num_rx;
 667         int ret;
 668
 669         CAAM_JR_DP_DEBUG("Jr[%p]Polling. limit[%d]", ring, nb_ops);
 670
 671         /* Poll job ring
 672          * If nb_ops < 0 -> poll JR until no more notifications are available.
 673          * If nb_ops > 0 -> poll JR until limit is reached.
 674          */
 675
 676         /* Run hw poll job ring */
 677         num_rx = hw_poll_job_ring(ring, ops, nb_ops, jr_qp);
 678         if (num_rx < 0) {
 679                 CAAM_JR_ERR("Error polling SEC engine (%d)", num_rx);
 680                 return 0;
 681         }
 682
 683         CAAM_JR_DP_DEBUG("Jr[%p].Jobs notified[%d]. ", ring, num_rx);
 684
 685         if (ring->jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
 686                 if (num_rx < nb_ops) {
 687                         ret = caam_jr_enable_irqs(ring->irq_fd);
 688                         SEC_ASSERT(ret == 0, ret,
 689                         "Failed to enable irqs for job ring %p", ring);
 690                 }
 691         } else if (ring->jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
 692
 693                 /* Always enable IRQ generation when in pure IRQ mode */
 694                 ret = caam_jr_enable_irqs(ring->irq_fd);
 695                 SEC_ASSERT(ret == 0, ret,
 696                         "Failed to enable irqs for job ring %p", ring);
 697         }
 698
 699         jr_qp->rx_pkts += num_rx;
 700
 701         return num_rx;
 702 }
 703
 704 /**
 705  * packet looks like:
 706  *              |<----data_len------->|
 707  *    |ip_header|ah_header|icv|payload|
 708  *              ^
 709  *              |
 710  *         mbuf->pkt.data
 711  */
 712 static inline struct caam_jr_op_ctx *
 713 build_auth_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
 714 {
 715         struct rte_crypto_sym_op *sym = op->sym;
 716         struct rte_mbuf *mbuf = sym->m_src;
 717         struct caam_jr_op_ctx *ctx;
 718         struct sec4_sg_entry *sg;
 719         int     length;
 720         struct sec_cdb *cdb;
 721         uint64_t sdesc_offset;
 722         struct sec_job_descriptor_t *jobdescr;
 723         uint8_t extra_segs;
 724
 725         if (is_decode(ses))
 726                 extra_segs = 2;
 727         else
 728                 extra_segs = 1;
 729
 730         if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) {
 731                 CAAM_JR_DP_ERR("Auth: Max sec segs supported is %d",
 732                                 MAX_SG_ENTRIES);
 733                 return NULL;
 734         }
 735
 736         ctx = caam_jr_alloc_ctx(ses);
 737         if (!ctx)
 738                 return NULL;
 739
 740         ctx->op = op;
 741
 742         cdb = ses->cdb;
 743         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
 744
 745         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
 746
 747         SEC_JD_INIT(jobdescr);
 748         SEC_JD_SET_SD(jobdescr,
 749                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
 750                 cdb->sh_hdr.hi.field.idlen);
 751
 752         /* output */
 753         SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr,
 754                         0, ses->digest_length);
 755
 756         /*input */
 757         sg = &ctx->sg[0];
 758         length = sym->auth.data.length;
 759         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) + sym->auth.data.offset);
 760         sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
 761
 762         /* Successive segs */
 763         mbuf = mbuf->next;
 764         while (mbuf) {
 765                 sg++;
 766                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
 767                 sg->len = cpu_to_caam32(mbuf->data_len);
 768                 mbuf = mbuf->next;
 769         }
 770
 771         if (is_decode(ses)) {
 772                 /* digest verification case */
 773                 sg++;
 774                 /* hash result or digest, save digest first */
 775                 rte_memcpy(ctx->digest, sym->auth.digest.data,
 776                            ses->digest_length);
 777 #if CAAM_JR_DBG
 778                 rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length);
 779 #endif
 780                 sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest));
 781                 sg->len = cpu_to_caam32(ses->digest_length);
 782                 length += ses->digest_length;
 783         } else {
 784                 sg->len -= ses->digest_length;
 785         }
 786
 787         /* last element*/
 788         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
 789
 790         SEC_JD_SET_IN_PTR(jobdescr,
 791                 (uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0, length);
 792         /* enabling sg list */
 793         (jobdescr)->seq_in.command.word  |= 0x01000000;
 794
 795         return ctx;
 796 }
 797
 798 static inline struct caam_jr_op_ctx *
 799 build_auth_only(struct rte_crypto_op *op, struct caam_jr_session *ses)
 800 {
 801         struct rte_crypto_sym_op *sym = op->sym;
 802         struct caam_jr_op_ctx *ctx;
 803         struct sec4_sg_entry *sg;
 804         rte_iova_t start_addr;
 805         struct sec_cdb *cdb;
 806         uint64_t sdesc_offset;
 807         struct sec_job_descriptor_t *jobdescr;
 808
 809         ctx = caam_jr_alloc_ctx(ses);
 810         if (!ctx)
 811                 return NULL;
 812
 813         ctx->op = op;
 814
 815         cdb = ses->cdb;
 816         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
 817
 818         start_addr = rte_pktmbuf_iova(sym->m_src);
 819
 820         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
 821
 822         SEC_JD_INIT(jobdescr);
 823         SEC_JD_SET_SD(jobdescr,
 824                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
 825                 cdb->sh_hdr.hi.field.idlen);
 826
 827         /* output */
 828         SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr,
 829                         0, ses->digest_length);
 830
 831         /*input */
 832         if (is_decode(ses)) {
 833                 sg = &ctx->sg[0];
 834                 SEC_JD_SET_IN_PTR(jobdescr,
 835                         (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0,
 836                         (sym->auth.data.length + ses->digest_length));
 837                 /* enabling sg list */
 838                 (jobdescr)->seq_in.command.word  |= 0x01000000;
 839
 840                 /* hash result or digest, save digest first */
 841                 rte_memcpy(ctx->digest, sym->auth.digest.data,
 842                            ses->digest_length);
 843                 sg->ptr = cpu_to_caam64(start_addr + sym->auth.data.offset);
 844                 sg->len = cpu_to_caam32(sym->auth.data.length);
 845
 846 #if CAAM_JR_DBG
 847                 rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length);
 848 #endif
 849                 /* let's check digest by hw */
 850                 sg++;
 851                 sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest));
 852                 sg->len = cpu_to_caam32(ses->digest_length);
 853                 /* last element*/
 854                 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
 855         } else {
 856                 SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)start_addr,
 857                         sym->auth.data.offset, sym->auth.data.length);
 858         }
 859         return ctx;
 860 }
 861
 862 static inline struct caam_jr_op_ctx *
 863 build_cipher_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
 864 {
 865         struct rte_crypto_sym_op *sym = op->sym;
 866         struct rte_mbuf *mbuf = sym->m_src;
 867         struct caam_jr_op_ctx *ctx;
 868         struct sec4_sg_entry *sg, *in_sg;
 869         int length;
 870         struct sec_cdb *cdb;
 871         uint64_t sdesc_offset;
 872         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 873                         ses->iv.offset);
 874         struct sec_job_descriptor_t *jobdescr;
 875         uint8_t reg_segs;
 876
 877         if (sym->m_dst) {
 878                 mbuf = sym->m_dst;
 879                 reg_segs = mbuf->nb_segs + sym->m_src->nb_segs + 2;
 880         } else {
 881                 mbuf = sym->m_src;
 882                 reg_segs = mbuf->nb_segs * 2 + 2;
 883         }
 884
 885         if (reg_segs > MAX_SG_ENTRIES) {
 886                 CAAM_JR_DP_ERR("Cipher: Max sec segs supported is %d",
 887                                 MAX_SG_ENTRIES);
 888                 return NULL;
 889         }
 890
 891         ctx = caam_jr_alloc_ctx(ses);
 892         if (!ctx)
 893                 return NULL;
 894
 895         ctx->op = op;
 896         cdb = ses->cdb;
 897         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
 898
 899         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
 900
 901         SEC_JD_INIT(jobdescr);
 902         SEC_JD_SET_SD(jobdescr,
 903                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
 904                 cdb->sh_hdr.hi.field.idlen);
 905
 906 #if CAAM_JR_DBG
 907         CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d",
 908                         sym->m_src->data_off, sym->cipher.data.offset,
 909                         sym->cipher.data.length, ses->iv.length);
 910 #endif
 911         /* output */
 912         if (sym->m_dst)
 913                 mbuf = sym->m_dst;
 914         else
 915                 mbuf = sym->m_src;
 916
 917         sg = &ctx->sg[0];
 918         length = sym->cipher.data.length;
 919
 920         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
 921                 + sym->cipher.data.offset);
 922         sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset);
 923
 924         /* Successive segs */
 925         mbuf = mbuf->next;
 926         while (mbuf) {
 927                 sg++;
 928                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
 929                 sg->len = cpu_to_caam32(mbuf->data_len);
 930                 mbuf = mbuf->next;
 931         }
 932         /* last element*/
 933         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
 934
 935         SEC_JD_SET_OUT_PTR(jobdescr,
 936                         (uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0,
 937                         length);
 938         /*enabling sg bit */
 939         (jobdescr)->seq_out.command.word  |= 0x01000000;
 940
 941         /*input */
 942         sg++;
 943         mbuf = sym->m_src;
 944         in_sg = sg;
 945
 946         length = sym->cipher.data.length + ses->iv.length;
 947
 948         /* IV */
 949         sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
 950         sg->len = cpu_to_caam32(ses->iv.length);
 951
 952         /* 1st seg */
 953         sg++;
 954         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
 955                                 + sym->cipher.data.offset);
 956         sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset);
 957
 958         /* Successive segs */
 959         mbuf = mbuf->next;
 960         while (mbuf) {
 961                 sg++;
 962                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
 963                 sg->len = cpu_to_caam32(mbuf->data_len);
 964                 mbuf = mbuf->next;
 965         }
 966         /* last element*/
 967         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
 968
 969
 970         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, in_sg), 0,
 971                                 length);
 972         /*enabling sg bit */
 973         (jobdescr)->seq_in.command.word  |= 0x01000000;
 974
 975         return ctx;
 976 }
 977
 978 static inline struct caam_jr_op_ctx *
 979 build_cipher_only(struct rte_crypto_op *op, struct caam_jr_session *ses)
 980 {
 981         struct rte_crypto_sym_op *sym = op->sym;
 982         struct caam_jr_op_ctx *ctx;
 983         struct sec4_sg_entry *sg;
 984         rte_iova_t src_start_addr, dst_start_addr;
 985         struct sec_cdb *cdb;
 986         uint64_t sdesc_offset;
 987         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 988                         ses->iv.offset);
 989         struct sec_job_descriptor_t *jobdescr;
 990
 991         ctx = caam_jr_alloc_ctx(ses);
 992         if (!ctx)
 993                 return NULL;
 994
 995         ctx->op = op;
 996         cdb = ses->cdb;
 997         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
 998
 999         src_start_addr = rte_pktmbuf_iova(sym->m_src);
1000         if (sym->m_dst)
1001                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
1002         else
1003                 dst_start_addr = src_start_addr;
1004
1005         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1006
1007         SEC_JD_INIT(jobdescr);
1008         SEC_JD_SET_SD(jobdescr,
1009                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1010                 cdb->sh_hdr.hi.field.idlen);
1011
1012 #if CAAM_JR_DBG
1013         CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d",
1014                         sym->m_src->data_off, sym->cipher.data.offset,
1015                         sym->cipher.data.length, ses->iv.length);
1016 #endif
1017         /* output */
1018         SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr,
1019                         sym->cipher.data.offset,
1020                         sym->cipher.data.length + ses->iv.length);
1021
1022         /*input */
1023         sg = &ctx->sg[0];
1024         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0,
1025                                 sym->cipher.data.length + ses->iv.length);
1026         /*enabling sg bit */
1027         (jobdescr)->seq_in.command.word  |= 0x01000000;
1028
1029         sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1030         sg->len = cpu_to_caam32(ses->iv.length);
1031
1032         sg = &ctx->sg[1];
1033         sg->ptr = cpu_to_caam64(src_start_addr + sym->cipher.data.offset);
1034         sg->len = cpu_to_caam32(sym->cipher.data.length);
1035         /* last element*/
1036         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1037
1038         return ctx;
1039 }
1040
1041 /* For decapsulation:
1042  *     Input:
1043  * +----+----------------+--------------------------------+-----+
1044  * | IV | Auth-only data | Authenticated & Encrypted data | ICV |
1045  * +----+----------------+--------------------------------+-----+
1046  *     Output:
1047  * +----+--------------------------+
1048  * | Decrypted & authenticated data |
1049  * +----+--------------------------+
1050  */
1051
1052 static inline struct caam_jr_op_ctx *
1053 build_cipher_auth_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
1054 {
1055         struct rte_crypto_sym_op *sym = op->sym;
1056         struct caam_jr_op_ctx *ctx;
1057         struct sec4_sg_entry *sg, *out_sg, *in_sg;
1058         struct rte_mbuf *mbuf;
1059         uint32_t length = 0;
1060         struct sec_cdb *cdb;
1061         uint64_t sdesc_offset;
1062         uint8_t req_segs;
1063         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1064                         ses->iv.offset);
1065         struct sec_job_descriptor_t *jobdescr;
1066         uint16_t auth_hdr_len = sym->cipher.data.offset -
1067                         sym->auth.data.offset;
1068         uint16_t auth_tail_len = sym->auth.data.length -
1069                         sym->cipher.data.length - auth_hdr_len;
1070         uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
1071
1072         if (sym->m_dst) {
1073                 mbuf = sym->m_dst;
1074                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3;
1075         } else {
1076                 mbuf = sym->m_src;
1077                 req_segs = mbuf->nb_segs * 2 + 3;
1078         }
1079
1080         if (req_segs > MAX_SG_ENTRIES) {
1081                 CAAM_JR_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
1082                                 MAX_SG_ENTRIES);
1083                 return NULL;
1084         }
1085
1086         ctx = caam_jr_alloc_ctx(ses);
1087         if (!ctx)
1088                 return NULL;
1089
1090         ctx->op = op;
1091         cdb = ses->cdb;
1092         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
1093
1094         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1095
1096         SEC_JD_INIT(jobdescr);
1097         SEC_JD_SET_SD(jobdescr,
1098                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1099                 cdb->sh_hdr.hi.field.idlen);
1100
1101         /* output */
1102         if (sym->m_dst)
1103                 mbuf = sym->m_dst;
1104         else
1105                 mbuf = sym->m_src;
1106
1107         out_sg = &ctx->sg[0];
1108         if (is_encode(ses))
1109                 length = sym->auth.data.length + ses->digest_length;
1110         else
1111                 length = sym->auth.data.length;
1112
1113         sg = &ctx->sg[0];
1114
1115         /* 1st seg */
1116         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
1117                 + sym->auth.data.offset);
1118         sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
1119
1120         /* Successive segs */
1121         mbuf = mbuf->next;
1122         while (mbuf) {
1123                 sg++;
1124                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
1125                 sg->len = cpu_to_caam32(mbuf->data_len);
1126                 mbuf = mbuf->next;
1127         }
1128
1129         if (is_encode(ses)) {
1130                 /* set auth output */
1131                 sg++;
1132                 sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr);
1133                 sg->len = cpu_to_caam32(ses->digest_length);
1134         }
1135         /* last element*/
1136         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1137
1138         SEC_JD_SET_OUT_PTR(jobdescr,
1139                            (uint64_t)caam_jr_dma_vtop(out_sg), 0, length);
1140         /* set sg bit */
1141         (jobdescr)->seq_out.command.word  |= 0x01000000;
1142
1143         /* input */
1144         sg++;
1145         mbuf = sym->m_src;
1146         in_sg = sg;
1147         if (is_encode(ses))
1148                 length = ses->iv.length + sym->auth.data.length;
1149         else
1150                 length = ses->iv.length + sym->auth.data.length
1151                                                 + ses->digest_length;
1152
1153         sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1154         sg->len = cpu_to_caam32(ses->iv.length);
1155
1156         sg++;
1157         /* 1st seg */
1158         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
1159                 + sym->auth.data.offset);
1160         sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
1161
1162         /* Successive segs */
1163         mbuf = mbuf->next;
1164         while (mbuf) {
1165                 sg++;
1166                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
1167                 sg->len = cpu_to_caam32(mbuf->data_len);
1168                 mbuf = mbuf->next;
1169         }
1170
1171         if (is_decode(ses)) {
1172                 sg++;
1173                 rte_memcpy(ctx->digest, sym->auth.digest.data,
1174                        ses->digest_length);
1175                 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest));
1176                 sg->len = cpu_to_caam32(ses->digest_length);
1177         }
1178         /* last element*/
1179         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1180
1181         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(in_sg), 0,
1182                                 length);
1183         /* set sg bit */
1184         (jobdescr)->seq_in.command.word  |= 0x01000000;
1185         /* Auth_only_len is set as 0 in descriptor and it is
1186          * overwritten here in the jd which will update
1187          * the DPOVRD reg.
1188          */
1189         if (auth_only_len)
1190                 /* set sg bit */
1191                 (jobdescr)->dpovrd = 0x80000000 | auth_only_len;
1192
1193         return ctx;
1194 }
1195
1196 static inline struct caam_jr_op_ctx *
1197 build_cipher_auth(struct rte_crypto_op *op, struct caam_jr_session *ses)
1198 {
1199         struct rte_crypto_sym_op *sym = op->sym;
1200         struct caam_jr_op_ctx *ctx;
1201         struct sec4_sg_entry *sg;
1202         rte_iova_t src_start_addr, dst_start_addr;
1203         uint32_t length = 0;
1204         struct sec_cdb *cdb;
1205         uint64_t sdesc_offset;
1206         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1207                         ses->iv.offset);
1208         struct sec_job_descriptor_t *jobdescr;
1209         uint16_t auth_hdr_len = sym->cipher.data.offset -
1210                         sym->auth.data.offset;
1211         uint16_t auth_tail_len = sym->auth.data.length -
1212                         sym->cipher.data.length - auth_hdr_len;
1213         uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
1214
1215         src_start_addr = rte_pktmbuf_iova(sym->m_src);
1216         if (sym->m_dst)
1217                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
1218         else
1219                 dst_start_addr = src_start_addr;
1220
1221         ctx = caam_jr_alloc_ctx(ses);
1222         if (!ctx)
1223                 return NULL;
1224
1225         ctx->op = op;
1226         cdb = ses->cdb;
1227         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
1228
1229         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1230
1231         SEC_JD_INIT(jobdescr);
1232         SEC_JD_SET_SD(jobdescr,
1233                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1234                 cdb->sh_hdr.hi.field.idlen);
1235
1236         /* input */
1237         sg = &ctx->sg[0];
1238         if (is_encode(ses)) {
1239                 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1240                 sg->len = cpu_to_caam32(ses->iv.length);
1241                 length += ses->iv.length;
1242
1243                 sg++;
1244                 sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset);
1245                 sg->len = cpu_to_caam32(sym->auth.data.length);
1246                 length += sym->auth.data.length;
1247                 /* last element*/
1248                 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1249         } else {
1250                 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1251                 sg->len = cpu_to_caam32(ses->iv.length);
1252                 length += ses->iv.length;
1253
1254                 sg++;
1255                 sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset);
1256                 sg->len = cpu_to_caam32(sym->auth.data.length);
1257                 length += sym->auth.data.length;
1258
1259                 rte_memcpy(ctx->digest, sym->auth.digest.data,
1260                        ses->digest_length);
1261                 sg++;
1262                 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest));
1263                 sg->len = cpu_to_caam32(ses->digest_length);
1264                 length += ses->digest_length;
1265                 /* last element*/
1266                 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1267         }
1268
1269         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(&ctx->sg[0]), 0,
1270                                 length);
1271         /* set sg bit */
1272         (jobdescr)->seq_in.command.word  |= 0x01000000;
1273
1274         /* output */
1275         sg = &ctx->sg[6];
1276
1277         sg->ptr = cpu_to_caam64(dst_start_addr + sym->cipher.data.offset);
1278         sg->len = cpu_to_caam32(sym->cipher.data.length);
1279         length = sym->cipher.data.length;
1280
1281         if (is_encode(ses)) {
1282                 /* set auth output */
1283                 sg++;
1284                 sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr);
1285                 sg->len = cpu_to_caam32(ses->digest_length);
1286                 length += ses->digest_length;
1287         }
1288         /* last element*/
1289         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1290
1291         SEC_JD_SET_OUT_PTR(jobdescr,
1292                            (uint64_t)caam_jr_dma_vtop(&ctx->sg[6]), 0, length);
1293         /* set sg bit */
1294         (jobdescr)->seq_out.command.word  |= 0x01000000;
1295
1296         /* Auth_only_len is set as 0 in descriptor and it is
1297          * overwritten here in the jd which will update
1298          * the DPOVRD reg.
1299          */
1300         if (auth_only_len)
1301                 /* set sg bit */
1302                 (jobdescr)->dpovrd = 0x80000000 | auth_only_len;
1303
1304         return ctx;
1305 }
1306
1307 static inline struct caam_jr_op_ctx *
1308 build_proto(struct rte_crypto_op *op, struct caam_jr_session *ses)
1309 {
1310         struct rte_crypto_sym_op *sym = op->sym;
1311         struct caam_jr_op_ctx *ctx = NULL;
1312         phys_addr_t src_start_addr, dst_start_addr;
1313         struct sec_cdb *cdb;
1314         uint64_t sdesc_offset;
1315         struct sec_job_descriptor_t *jobdescr;
1316
1317         ctx = caam_jr_alloc_ctx(ses);
1318         if (!ctx)
1319                 return NULL;
1320         ctx->op = op;
1321
1322         src_start_addr = rte_pktmbuf_iova(sym->m_src);
1323         if (sym->m_dst)
1324                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
1325         else
1326                 dst_start_addr = src_start_addr;
1327
1328         cdb = ses->cdb;
1329         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
1330
1331         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1332
1333         SEC_JD_INIT(jobdescr);
1334         SEC_JD_SET_SD(jobdescr,
1335                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1336                         cdb->sh_hdr.hi.field.idlen);
1337
1338         /* output */
1339         SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr, 0,
1340                         sym->m_src->buf_len - sym->m_src->data_off);
1341         /* input */
1342         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)src_start_addr, 0,
1343                         sym->m_src->pkt_len);
1344         sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
1345
1346         return ctx;
1347 }
1348
1349 static int
1350 caam_jr_enqueue_op(struct rte_crypto_op *op, struct caam_jr_qp *qp)
1351 {
1352         struct sec_job_ring_t *ring = qp->ring;
1353         struct caam_jr_session *ses;
1354         struct caam_jr_op_ctx *ctx = NULL;
1355         struct sec_job_descriptor_t *jobdescr __rte_unused;
1356
1357         switch (op->sess_type) {
1358         case RTE_CRYPTO_OP_WITH_SESSION:
1359                 ses = (struct caam_jr_session *)
1360                 get_sym_session_private_data(op->sym->session,
1361                                         cryptodev_driver_id);
1362                 break;
1363         case RTE_CRYPTO_OP_SECURITY_SESSION:
1364                 ses = (struct caam_jr_session *)
1365                         get_sec_session_private_data(
1366                                         op->sym->sec_session);
1367                 break;
1368         default:
1369                 CAAM_JR_DP_ERR("sessionless crypto op not supported");
1370                 qp->tx_errs++;
1371                 return -1;
1372         }
1373
1374         if (unlikely(!ses->qp || ses->qp != qp)) {
1375                 CAAM_JR_DP_DEBUG("Old:sess->qp=%p New qp = %p\n", ses->qp, qp);
1376                 ses->qp = qp;
1377                 caam_jr_prep_cdb(ses);
1378         }
1379
1380         if (rte_pktmbuf_is_contiguous(op->sym->m_src)) {
1381                 if (is_auth_cipher(ses))
1382                         ctx = build_cipher_auth(op, ses);
1383                 else if (is_aead(ses))
1384                         goto err1;
1385                 else if (is_auth_only(ses))
1386                         ctx = build_auth_only(op, ses);
1387                 else if (is_cipher_only(ses))
1388                         ctx = build_cipher_only(op, ses);
1389                 else if (is_proto_ipsec(ses))
1390                         ctx = build_proto(op, ses);
1391         } else {
1392                 if (is_auth_cipher(ses))
1393                         ctx = build_cipher_auth_sg(op, ses);
1394                 else if (is_aead(ses))
1395                         goto err1;
1396                 else if (is_auth_only(ses))
1397                         ctx = build_auth_only_sg(op, ses);
1398                 else if (is_cipher_only(ses))
1399                         ctx = build_cipher_only_sg(op, ses);
1400         }
1401 err1:
1402         if (unlikely(!ctx)) {
1403                 qp->tx_errs++;
1404                 CAAM_JR_ERR("not supported sec op");
1405                 return -1;
1406         }
1407 #if CAAM_JR_DBG
1408         if (is_decode(ses))
1409                 rte_hexdump(stdout, "DECODE",
1410                         rte_pktmbuf_mtod(op->sym->m_src, void *),
1411                         rte_pktmbuf_data_len(op->sym->m_src));
1412         else
1413                 rte_hexdump(stdout, "ENCODE",
1414                         rte_pktmbuf_mtod(op->sym->m_src, void *),
1415                         rte_pktmbuf_data_len(op->sym->m_src));
1416
1417         printf("\n JD before conversion\n");
1418         for (int i = 0; i < 12; i++)
1419                 printf("\n 0x%08x", ctx->jobdes.desc[i]);
1420 #endif
1421
1422         CAAM_JR_DP_DEBUG("Jr[%p] pi[%d] ci[%d].Before sending desc",
1423                       ring, ring->pidx, ring->cidx);
1424
1425         /* todo - do we want to retry */
1426         if (SEC_JOB_RING_IS_FULL(ring->pidx, ring->cidx,
1427                          SEC_JOB_RING_SIZE, SEC_JOB_RING_SIZE)) {
1428                 CAAM_JR_DP_DEBUG("Ring FULL Jr[%p] pi[%d] ci[%d].Size = %d",
1429                               ring, ring->pidx, ring->cidx, SEC_JOB_RING_SIZE);
1430                 caam_jr_op_ending(ctx);
1431                 qp->tx_ring_full++;
1432                 return -EBUSY;
1433         }
1434
1435 #if CORE_BYTE_ORDER != CAAM_BYTE_ORDER
1436         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1437
1438         jobdescr->deschdr.command.word =
1439                 cpu_to_caam32(jobdescr->deschdr.command.word);
1440         jobdescr->sd_ptr = cpu_to_caam64(jobdescr->sd_ptr);
1441         jobdescr->seq_out.command.word =
1442                 cpu_to_caam32(jobdescr->seq_out.command.word);
1443         jobdescr->seq_out_ptr = cpu_to_caam64(jobdescr->seq_out_ptr);
1444         jobdescr->out_ext_length = cpu_to_caam32(jobdescr->out_ext_length);
1445         jobdescr->seq_in.command.word =
1446                 cpu_to_caam32(jobdescr->seq_in.command.word);
1447         jobdescr->seq_in_ptr = cpu_to_caam64(jobdescr->seq_in_ptr);
1448         jobdescr->in_ext_length = cpu_to_caam32(jobdescr->in_ext_length);
1449         jobdescr->load_dpovrd.command.word =
1450                 cpu_to_caam32(jobdescr->load_dpovrd.command.word);
1451         jobdescr->dpovrd = cpu_to_caam32(jobdescr->dpovrd);
1452 #endif
1453
1454         /* Set ptr in input ring to current descriptor  */
1455         sec_write_addr(&ring->input_ring[ring->pidx],
1456                         (phys_addr_t)caam_jr_vtop_ctx(ctx, ctx->jobdes.desc));
1457         rte_smp_wmb();
1458
1459         /* Notify HW that a new job is enqueued */
1460         hw_enqueue_desc_on_job_ring(ring);
1461
1462         /* increment the producer index for the current job ring */
1463         ring->pidx = SEC_CIRCULAR_COUNTER(ring->pidx, SEC_JOB_RING_SIZE);
1464
1465         return 0;
1466 }
1467
1468 static uint16_t
1469 caam_jr_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1470                        uint16_t nb_ops)
1471 {
1472         /* Function to transmit the frames to given device and queuepair */
1473         uint32_t loop;
1474         int32_t ret;
1475         struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp;
1476         uint16_t num_tx = 0;
1477         /*Prepare each packet which is to be sent*/
1478         for (loop = 0; loop < nb_ops; loop++) {
1479                 ret = caam_jr_enqueue_op(ops[loop], jr_qp);
1480                 if (!ret)
1481                         num_tx++;
1482         }
1483
1484         jr_qp->tx_pkts += num_tx;
1485
1486         return num_tx;
1487 }
1488
1489 /* Release queue pair */
1490 static int
1491 caam_jr_queue_pair_release(struct rte_cryptodev *dev,
1492                            uint16_t qp_id)
1493 {
1494         struct sec_job_ring_t *internals;
1495         struct caam_jr_qp *qp = NULL;
1496
1497         PMD_INIT_FUNC_TRACE();
1498         CAAM_JR_DEBUG("dev =%p, queue =%d", dev, qp_id);
1499
1500         internals = dev->data->dev_private;
1501         if (qp_id >= internals->max_nb_queue_pairs) {
1502                 CAAM_JR_ERR("Max supported qpid %d",
1503                              internals->max_nb_queue_pairs);
1504                 return -EINVAL;
1505         }
1506
1507         qp = &internals->qps[qp_id];
1508         qp->ring = NULL;
1509         dev->data->queue_pairs[qp_id] = NULL;
1510
1511         return 0;
1512 }
1513
1514 /* Setup a queue pair */
1515 static int
1516 caam_jr_queue_pair_setup(
1517                 struct rte_cryptodev *dev, uint16_t qp_id,
1518                 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
1519                 __rte_unused int socket_id)
1520 {
1521         struct sec_job_ring_t *internals;
1522         struct caam_jr_qp *qp = NULL;
1523
1524         PMD_INIT_FUNC_TRACE();
1525         CAAM_JR_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);
1526
1527         internals = dev->data->dev_private;
1528         if (qp_id >= internals->max_nb_queue_pairs) {
1529                 CAAM_JR_ERR("Max supported qpid %d",
1530                              internals->max_nb_queue_pairs);
1531                 return -EINVAL;
1532         }
1533
1534         qp = &internals->qps[qp_id];
1535         qp->ring = internals;
1536         dev->data->queue_pairs[qp_id] = qp;
1537
1538         return 0;
1539 }
1540
1541 /* Returns the size of the aesni gcm session structure */
1542 static unsigned int
1543 caam_jr_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
1544 {
1545         PMD_INIT_FUNC_TRACE();
1546
1547         return sizeof(struct caam_jr_session);
1548 }
1549
1550 static int
1551 caam_jr_cipher_init(struct rte_cryptodev *dev __rte_unused,
1552                     struct rte_crypto_sym_xform *xform,
1553                     struct caam_jr_session *session)
1554 {
1555         session->cipher_alg = xform->cipher.algo;
1556         session->iv.length = xform->cipher.iv.length;
1557         session->iv.offset = xform->cipher.iv.offset;
1558         session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
1559                                                RTE_CACHE_LINE_SIZE);
1560         if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
1561                 CAAM_JR_ERR("No Memory for cipher key\n");
1562                 return -ENOMEM;
1563         }
1564         session->cipher_key.length = xform->cipher.key.length;
1565
1566         memcpy(session->cipher_key.data, xform->cipher.key.data,
1567                xform->cipher.key.length);
1568         session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
1569                         DIR_ENC : DIR_DEC;
1570
1571         return 0;
1572 }
1573
1574 static int
1575 caam_jr_auth_init(struct rte_cryptodev *dev __rte_unused,
1576                   struct rte_crypto_sym_xform *xform,
1577                   struct caam_jr_session *session)
1578 {
1579         session->auth_alg = xform->auth.algo;
1580         session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
1581                                              RTE_CACHE_LINE_SIZE);
1582         if (session->auth_key.data == NULL && xform->auth.key.length > 0) {
1583                 CAAM_JR_ERR("No Memory for auth key\n");
1584                 return -ENOMEM;
1585         }
1586         session->auth_key.length = xform->auth.key.length;
1587         session->digest_length = xform->auth.digest_length;
1588
1589         memcpy(session->auth_key.data, xform->auth.key.data,
1590                xform->auth.key.length);
1591         session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
1592                         DIR_ENC : DIR_DEC;
1593
1594         return 0;
1595 }
1596
1597 static int
1598 caam_jr_aead_init(struct rte_cryptodev *dev __rte_unused,
1599                   struct rte_crypto_sym_xform *xform,
1600                   struct caam_jr_session *session)
1601 {
1602         session->aead_alg = xform->aead.algo;
1603         session->iv.length = xform->aead.iv.length;
1604         session->iv.offset = xform->aead.iv.offset;
1605         session->auth_only_len = xform->aead.aad_length;
1606         session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length,
1607                                              RTE_CACHE_LINE_SIZE);
1608         if (session->aead_key.data == NULL && xform->aead.key.length > 0) {
1609                 CAAM_JR_ERR("No Memory for aead key\n");
1610                 return -ENOMEM;
1611         }
1612         session->aead_key.length = xform->aead.key.length;
1613         session->digest_length = xform->aead.digest_length;
1614
1615         memcpy(session->aead_key.data, xform->aead.key.data,
1616                xform->aead.key.length);
1617         session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
1618                         DIR_ENC : DIR_DEC;
1619
1620         return 0;
1621 }
1622
1623 static int
1624 caam_jr_set_session_parameters(struct rte_cryptodev *dev,
1625                                struct rte_crypto_sym_xform *xform, void *sess)
1626 {
1627         struct sec_job_ring_t *internals = dev->data->dev_private;
1628         struct caam_jr_session *session = sess;
1629
1630         PMD_INIT_FUNC_TRACE();
1631
1632         if (unlikely(sess == NULL)) {
1633                 CAAM_JR_ERR("invalid session struct");
1634                 return -EINVAL;
1635         }
1636
1637         /* Default IV length = 0 */
1638         session->iv.length = 0;
1639
1640         /* Cipher Only */
1641         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
1642                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1643                 caam_jr_cipher_init(dev, xform, session);
1644
1645         /* Authentication Only */
1646         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1647                    xform->next == NULL) {
1648                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
1649                 caam_jr_auth_init(dev, xform, session);
1650
1651         /* Cipher then Authenticate */
1652         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1653                    xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
1654                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1655                         caam_jr_cipher_init(dev, xform, session);
1656                         caam_jr_auth_init(dev, xform->next, session);
1657                 } else {
1658                         CAAM_JR_ERR("Not supported: Auth then Cipher");
1659                         goto err1;
1660                 }
1661
1662         /* Authenticate then Cipher */
1663         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1664                    xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
1665                 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
1666                         caam_jr_auth_init(dev, xform, session);
1667                         caam_jr_cipher_init(dev, xform->next, session);
1668                 } else {
1669                         CAAM_JR_ERR("Not supported: Auth then Cipher");
1670                         goto err1;
1671                 }
1672
1673         /* AEAD operation for AES-GCM kind of Algorithms */
1674         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
1675                    xform->next == NULL) {
1676                 caam_jr_aead_init(dev, xform, session);
1677
1678         } else {
1679                 CAAM_JR_ERR("Invalid crypto type");
1680                 return -EINVAL;
1681         }
1682         session->ctx_pool = internals->ctx_pool;
1683
1684         return 0;
1685
1686 err1:
1687         rte_free(session->cipher_key.data);
1688         rte_free(session->auth_key.data);
1689         memset(session, 0, sizeof(struct caam_jr_session));
1690
1691         return -EINVAL;
1692 }
1693
1694 static int
1695 caam_jr_sym_session_configure(struct rte_cryptodev *dev,
1696                               struct rte_crypto_sym_xform *xform,
1697                               struct rte_cryptodev_sym_session *sess,
1698                               struct rte_mempool *mempool)
1699 {
1700         void *sess_private_data;
1701         int ret;
1702
1703         PMD_INIT_FUNC_TRACE();
1704
1705         if (rte_mempool_get(mempool, &sess_private_data)) {
1706                 CAAM_JR_ERR("Couldn't get object from session mempool");
1707                 return -ENOMEM;
1708         }
1709
1710         memset(sess_private_data, 0, sizeof(struct caam_jr_session));
1711         ret = caam_jr_set_session_parameters(dev, xform, sess_private_data);
1712         if (ret != 0) {
1713                 CAAM_JR_ERR("failed to configure session parameters");
1714                 /* Return session to mempool */
1715                 rte_mempool_put(mempool, sess_private_data);
1716                 return ret;
1717         }
1718
1719         set_sym_session_private_data(sess, dev->driver_id, sess_private_data);
1720
1721         return 0;
1722 }
1723
1724 /* Clear the memory of session so it doesn't leave key material behind */
1725 static void
1726 caam_jr_sym_session_clear(struct rte_cryptodev *dev,
1727                 struct rte_cryptodev_sym_session *sess)
1728 {
1729         uint8_t index = dev->driver_id;
1730         void *sess_priv = get_sym_session_private_data(sess, index);
1731         struct caam_jr_session *s = (struct caam_jr_session *)sess_priv;
1732
1733         PMD_INIT_FUNC_TRACE();
1734
1735         if (sess_priv) {
1736                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1737
1738                 rte_free(s->cipher_key.data);
1739                 rte_free(s->auth_key.data);
1740                 memset(s, 0, sizeof(struct caam_jr_session));
1741                 set_sym_session_private_data(sess, index, NULL);
1742                 rte_mempool_put(sess_mp, sess_priv);
1743         }
1744 }
1745
1746 static int
1747 caam_jr_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
1748                           struct rte_security_session_conf *conf,
1749                           void *sess)
1750 {
1751         struct sec_job_ring_t *internals = dev->data->dev_private;
1752         struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
1753         struct rte_crypto_auth_xform *auth_xform;
1754         struct rte_crypto_cipher_xform *cipher_xform;
1755         struct caam_jr_session *session = (struct caam_jr_session *)sess;
1756
1757         PMD_INIT_FUNC_TRACE();
1758
1759         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
1760                 cipher_xform = &conf->crypto_xform->cipher;
1761                 auth_xform = &conf->crypto_xform->next->auth;
1762         } else {
1763                 auth_xform = &conf->crypto_xform->auth;
1764                 cipher_xform = &conf->crypto_xform->next->cipher;
1765         }
1766         session->proto_alg = conf->protocol;
1767         session->cipher_key.data = rte_zmalloc(NULL,
1768                                                cipher_xform->key.length,
1769                                                RTE_CACHE_LINE_SIZE);
1770         if (session->cipher_key.data == NULL &&
1771                         cipher_xform->key.length > 0) {
1772                 CAAM_JR_ERR("No Memory for cipher key\n");
1773                 return -ENOMEM;
1774         }
1775
1776         session->cipher_key.length = cipher_xform->key.length;
1777         session->auth_key.data = rte_zmalloc(NULL,
1778                                         auth_xform->key.length,
1779                                         RTE_CACHE_LINE_SIZE);
1780         if (session->auth_key.data == NULL &&
1781                         auth_xform->key.length > 0) {
1782                 CAAM_JR_ERR("No Memory for auth key\n");
1783                 rte_free(session->cipher_key.data);
1784                 return -ENOMEM;
1785         }
1786         session->auth_key.length = auth_xform->key.length;
1787         memcpy(session->cipher_key.data, cipher_xform->key.data,
1788                         cipher_xform->key.length);
1789         memcpy(session->auth_key.data, auth_xform->key.data,
1790                         auth_xform->key.length);
1791
1792         switch (auth_xform->algo) {
1793         case RTE_CRYPTO_AUTH_SHA1_HMAC:
1794                 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
1795                 break;
1796         case RTE_CRYPTO_AUTH_MD5_HMAC:
1797                 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
1798                 break;
1799         case RTE_CRYPTO_AUTH_SHA256_HMAC:
1800                 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
1801                 break;
1802         case RTE_CRYPTO_AUTH_SHA384_HMAC:
1803                 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
1804                 break;
1805         case RTE_CRYPTO_AUTH_SHA512_HMAC:
1806                 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
1807                 break;
1808         case RTE_CRYPTO_AUTH_AES_CMAC:
1809                 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
1810                 break;
1811         case RTE_CRYPTO_AUTH_NULL:
1812                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1813                 break;
1814         case RTE_CRYPTO_AUTH_SHA224_HMAC:
1815         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
1816         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
1817         case RTE_CRYPTO_AUTH_SHA1:
1818         case RTE_CRYPTO_AUTH_SHA256:
1819         case RTE_CRYPTO_AUTH_SHA512:
1820         case RTE_CRYPTO_AUTH_SHA224:
1821         case RTE_CRYPTO_AUTH_SHA384:
1822         case RTE_CRYPTO_AUTH_MD5:
1823         case RTE_CRYPTO_AUTH_AES_GMAC:
1824         case RTE_CRYPTO_AUTH_KASUMI_F9:
1825         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
1826         case RTE_CRYPTO_AUTH_ZUC_EIA3:
1827                 CAAM_JR_ERR("Crypto: Unsupported auth alg %u\n",
1828                         auth_xform->algo);
1829                 goto out;
1830         default:
1831                 CAAM_JR_ERR("Crypto: Undefined Auth specified %u\n",
1832                         auth_xform->algo);
1833                 goto out;
1834         }
1835
1836         switch (cipher_xform->algo) {
1837         case RTE_CRYPTO_CIPHER_AES_CBC:
1838                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
1839                 break;
1840         case RTE_CRYPTO_CIPHER_3DES_CBC:
1841                 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
1842                 break;
1843         case RTE_CRYPTO_CIPHER_AES_CTR:
1844                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
1845                 break;
1846         case RTE_CRYPTO_CIPHER_NULL:
1847         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
1848         case RTE_CRYPTO_CIPHER_3DES_ECB:
1849         case RTE_CRYPTO_CIPHER_AES_ECB:
1850         case RTE_CRYPTO_CIPHER_KASUMI_F8:
1851                 CAAM_JR_ERR("Crypto: Unsupported Cipher alg %u\n",
1852                         cipher_xform->algo);
1853                 goto out;
1854         default:
1855                 CAAM_JR_ERR("Crypto: Undefined Cipher specified %u\n",
1856                         cipher_xform->algo);
1857                 goto out;
1858         }
1859
1860         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
1861                 memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) +
1862                                 sizeof(session->ip4_hdr));
1863                 session->ip4_hdr.ip_v = IPVERSION;
1864                 session->ip4_hdr.ip_hl = 5;
1865                 session->ip4_hdr.ip_len = rte_cpu_to_be_16(
1866                                                 sizeof(session->ip4_hdr));
1867                 session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
1868                 session->ip4_hdr.ip_id = 0;
1869                 session->ip4_hdr.ip_off = 0;
1870                 session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
1871                 session->ip4_hdr.ip_p = (ipsec_xform->proto ==
1872                                 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP
1873                                 : IPPROTO_AH;
1874                 session->ip4_hdr.ip_sum = 0;
1875                 session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
1876                 session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
1877                 session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
1878                                                 (void *)&session->ip4_hdr,
1879                                                 sizeof(struct ip));
1880
1881                 session->encap_pdb.options =
1882                         (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
1883                         PDBOPTS_ESP_OIHI_PDB_INL |
1884                         PDBOPTS_ESP_IVSRC |
1885                         PDBHMO_ESP_ENCAP_DTTL;
1886                 if (ipsec_xform->options.esn)
1887                         session->encap_pdb.options |= PDBOPTS_ESP_ESN;
1888                 session->encap_pdb.spi = ipsec_xform->spi;
1889                 session->encap_pdb.ip_hdr_len = sizeof(struct ip);
1890
1891                 session->dir = DIR_ENC;
1892         } else if (ipsec_xform->direction ==
1893                         RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
1894                 memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
1895                 session->decap_pdb.options = sizeof(struct ip) << 16;
1896                 if (ipsec_xform->options.esn)
1897                         session->decap_pdb.options |= PDBOPTS_ESP_ESN;
1898                 session->dir = DIR_DEC;
1899         } else
1900                 goto out;
1901         session->ctx_pool = internals->ctx_pool;
1902
1903         return 0;
1904 out:
1905         rte_free(session->auth_key.data);
1906         rte_free(session->cipher_key.data);
1907         memset(session, 0, sizeof(struct caam_jr_session));
1908         return -1;
1909 }
1910
1911 static int
1912 caam_jr_security_session_create(void *dev,
1913                                 struct rte_security_session_conf *conf,
1914                                 struct rte_security_session *sess,
1915                                 struct rte_mempool *mempool)
1916 {
1917         void *sess_private_data;
1918         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
1919         int ret;
1920
1921         if (rte_mempool_get(mempool, &sess_private_data)) {
1922                 CAAM_JR_ERR("Couldn't get object from session mempool");
1923                 return -ENOMEM;
1924         }
1925
1926         switch (conf->protocol) {
1927         case RTE_SECURITY_PROTOCOL_IPSEC:
1928                 ret = caam_jr_set_ipsec_session(cdev, conf,
1929                                 sess_private_data);
1930                 break;
1931         case RTE_SECURITY_PROTOCOL_MACSEC:
1932                 return -ENOTSUP;
1933         default:
1934                 return -EINVAL;
1935         }
1936         if (ret != 0) {
1937                 CAAM_JR_ERR("failed to configure session parameters");
1938                 /* Return session to mempool */
1939                 rte_mempool_put(mempool, sess_private_data);
1940                 return ret;
1941         }
1942
1943         set_sec_session_private_data(sess, sess_private_data);
1944
1945         return ret;
1946 }
1947
1948 /* Clear the memory of session so it doesn't leave key material behind */
1949 static int
1950 caam_jr_security_session_destroy(void *dev __rte_unused,
1951                                  struct rte_security_session *sess)
1952 {
1953         PMD_INIT_FUNC_TRACE();
1954         void *sess_priv = get_sec_session_private_data(sess);
1955
1956         struct caam_jr_session *s = (struct caam_jr_session *)sess_priv;
1957
1958         if (sess_priv) {
1959                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1960
1961                 rte_free(s->cipher_key.data);
1962                 rte_free(s->auth_key.data);
1963                 memset(sess, 0, sizeof(struct caam_jr_session));
1964                 set_sec_session_private_data(sess, NULL);
1965                 rte_mempool_put(sess_mp, sess_priv);
1966         }
1967         return 0;
1968 }
1969
1970
1971 static int
1972 caam_jr_dev_configure(struct rte_cryptodev *dev,
1973                        struct rte_cryptodev_config *config __rte_unused)
1974 {
1975         char str[20];
1976         struct sec_job_ring_t *internals;
1977
1978         PMD_INIT_FUNC_TRACE();
1979
1980         internals = dev->data->dev_private;
1981         snprintf(str, sizeof(str), "ctx_pool_%d", dev->data->dev_id);
1982         if (!internals->ctx_pool) {
1983                 internals->ctx_pool = rte_mempool_create((const char *)str,
1984                                                 CTX_POOL_NUM_BUFS,
1985                                                 sizeof(struct caam_jr_op_ctx),
1986                                                 CTX_POOL_CACHE_SIZE, 0,
1987                                                 NULL, NULL, NULL, NULL,
1988                                                 SOCKET_ID_ANY, 0);
1989                 if (!internals->ctx_pool) {
1990                         CAAM_JR_ERR("%s create failed\n", str);
1991                         return -ENOMEM;
1992                 }
1993         } else
1994                 CAAM_JR_INFO("mempool already created for dev_id : %d",
1995                                 dev->data->dev_id);
1996
1997         return 0;
1998 }
1999
2000 static int
2001 caam_jr_dev_start(struct rte_cryptodev *dev __rte_unused)
2002 {
2003         PMD_INIT_FUNC_TRACE();
2004         return 0;
2005 }
2006
2007 static void
2008 caam_jr_dev_stop(struct rte_cryptodev *dev __rte_unused)
2009 {
2010         PMD_INIT_FUNC_TRACE();
2011 }
2012
2013 static int
2014 caam_jr_dev_close(struct rte_cryptodev *dev)
2015 {
2016         struct sec_job_ring_t *internals;
2017
2018         PMD_INIT_FUNC_TRACE();
2019
2020         if (dev == NULL)
2021                 return -ENOMEM;
2022
2023         internals = dev->data->dev_private;
2024         rte_mempool_free(internals->ctx_pool);
2025         internals->ctx_pool = NULL;
2026
2027         return 0;
2028 }
2029
2030 static void
2031 caam_jr_dev_infos_get(struct rte_cryptodev *dev,
2032                        struct rte_cryptodev_info *info)
2033 {
2034         struct sec_job_ring_t *internals = dev->data->dev_private;
2035
2036         PMD_INIT_FUNC_TRACE();
2037         if (info != NULL) {
2038                 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
2039                 info->feature_flags = dev->feature_flags;
2040                 info->capabilities = caam_jr_get_cryptodev_capabilities();
2041                 info->sym.max_nb_sessions = internals->max_nb_sessions;
2042                 info->driver_id = cryptodev_driver_id;
2043         }
2044 }
2045
2046 static struct rte_cryptodev_ops caam_jr_ops = {
2047         .dev_configure        = caam_jr_dev_configure,
2048         .dev_start            = caam_jr_dev_start,
2049         .dev_stop             = caam_jr_dev_stop,
2050         .dev_close            = caam_jr_dev_close,
2051         .dev_infos_get        = caam_jr_dev_infos_get,
2052         .stats_get            = caam_jr_stats_get,
2053         .stats_reset          = caam_jr_stats_reset,
2054         .queue_pair_setup     = caam_jr_queue_pair_setup,
2055         .queue_pair_release   = caam_jr_queue_pair_release,
2056         .sym_session_get_size = caam_jr_sym_session_get_size,
2057         .sym_session_configure = caam_jr_sym_session_configure,
2058         .sym_session_clear    = caam_jr_sym_session_clear
2059 };
2060
2061 static struct rte_security_ops caam_jr_security_ops = {
2062         .session_create = caam_jr_security_session_create,
2063         .session_update = NULL,
2064         .session_stats_get = NULL,
2065         .session_destroy = caam_jr_security_session_destroy,
2066         .set_pkt_metadata = NULL,
2067         .capabilities_get = caam_jr_get_security_capabilities
2068 };
2069
2070 /* @brief Flush job rings of any processed descs.
2071  * The processed descs are silently dropped,
2072  * WITHOUT being notified to UA.
2073  */
2074 static void
2075 close_job_ring(struct sec_job_ring_t *job_ring)
2076 {
2077         if (job_ring->irq_fd) {
2078                 /* Producer index is frozen. If consumer index is not equal
2079                  * with producer index, then we have descs to flush.
2080                  */
2081                 while (job_ring->pidx != job_ring->cidx)
2082                         hw_flush_job_ring(job_ring, false, NULL);
2083
2084                 /* free the uio job ring */
2085                 free_job_ring(job_ring->irq_fd);
2086                 job_ring->irq_fd = 0;
2087                 caam_jr_dma_free(job_ring->input_ring);
2088                 caam_jr_dma_free(job_ring->output_ring);
2089                 g_job_rings_no--;
2090         }
2091 }
2092
2093 /** @brief Release the software and hardware resources tied to a job ring.
2094  * @param [in] job_ring The job ring
2095  *
2096  * @retval  0 for success
2097  * @retval  -1 for error
2098  */
2099 static int
2100 shutdown_job_ring(struct sec_job_ring_t *job_ring)
2101 {
2102         int ret = 0;
2103
2104         PMD_INIT_FUNC_TRACE();
2105         ASSERT(job_ring != NULL);
2106         ret = hw_shutdown_job_ring(job_ring);
2107         SEC_ASSERT(ret == 0, ret,
2108                 "Failed to shutdown hardware job ring %p",
2109                 job_ring);
2110
2111         if (job_ring->coalescing_en)
2112                 hw_job_ring_disable_coalescing(job_ring);
2113
2114         if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL) {
2115                 ret = caam_jr_disable_irqs(job_ring->irq_fd);
2116                 SEC_ASSERT(ret == 0, ret,
2117                 "Failed to disable irqs for job ring %p",
2118                 job_ring);
2119         }
2120
2121         return ret;
2122 }
2123
2124 /*
2125  * @brief Release the resources used by the SEC user space driver.
2126  *
2127  * Reset and release SEC's job rings indicated by the User Application at
2128  * init_job_ring() and free any memory allocated internally.
2129  * Call once during application tear down.
2130  *
2131  * @note In case there are any descriptors in-flight (descriptors received by
2132  * SEC driver for processing and for which no response was yet provided to UA),
2133  * the descriptors are discarded without any notifications to User Application.
2134  *
2135  * @retval ::0                  is returned for a successful execution
2136  * @retval ::-1         is returned if SEC driver release is in progress
2137  */
2138 static int
2139 caam_jr_dev_uninit(struct rte_cryptodev *dev)
2140 {
2141         struct sec_job_ring_t *internals;
2142
2143         PMD_INIT_FUNC_TRACE();
2144         if (dev == NULL)
2145                 return -ENODEV;
2146
2147         internals = dev->data->dev_private;
2148         rte_free(dev->security_ctx);
2149
2150         /* If any descriptors in flight , poll and wait
2151          * until all descriptors are received and silently discarded.
2152          */
2153         if (internals) {
2154                 shutdown_job_ring(internals);
2155                 close_job_ring(internals);
2156                 rte_mempool_free(internals->ctx_pool);
2157         }
2158
2159         CAAM_JR_INFO("Closing crypto device %s", dev->data->name);
2160
2161         /* last caam jr instance) */
2162         if (g_job_rings_no == 0)
2163                 g_driver_state = SEC_DRIVER_STATE_IDLE;
2164
2165         return SEC_SUCCESS;
2166 }
2167
2168 /* @brief Initialize the software and hardware resources tied to a job ring.
2169  * @param [in] jr_mode;         Model to be used by SEC Driver to receive
2170  *                              notifications from SEC.  Can be either
2171  *                              of the three: #SEC_NOTIFICATION_TYPE_NAPI
2172  *                              #SEC_NOTIFICATION_TYPE_IRQ or
2173  *                              #SEC_NOTIFICATION_TYPE_POLL
2174  * @param [in] NAPI_mode        The NAPI work mode to configure a job ring at
2175  *                              startup. Used only when #SEC_NOTIFICATION_TYPE
2176  *                              is set to #SEC_NOTIFICATION_TYPE_NAPI.
2177  * @param [in] irq_coalescing_timer This value determines the maximum
2178  *                                      amount of time after processing a
2179  *                                      descriptor before raising an interrupt.
2180  * @param [in] irq_coalescing_count This value determines how many
2181  *                                      descriptors are completed before
2182  *                                      raising an interrupt.
2183  * @param [in] reg_base_addr,   The job ring base address register
2184  * @param [in] irq_id           The job ring interrupt identification number.
2185  * @retval  job_ring_handle for successful job ring configuration
2186  * @retval  NULL on error
2187  *
2188  */
2189 static void *
2190 init_job_ring(void *reg_base_addr, uint32_t irq_id)
2191 {
2192         struct sec_job_ring_t *job_ring = NULL;
2193         int i, ret = 0;
2194         int jr_mode = SEC_NOTIFICATION_TYPE_POLL;
2195         int napi_mode = 0;
2196         int irq_coalescing_timer = 0;
2197         int irq_coalescing_count = 0;
2198
2199         for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
2200                 if (g_job_rings[i].irq_fd == 0) {
2201                         job_ring = &g_job_rings[i];
2202                         g_job_rings_no++;
2203                         break;
2204                 }
2205         }
2206         if (job_ring == NULL) {
2207                 CAAM_JR_ERR("No free job ring\n");
2208                 return NULL;
2209         }
2210
2211         job_ring->register_base_addr = reg_base_addr;
2212         job_ring->jr_mode = jr_mode;
2213         job_ring->napi_mode = 0;
2214         job_ring->irq_fd = irq_id;
2215
2216         /* Allocate mem for input and output ring */
2217
2218         /* Allocate memory for input ring */
2219         job_ring->input_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
2220                                 SEC_DMA_MEM_INPUT_RING_SIZE);
2221         memset(job_ring->input_ring, 0, SEC_DMA_MEM_INPUT_RING_SIZE);
2222
2223         /* Allocate memory for output ring */
2224         job_ring->output_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
2225                                 SEC_DMA_MEM_OUTPUT_RING_SIZE);
2226         memset(job_ring->output_ring, 0, SEC_DMA_MEM_OUTPUT_RING_SIZE);
2227
2228         /* Reset job ring in SEC hw and configure job ring registers */
2229         ret = hw_reset_job_ring(job_ring);
2230         if (ret != 0) {
2231                 CAAM_JR_ERR("Failed to reset hardware job ring");
2232                 goto cleanup;
2233         }
2234
2235         if (jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
2236         /* When SEC US driver works in NAPI mode, the UA can select
2237          * if the driver starts with IRQs on or off.
2238          */
2239                 if (napi_mode == SEC_STARTUP_INTERRUPT_MODE) {
2240                         CAAM_JR_INFO("Enabling DONE IRQ generationon job ring - %p",
2241                                 job_ring);
2242                         ret = caam_jr_enable_irqs(job_ring->irq_fd);
2243                         if (ret != 0) {
2244                                 CAAM_JR_ERR("Failed to enable irqs for job ring");
2245                                 goto cleanup;
2246                         }
2247                 }
2248         } else if (jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
2249         /* When SEC US driver works in pure interrupt mode,
2250          * IRQ's are always enabled.
2251          */
2252                 CAAM_JR_INFO("Enabling DONE IRQ generation on job ring - %p",
2253                          job_ring);
2254                 ret = caam_jr_enable_irqs(job_ring->irq_fd);
2255                 if (ret != 0) {
2256                         CAAM_JR_ERR("Failed to enable irqs for job ring");
2257                         goto cleanup;
2258                 }
2259         }
2260         if (irq_coalescing_timer || irq_coalescing_count) {
2261                 hw_job_ring_set_coalescing_param(job_ring,
2262                          irq_coalescing_timer,
2263                          irq_coalescing_count);
2264
2265                 hw_job_ring_enable_coalescing(job_ring);
2266                 job_ring->coalescing_en = 1;
2267         }
2268
2269         job_ring->jr_state = SEC_JOB_RING_STATE_STARTED;
2270         job_ring->max_nb_queue_pairs = RTE_CAAM_MAX_NB_SEC_QPS;
2271         job_ring->max_nb_sessions = RTE_CAAM_JR_PMD_MAX_NB_SESSIONS;
2272
2273         return job_ring;
2274 cleanup:
2275         caam_jr_dma_free(job_ring->output_ring);
2276         caam_jr_dma_free(job_ring->input_ring);
2277         return NULL;
2278 }
2279
2280
2281 static int
2282 caam_jr_dev_init(const char *name,
2283                  struct rte_vdev_device *vdev,
2284                  struct rte_cryptodev_pmd_init_params *init_params)
2285 {
2286         struct rte_cryptodev *dev;
2287         struct rte_security_ctx *security_instance;
2288         struct uio_job_ring *job_ring;
2289         char str[RTE_CRYPTODEV_NAME_MAX_LEN];
2290
2291         PMD_INIT_FUNC_TRACE();
2292
2293         /* Validate driver state */
2294         if (g_driver_state == SEC_DRIVER_STATE_IDLE) {
2295                 g_job_rings_max = sec_configure();
2296                 if (!g_job_rings_max) {
2297                         CAAM_JR_ERR("No job ring detected on UIO !!!!");
2298                         return -1;
2299                 }
2300                 /* Update driver state */
2301                 g_driver_state = SEC_DRIVER_STATE_STARTED;
2302         }
2303
2304         if (g_job_rings_no >= g_job_rings_max) {
2305                 CAAM_JR_ERR("No more job rings available max=%d!!!!",
2306                                 g_job_rings_max);
2307                 return -1;
2308         }
2309
2310         job_ring = config_job_ring();
2311         if (job_ring == NULL) {
2312                 CAAM_JR_ERR("failed to create job ring");
2313                 goto init_error;
2314         }
2315
2316         snprintf(str, sizeof(str), "caam_jr%d", job_ring->jr_id);
2317
2318         dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
2319         if (dev == NULL) {
2320                 CAAM_JR_ERR("failed to create cryptodev vdev");
2321                 goto cleanup;
2322         }
2323         /*TODO free it during teardown*/
2324         dev->data->dev_private = init_job_ring(job_ring->register_base_addr,
2325                                                 job_ring->uio_fd);
2326
2327         if (!dev->data->dev_private) {
2328                 CAAM_JR_ERR("Ring memory allocation failed\n");
2329                 goto cleanup2;
2330         }
2331
2332         dev->driver_id = cryptodev_driver_id;
2333         dev->dev_ops = &caam_jr_ops;
2334
2335         /* register rx/tx burst functions for data path */
2336         dev->dequeue_burst = caam_jr_dequeue_burst;
2337         dev->enqueue_burst = caam_jr_enqueue_burst;
2338         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2339                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
2340                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2341                         RTE_CRYPTODEV_FF_SECURITY |
2342                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
2343                         RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
2344                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
2345                         RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
2346                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
2347
2348         /* For secondary processes, we don't initialise any further as primary
2349          * has already done this work. Only check we don't need a different
2350          * RX function
2351          */
2352         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2353                 CAAM_JR_WARN("Device already init by primary process");
2354                 return 0;
2355         }
2356
2357         /*TODO free it during teardown*/
2358         security_instance = rte_malloc("caam_jr",
2359                                 sizeof(struct rte_security_ctx), 0);
2360         if (security_instance == NULL) {
2361                 CAAM_JR_ERR("memory allocation failed\n");
2362                 //todo error handling.
2363                 goto cleanup2;
2364         }
2365
2366         security_instance->device = (void *)dev;
2367         security_instance->ops = &caam_jr_security_ops;
2368         security_instance->sess_cnt = 0;
2369         dev->security_ctx = security_instance;
2370
2371         RTE_LOG(INFO, PMD, "%s cryptodev init\n", dev->data->name);
2372
2373         return 0;
2374
2375 cleanup2:
2376         caam_jr_dev_uninit(dev);
2377         rte_cryptodev_pmd_release_device(dev);
2378 cleanup:
2379         free_job_ring(job_ring->uio_fd);
2380 init_error:
2381         CAAM_JR_ERR("driver %s: cryptodev_caam_jr_create failed",
2382                         init_params->name);
2383
2384         return -ENXIO;
2385 }
2386
2387 /** Initialise CAAM JR crypto device */
2388 static int
2389 cryptodev_caam_jr_probe(struct rte_vdev_device *vdev)
2390 {
2391         struct rte_cryptodev_pmd_init_params init_params = {
2392                 "",
2393                 sizeof(struct sec_job_ring_t),
2394                 rte_socket_id(),
2395                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
2396         };
2397         const char *name;
2398         const char *input_args;
2399
2400         name = rte_vdev_device_name(vdev);
2401         if (name == NULL)
2402                 return -EINVAL;
2403
2404         input_args = rte_vdev_device_args(vdev);
2405         rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
2406
2407         /* if sec device version is not configured */
2408         if (!rta_get_sec_era()) {
2409                 const struct device_node *caam_node;
2410
2411                 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
2412                         const uint32_t *prop = of_get_property(caam_node,
2413                                         "fsl,sec-era",
2414                                         NULL);
2415                         if (prop) {
2416                                 rta_set_sec_era(
2417                                         INTL_SEC_ERA(cpu_to_caam32(*prop)));
2418                                 break;
2419                         }
2420                 }
2421         }
2422 #ifdef RTE_LIBRTE_PMD_CAAM_JR_BE
2423         if (rta_get_sec_era() > RTA_SEC_ERA_8) {
2424                 RTE_LOG(ERR, PMD,
2425                 "CAAM is compiled in BE mode for device with sec era > 8???\n");
2426                 return -EINVAL;
2427         }
2428 #endif
2429
2430         return caam_jr_dev_init(name, vdev, &init_params);
2431 }
2432
2433 /** Uninitialise CAAM JR crypto device */
2434 static int
2435 cryptodev_caam_jr_remove(struct rte_vdev_device *vdev)
2436 {
2437         struct rte_cryptodev *cryptodev;
2438         const char *name;
2439
2440         name = rte_vdev_device_name(vdev);
2441         if (name == NULL)
2442                 return -EINVAL;
2443
2444         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
2445         if (cryptodev == NULL)
2446                 return -ENODEV;
2447
2448         caam_jr_dev_uninit(cryptodev);
2449
2450         return rte_cryptodev_pmd_destroy(cryptodev);
2451 }
2452
2453 static struct rte_vdev_driver cryptodev_caam_jr_drv = {
2454         .probe = cryptodev_caam_jr_probe,
2455         .remove = cryptodev_caam_jr_remove
2456 };
2457
2458 static struct cryptodev_driver caam_jr_crypto_drv;
2459
2460 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CAAM_JR_PMD, cryptodev_caam_jr_drv);
2461 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CAAM_JR_PMD,
2462         "max_nb_queue_pairs=<int>"
2463         "socket_id=<int>");
2464 RTE_PMD_REGISTER_CRYPTO_DRIVER(caam_jr_crypto_drv, cryptodev_caam_jr_drv.driver,
2465                 cryptodev_driver_id);
2466
2467 RTE_INIT(caam_jr_init_log)
2468 {
2469         caam_jr_logtype = rte_log_register("pmd.crypto.caam");
2470         if (caam_jr_logtype >= 0)
2471                 rte_log_set_level(caam_jr_logtype, RTE_LOG_NOTICE);
2472 }