drivers/crypto/caam_jr/caam_jr.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017-2019 NXP
   3  */
   4
   5 #include <fcntl.h>
   6 #include <unistd.h>
   7 #include <sched.h>
   8 #include <net/if.h>
   9
  10 #include <rte_byteorder.h>
  11 #include <rte_common.h>
  12 #include <rte_cryptodev_pmd.h>
  13 #include <rte_crypto.h>
  14 #include <rte_cryptodev.h>
  15 #include <rte_bus_vdev.h>
  16 #include <rte_malloc.h>
  17 #include <rte_security_driver.h>
  18 #include <rte_hexdump.h>
  19
  20 #include <caam_jr_capabilities.h>
  21 #include <caam_jr_config.h>
  22 #include <caam_jr_hw_specific.h>
  23 #include <caam_jr_pvt.h>
  24 #include <caam_jr_desc.h>
  25 #include <caam_jr_log.h>
  26
  27 /* RTA header files */
  28 #include <desc/common.h>
  29 #include <desc/algo.h>
  30 #include <dpaa_of.h>
  31 #ifdef RTE_LIBRTE_PMD_CAAM_JR_DEBUG
  32 #define CAAM_JR_DBG    1
  33 #else
  34 #define CAAM_JR_DBG     0
  35 #endif
  36 #define CRYPTODEV_NAME_CAAM_JR_PMD      crypto_caam_jr
  37 static uint8_t cryptodev_driver_id;
  38 int caam_jr_logtype;
  39
  40 /* Lists the states possible for the SEC user space driver. */
  41 enum sec_driver_state_e {
  42         SEC_DRIVER_STATE_IDLE,          /* Driver not initialized */
  43         SEC_DRIVER_STATE_STARTED,       /* Driver initialized and can be used*/
  44         SEC_DRIVER_STATE_RELEASE,       /* Driver release is in progress */
  45 };
  46
  47 /* Job rings used for communication with SEC HW */
  48 static struct sec_job_ring_t g_job_rings[MAX_SEC_JOB_RINGS];
  49
  50 /* The current state of SEC user space driver */
  51 static enum sec_driver_state_e g_driver_state = SEC_DRIVER_STATE_IDLE;
  52
  53 /* The number of job rings used by SEC user space driver */
  54 static int g_job_rings_no;
  55 static int g_job_rings_max;
  56
  57 struct sec_outring_entry {
  58         phys_addr_t desc;       /* Pointer to completed descriptor */
  59         uint32_t status;        /* Status for completed descriptor */
  60 } __rte_packed;
  61
  62 /* virtual address conversin when mempool support is available for ctx */
  63 static inline phys_addr_t
  64 caam_jr_vtop_ctx(struct caam_jr_op_ctx *ctx, void *vaddr)
  65 {
  66         return (size_t)vaddr - ctx->vtop_offset;
  67 }
  68
  69 static inline void
  70 caam_jr_op_ending(struct caam_jr_op_ctx *ctx)
  71 {
  72         /* report op status to sym->op and then free the ctx memory  */
  73         rte_mempool_put(ctx->ctx_pool, (void *)ctx);
  74 }
  75
  76 static inline struct caam_jr_op_ctx *
  77 caam_jr_alloc_ctx(struct caam_jr_session *ses)
  78 {
  79         struct caam_jr_op_ctx *ctx;
  80         int ret;
  81
  82         ret = rte_mempool_get(ses->ctx_pool, (void **)(&ctx));
  83         if (!ctx || ret) {
  84                 CAAM_JR_DP_WARN("Alloc sec descriptor failed!");
  85                 return NULL;
  86         }
  87         /*
  88          * Clear SG memory. There are 16 SG entries of 16 Bytes each.
  89          * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
  90          * to clear all the SG entries. caam_jr_alloc_ctx() is called for
  91          * each packet, memset is costlier than dcbz_64().
  92          */
  93         dcbz_64(&ctx->sg[SG_CACHELINE_0]);
  94         dcbz_64(&ctx->sg[SG_CACHELINE_1]);
  95         dcbz_64(&ctx->sg[SG_CACHELINE_2]);
  96         dcbz_64(&ctx->sg[SG_CACHELINE_3]);
  97
  98         ctx->ctx_pool = ses->ctx_pool;
  99         ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx);
 100
 101         return ctx;
 102 }
 103
 104 static
 105 void caam_jr_stats_get(struct rte_cryptodev *dev,
 106                         struct rte_cryptodev_stats *stats)
 107 {
 108         struct caam_jr_qp **qp = (struct caam_jr_qp **)
 109                                         dev->data->queue_pairs;
 110         int i;
 111
 112         PMD_INIT_FUNC_TRACE();
 113         if (stats == NULL) {
 114                 CAAM_JR_ERR("Invalid stats ptr NULL");
 115                 return;
 116         }
 117         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 118                 if (qp[i] == NULL) {
 119                         CAAM_JR_WARN("Uninitialised queue pair");
 120                         continue;
 121                 }
 122
 123                 stats->enqueued_count += qp[i]->tx_pkts;
 124                 stats->dequeued_count += qp[i]->rx_pkts;
 125                 stats->enqueue_err_count += qp[i]->tx_errs;
 126                 stats->dequeue_err_count += qp[i]->rx_errs;
 127                 CAAM_JR_INFO("extra stats:\n\tRX Poll ERR = %" PRIu64
 128                              "\n\tTX Ring Full = %" PRIu64,
 129                              qp[i]->rx_poll_err,
 130                              qp[i]->tx_ring_full);
 131         }
 132 }
 133
 134 static
 135 void caam_jr_stats_reset(struct rte_cryptodev *dev)
 136 {
 137         int i;
 138         struct caam_jr_qp **qp = (struct caam_jr_qp **)
 139                                    (dev->data->queue_pairs);
 140
 141         PMD_INIT_FUNC_TRACE();
 142         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 143                 if (qp[i] == NULL) {
 144                         CAAM_JR_WARN("Uninitialised queue pair");
 145                         continue;
 146                 }
 147                 qp[i]->rx_pkts = 0;
 148                 qp[i]->rx_errs = 0;
 149                 qp[i]->rx_poll_err = 0;
 150                 qp[i]->tx_pkts = 0;
 151                 qp[i]->tx_errs = 0;
 152                 qp[i]->tx_ring_full = 0;
 153         }
 154 }
 155
 156 static inline int
 157 is_cipher_only(struct caam_jr_session *ses)
 158 {
 159         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 160                 (ses->auth_alg == RTE_CRYPTO_AUTH_NULL));
 161 }
 162
 163 static inline int
 164 is_auth_only(struct caam_jr_session *ses)
 165 {
 166         return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) &&
 167                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL));
 168 }
 169
 170 static inline int
 171 is_aead(struct caam_jr_session *ses)
 172 {
 173         return ((ses->cipher_alg == 0) &&
 174                 (ses->auth_alg == 0) &&
 175                 (ses->aead_alg != 0));
 176 }
 177
 178 static inline int
 179 is_auth_cipher(struct caam_jr_session *ses)
 180 {
 181         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 182                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) &&
 183                 (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC));
 184 }
 185
 186 static inline int
 187 is_proto_ipsec(struct caam_jr_session *ses)
 188 {
 189         return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC);
 190 }
 191
 192 static inline int
 193 is_encode(struct caam_jr_session *ses)
 194 {
 195         return ses->dir == DIR_ENC;
 196 }
 197
 198 static inline int
 199 is_decode(struct caam_jr_session *ses)
 200 {
 201         return ses->dir == DIR_DEC;
 202 }
 203
 204 static inline void
 205 caam_auth_alg(struct caam_jr_session *ses, struct alginfo *alginfo_a)
 206 {
 207         switch (ses->auth_alg) {
 208         case RTE_CRYPTO_AUTH_NULL:
 209                 ses->digest_length = 0;
 210                 break;
 211         case RTE_CRYPTO_AUTH_MD5_HMAC:
 212                 alginfo_a->algtype =
 213                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 214                         OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5;
 215                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 216                 break;
 217         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 218                 alginfo_a->algtype =
 219                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 220                         OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1;
 221                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 222                 break;
 223         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 224                 alginfo_a->algtype =
 225                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 226                         OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224;
 227                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 228                 break;
 229         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 230                 alginfo_a->algtype =
 231                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 232                         OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256;
 233                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 234                 break;
 235         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 236                 alginfo_a->algtype =
 237                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 238                         OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384;
 239                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 240                 break;
 241         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 242                 alginfo_a->algtype =
 243                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 244                         OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512;
 245                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 246                 break;
 247         default:
 248                 CAAM_JR_DEBUG("unsupported auth alg %u", ses->auth_alg);
 249         }
 250 }
 251
 252 static inline void
 253 caam_cipher_alg(struct caam_jr_session *ses, struct alginfo *alginfo_c)
 254 {
 255         switch (ses->cipher_alg) {
 256         case RTE_CRYPTO_CIPHER_NULL:
 257                 break;
 258         case RTE_CRYPTO_CIPHER_AES_CBC:
 259                 alginfo_c->algtype =
 260                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 261                         OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES;
 262                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 263                 break;
 264         case RTE_CRYPTO_CIPHER_3DES_CBC:
 265                 alginfo_c->algtype =
 266                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 267                         OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES;
 268                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 269                 break;
 270         case RTE_CRYPTO_CIPHER_AES_CTR:
 271                 alginfo_c->algtype =
 272                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 273                         OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES;
 274                 alginfo_c->algmode = OP_ALG_AAI_CTR;
 275                 break;
 276         default:
 277                 CAAM_JR_DEBUG("unsupported cipher alg %d", ses->cipher_alg);
 278         }
 279 }
 280
 281 static inline void
 282 caam_aead_alg(struct caam_jr_session *ses, struct alginfo *alginfo)
 283 {
 284         switch (ses->aead_alg) {
 285         case RTE_CRYPTO_AEAD_AES_GCM:
 286                 alginfo->algtype = OP_ALG_ALGSEL_AES;
 287                 alginfo->algmode = OP_ALG_AAI_GCM;
 288                 break;
 289         default:
 290                 CAAM_JR_DEBUG("unsupported AEAD alg %d", ses->aead_alg);
 291         }
 292 }
 293
 294 /* prepare command block of the session */
 295 static int
 296 caam_jr_prep_cdb(struct caam_jr_session *ses)
 297 {
 298         struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0};
 299         int32_t shared_desc_len = 0;
 300         struct sec_cdb *cdb;
 301         int err;
 302 #if CAAM_BYTE_ORDER == CORE_BYTE_ORDER
 303         int swap = false;
 304 #else
 305         int swap = true;
 306 #endif
 307
 308         if (ses->cdb)
 309                 caam_jr_dma_free(ses->cdb);
 310
 311         cdb = caam_jr_dma_mem_alloc(L1_CACHE_BYTES, sizeof(struct sec_cdb));
 312         if (!cdb) {
 313                 CAAM_JR_ERR("failed to allocate memory for cdb\n");
 314                 return -1;
 315         }
 316
 317         ses->cdb = cdb;
 318
 319         memset(cdb, 0, sizeof(struct sec_cdb));
 320
 321         if (is_cipher_only(ses)) {
 322                 caam_cipher_alg(ses, &alginfo_c);
 323                 if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 324                         CAAM_JR_ERR("not supported cipher alg");
 325                         rte_free(cdb);
 326                         return -ENOTSUP;
 327                 }
 328
 329                 alginfo_c.key = (size_t)ses->cipher_key.data;
 330                 alginfo_c.keylen = ses->cipher_key.length;
 331                 alginfo_c.key_enc_flags = 0;
 332                 alginfo_c.key_type = RTA_DATA_IMM;
 333
 334                 shared_desc_len = cnstr_shdsc_blkcipher(
 335                                                 cdb->sh_desc, true,
 336                                                 swap, SHR_NEVER, &alginfo_c,
 337                                                 ses->iv.length,
 338                                                 ses->dir);
 339         } else if (is_auth_only(ses)) {
 340                 caam_auth_alg(ses, &alginfo_a);
 341                 if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 342                         CAAM_JR_ERR("not supported auth alg");
 343                         rte_free(cdb);
 344                         return -ENOTSUP;
 345                 }
 346
 347                 alginfo_a.key = (size_t)ses->auth_key.data;
 348                 alginfo_a.keylen = ses->auth_key.length;
 349                 alginfo_a.key_enc_flags = 0;
 350                 alginfo_a.key_type = RTA_DATA_IMM;
 351
 352                 shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true,
 353                                                    swap, SHR_NEVER, &alginfo_a,
 354                                                    !ses->dir,
 355                                                    ses->digest_length);
 356         } else if (is_aead(ses)) {
 357                 caam_aead_alg(ses, &alginfo);
 358                 if (alginfo.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 359                         CAAM_JR_ERR("not supported aead alg");
 360                         rte_free(cdb);
 361                         return -ENOTSUP;
 362                 }
 363                 alginfo.key = (size_t)ses->aead_key.data;
 364                 alginfo.keylen = ses->aead_key.length;
 365                 alginfo.key_enc_flags = 0;
 366                 alginfo.key_type = RTA_DATA_IMM;
 367
 368                 if (ses->dir == DIR_ENC)
 369                         shared_desc_len = cnstr_shdsc_gcm_encap(
 370                                         cdb->sh_desc, true, swap,
 371                                         SHR_NEVER, &alginfo,
 372                                         ses->iv.length,
 373                                         ses->digest_length);
 374                 else
 375                         shared_desc_len = cnstr_shdsc_gcm_decap(
 376                                         cdb->sh_desc, true, swap,
 377                                         SHR_NEVER, &alginfo,
 378                                         ses->iv.length,
 379                                         ses->digest_length);
 380         } else {
 381                 caam_cipher_alg(ses, &alginfo_c);
 382                 if (alginfo_c.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 383                         CAAM_JR_ERR("not supported cipher alg");
 384                         rte_free(cdb);
 385                         return -ENOTSUP;
 386                 }
 387
 388                 alginfo_c.key = (size_t)ses->cipher_key.data;
 389                 alginfo_c.keylen = ses->cipher_key.length;
 390                 alginfo_c.key_enc_flags = 0;
 391                 alginfo_c.key_type = RTA_DATA_IMM;
 392
 393                 caam_auth_alg(ses, &alginfo_a);
 394                 if (alginfo_a.algtype == (unsigned int)CAAM_JR_ALG_UNSUPPORT) {
 395                         CAAM_JR_ERR("not supported auth alg");
 396                         rte_free(cdb);
 397                         return -ENOTSUP;
 398                 }
 399
 400                 alginfo_a.key = (size_t)ses->auth_key.data;
 401                 alginfo_a.keylen = ses->auth_key.length;
 402                 alginfo_a.key_enc_flags = 0;
 403                 alginfo_a.key_type = RTA_DATA_IMM;
 404
 405                 cdb->sh_desc[0] = alginfo_c.keylen;
 406                 cdb->sh_desc[1] = alginfo_a.keylen;
 407                 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
 408                                        MIN_JOB_DESC_SIZE,
 409                                        (unsigned int *)cdb->sh_desc,
 410                                        &cdb->sh_desc[2], 2);
 411
 412                 if (err < 0) {
 413                         CAAM_JR_ERR("Crypto: Incorrect key lengths");
 414                         rte_free(cdb);
 415                         return err;
 416                 }
 417                 if (cdb->sh_desc[2] & 1)
 418                         alginfo_c.key_type = RTA_DATA_IMM;
 419                 else {
 420                         alginfo_c.key = (size_t)caam_jr_mem_vtop(
 421                                                 (void *)(size_t)alginfo_c.key);
 422                         alginfo_c.key_type = RTA_DATA_PTR;
 423                 }
 424                 if (cdb->sh_desc[2] & (1<<1))
 425                         alginfo_a.key_type = RTA_DATA_IMM;
 426                 else {
 427                         alginfo_a.key = (size_t)caam_jr_mem_vtop(
 428                                                 (void *)(size_t)alginfo_a.key);
 429                         alginfo_a.key_type = RTA_DATA_PTR;
 430                 }
 431                 cdb->sh_desc[0] = 0;
 432                 cdb->sh_desc[1] = 0;
 433                 cdb->sh_desc[2] = 0;
 434                 if (is_proto_ipsec(ses)) {
 435                         if (ses->dir == DIR_ENC) {
 436                                 shared_desc_len = cnstr_shdsc_ipsec_new_encap(
 437                                                 cdb->sh_desc,
 438                                                 true, swap, SHR_SERIAL,
 439                                                 &ses->encap_pdb,
 440                                                 (uint8_t *)&ses->ip4_hdr,
 441                                                 &alginfo_c, &alginfo_a);
 442                         } else if (ses->dir == DIR_DEC) {
 443                                 shared_desc_len = cnstr_shdsc_ipsec_new_decap(
 444                                                 cdb->sh_desc,
 445                                                 true, swap, SHR_SERIAL,
 446                                                 &ses->decap_pdb,
 447                                                 &alginfo_c, &alginfo_a);
 448                         }
 449                 } else {
 450                         /* Auth_only_len is overwritten in fd for each job */
 451                         shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
 452                                         true, swap, SHR_SERIAL,
 453                                         &alginfo_c, &alginfo_a,
 454                                         ses->iv.length,
 455                                         ses->digest_length, ses->dir);
 456                 }
 457         }
 458
 459         if (shared_desc_len < 0) {
 460                 CAAM_JR_ERR("error in preparing command block");
 461                 return shared_desc_len;
 462         }
 463
 464 #if CAAM_JR_DBG
 465         SEC_DUMP_DESC(cdb->sh_desc);
 466 #endif
 467
 468         cdb->sh_hdr.hi.field.idlen = shared_desc_len;
 469
 470         return 0;
 471 }
 472
 473 /* @brief Poll the HW for already processed jobs in the JR
 474  * and silently discard the available jobs or notify them to UA
 475  * with indicated error code.
 476  *
 477  * @param [in,out]  job_ring        The job ring to poll.
 478  * @param [in]  do_notify           Can be #TRUE or #FALSE. Indicates if
 479  *                                  descriptors are to be discarded
 480  *                                  or notified to UA with given error_code.
 481  * @param [out] notified_descs    Number of notified descriptors. Can be NULL
 482  *                                      if do_notify is #FALSE
 483  */
 484 static void
 485 hw_flush_job_ring(struct sec_job_ring_t *job_ring,
 486                   uint32_t do_notify,
 487                   uint32_t *notified_descs)
 488 {
 489         int32_t jobs_no_to_discard = 0;
 490         int32_t discarded_descs_no = 0;
 491
 492         CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Flushing jr notify desc=[%d]",
 493                 job_ring, job_ring->pidx, job_ring->cidx, do_notify);
 494
 495         jobs_no_to_discard = hw_get_no_finished_jobs(job_ring);
 496
 497         /* Discard all jobs */
 498         CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Discarding %d descs",
 499                   job_ring, job_ring->pidx, job_ring->cidx,
 500                   jobs_no_to_discard);
 501
 502         while (jobs_no_to_discard > discarded_descs_no) {
 503                 discarded_descs_no++;
 504                 /* Now increment the consumer index for the current job ring,
 505                  * AFTER saving job in temporary location!
 506                  * Increment the consumer index for the current job ring
 507                  */
 508                 job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
 509                                          SEC_JOB_RING_SIZE);
 510
 511                 hw_remove_entries(job_ring, 1);
 512         }
 513
 514         if (do_notify == true) {
 515                 ASSERT(notified_descs != NULL);
 516                 *notified_descs = discarded_descs_no;
 517         }
 518 }
 519
 520 /* @brief Poll the HW for already processed jobs in the JR
 521  * and notify the available jobs to UA.
 522  *
 523  * @param [in]  job_ring        The job ring to poll.
 524  * @param [in]  limit           The maximum number of jobs to notify.
 525  *                              If set to negative value, all available jobs are
 526  *                              notified.
 527  *
 528  * @retval >=0 for No of jobs notified to UA.
 529  * @retval -1 for error
 530  */
 531 static int
 532 hw_poll_job_ring(struct sec_job_ring_t *job_ring,
 533                  struct rte_crypto_op **ops, int32_t limit,
 534                  struct caam_jr_qp *jr_qp)
 535 {
 536         int32_t jobs_no_to_notify = 0; /* the number of done jobs to notify*/
 537         int32_t number_of_jobs_available = 0;
 538         int32_t notified_descs_no = 0;
 539         uint32_t sec_error_code = 0;
 540         struct job_descriptor *current_desc;
 541         phys_addr_t current_desc_addr;
 542         phys_addr_t *temp_addr;
 543         struct caam_jr_op_ctx *ctx;
 544
 545         /* TODO check for ops have memory*/
 546         /* check here if any JR error that cannot be written
 547          * in the output status word has occurred
 548          */
 549         if (JR_REG_JRINT_JRE_EXTRACT(GET_JR_REG(JRINT, job_ring))) {
 550                 CAAM_JR_INFO("err received");
 551                 sec_error_code = JR_REG_JRINT_ERR_TYPE_EXTRACT(
 552                                         GET_JR_REG(JRINT, job_ring));
 553                 if (unlikely(sec_error_code)) {
 554                         hw_job_ring_error_print(job_ring, sec_error_code);
 555                         return -1;
 556                 }
 557         }
 558         /* compute the number of jobs available in the job ring based on the
 559          * producer and consumer index values.
 560          */
 561         number_of_jobs_available = hw_get_no_finished_jobs(job_ring);
 562         /* Compute the number of notifications that need to be raised to UA
 563          * If limit > total number of done jobs -> notify all done jobs
 564          * If limit = 0 -> error
 565          * If limit < total number of done jobs -> notify a number
 566          * of done jobs equal with limit
 567          */
 568         jobs_no_to_notify = (limit > number_of_jobs_available) ?
 569                                 number_of_jobs_available : limit;
 570         CAAM_JR_DP_DEBUG(
 571                 "Jr[%p] pi[%d] ci[%d].limit =%d Available=%d.Jobs to notify=%d",
 572                 job_ring, job_ring->pidx, job_ring->cidx,
 573                 limit, number_of_jobs_available, jobs_no_to_notify);
 574
 575         rte_smp_rmb();
 576
 577         while (jobs_no_to_notify > notified_descs_no) {
 578                 static uint64_t false_alarm;
 579                 static uint64_t real_poll;
 580
 581                 /* Get job status here */
 582                 sec_error_code = job_ring->output_ring[job_ring->cidx].status;
 583                 /* Get completed descriptor */
 584                 temp_addr = &(job_ring->output_ring[job_ring->cidx].desc);
 585                 current_desc_addr = (phys_addr_t)sec_read_addr(temp_addr);
 586
 587                 real_poll++;
 588                 /* todo check if it is false alarm no desc present */
 589                 if (!current_desc_addr) {
 590                         false_alarm++;
 591                         printf("false alarm %" PRIu64 "real %" PRIu64
 592                                 " sec_err =0x%x cidx Index =0%d\n",
 593                                 false_alarm, real_poll,
 594                                 sec_error_code, job_ring->cidx);
 595                         rte_panic("CAAM JR descriptor NULL");
 596                         return notified_descs_no;
 597                 }
 598                 current_desc = (struct job_descriptor *)
 599                                 caam_jr_dma_ptov(current_desc_addr);
 600                 /* now increment the consumer index for the current job ring,
 601                  * AFTER saving job in temporary location!
 602                  */
 603                 job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
 604                                  SEC_JOB_RING_SIZE);
 605                 /* Signal that the job has been processed and the slot is free*/
 606                 hw_remove_entries(job_ring, 1);
 607                 /*TODO for multiple ops, packets*/
 608                 ctx = container_of(current_desc, struct caam_jr_op_ctx, jobdes);
 609                 if (unlikely(sec_error_code)) {
 610                         CAAM_JR_ERR("desc at cidx %d generated error 0x%x\n",
 611                                 job_ring->cidx, sec_error_code);
 612                         hw_handle_job_ring_error(job_ring, sec_error_code);
 613                         //todo improve with exact errors
 614                         ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 615                         jr_qp->rx_errs++;
 616                 } else {
 617                         ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 618 #if CAAM_JR_DBG
 619                         if (ctx->op->sym->m_dst) {
 620                                 rte_hexdump(stdout, "PROCESSED",
 621                                 rte_pktmbuf_mtod(ctx->op->sym->m_dst, void *),
 622                                 rte_pktmbuf_data_len(ctx->op->sym->m_dst));
 623                         } else {
 624                                 rte_hexdump(stdout, "PROCESSED",
 625                                 rte_pktmbuf_mtod(ctx->op->sym->m_src, void *),
 626                                 rte_pktmbuf_data_len(ctx->op->sym->m_src));
 627                         }
 628 #endif
 629                 }
 630                 if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 631                         struct ip *ip4_hdr;
 632
 633                         if (ctx->op->sym->m_dst) {
 634                                 /*TODO check for ip header or other*/
 635                                 ip4_hdr = (struct ip *)
 636                                 rte_pktmbuf_mtod(ctx->op->sym->m_dst, char*);
 637                                 ctx->op->sym->m_dst->pkt_len =
 638                                         rte_be_to_cpu_16(ip4_hdr->ip_len);
 639                                 ctx->op->sym->m_dst->data_len =
 640                                         rte_be_to_cpu_16(ip4_hdr->ip_len);
 641                         } else {
 642                                 ip4_hdr = (struct ip *)
 643                                 rte_pktmbuf_mtod(ctx->op->sym->m_src, char*);
 644                                 ctx->op->sym->m_src->pkt_len =
 645                                         rte_be_to_cpu_16(ip4_hdr->ip_len);
 646                                 ctx->op->sym->m_src->data_len =
 647                                         rte_be_to_cpu_16(ip4_hdr->ip_len);
 648                         }
 649                 }
 650                 *ops = ctx->op;
 651                 caam_jr_op_ending(ctx);
 652                 ops++;
 653                 notified_descs_no++;
 654         }
 655         return notified_descs_no;
 656 }
 657
 658 static uint16_t
 659 caam_jr_dequeue_burst(void *qp, struct rte_crypto_op **ops,
 660                        uint16_t nb_ops)
 661 {
 662         struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp;
 663         struct sec_job_ring_t *ring = jr_qp->ring;
 664         int num_rx;
 665         int ret;
 666
 667         CAAM_JR_DP_DEBUG("Jr[%p]Polling. limit[%d]", ring, nb_ops);
 668
 669         /* Poll job ring
 670          * If nb_ops < 0 -> poll JR until no more notifications are available.
 671          * If nb_ops > 0 -> poll JR until limit is reached.
 672          */
 673
 674         /* Run hw poll job ring */
 675         num_rx = hw_poll_job_ring(ring, ops, nb_ops, jr_qp);
 676         if (num_rx < 0) {
 677                 CAAM_JR_ERR("Error polling SEC engine (%d)", num_rx);
 678                 return 0;
 679         }
 680
 681         CAAM_JR_DP_DEBUG("Jr[%p].Jobs notified[%d]. ", ring, num_rx);
 682
 683         if (ring->jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
 684                 if (num_rx < nb_ops) {
 685                         ret = caam_jr_enable_irqs(ring->irq_fd);
 686                         SEC_ASSERT(ret == 0, ret,
 687                         "Failed to enable irqs for job ring %p", ring);
 688                 }
 689         } else if (ring->jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
 690
 691                 /* Always enable IRQ generation when in pure IRQ mode */
 692                 ret = caam_jr_enable_irqs(ring->irq_fd);
 693                 SEC_ASSERT(ret == 0, ret,
 694                         "Failed to enable irqs for job ring %p", ring);
 695         }
 696
 697         jr_qp->rx_pkts += num_rx;
 698
 699         return num_rx;
 700 }
 701
 702 /**
 703  * packet looks like:
 704  *              |<----data_len------->|
 705  *    |ip_header|ah_header|icv|payload|
 706  *              ^
 707  *              |
 708  *         mbuf->pkt.data
 709  */
 710 static inline struct caam_jr_op_ctx *
 711 build_auth_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
 712 {
 713         struct rte_crypto_sym_op *sym = op->sym;
 714         struct rte_mbuf *mbuf = sym->m_src;
 715         struct caam_jr_op_ctx *ctx;
 716         struct sec4_sg_entry *sg;
 717         int     length;
 718         struct sec_cdb *cdb;
 719         uint64_t sdesc_offset;
 720         struct sec_job_descriptor_t *jobdescr;
 721         uint8_t extra_segs;
 722
 723         if (is_decode(ses))
 724                 extra_segs = 2;
 725         else
 726                 extra_segs = 1;
 727
 728         if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) {
 729                 CAAM_JR_DP_ERR("Auth: Max sec segs supported is %d",
 730                                 MAX_SG_ENTRIES);
 731                 return NULL;
 732         }
 733
 734         ctx = caam_jr_alloc_ctx(ses);
 735         if (!ctx)
 736                 return NULL;
 737
 738         ctx->op = op;
 739
 740         cdb = ses->cdb;
 741         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
 742
 743         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
 744
 745         SEC_JD_INIT(jobdescr);
 746         SEC_JD_SET_SD(jobdescr,
 747                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
 748                 cdb->sh_hdr.hi.field.idlen);
 749
 750         /* output */
 751         SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr,
 752                         0, ses->digest_length);
 753
 754         /*input */
 755         sg = &ctx->sg[0];
 756         length = sym->auth.data.length;
 757         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf) + sym->auth.data.offset);
 758         sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
 759
 760         /* Successive segs */
 761         mbuf = mbuf->next;
 762         while (mbuf) {
 763                 sg++;
 764                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
 765                 sg->len = cpu_to_caam32(mbuf->data_len);
 766                 mbuf = mbuf->next;
 767         }
 768
 769         if (is_decode(ses)) {
 770                 /* digest verification case */
 771                 sg++;
 772                 /* hash result or digest, save digest first */
 773                 rte_memcpy(ctx->digest, sym->auth.digest.data,
 774                            ses->digest_length);
 775 #if CAAM_JR_DBG
 776                 rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length);
 777 #endif
 778                 sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest));
 779                 sg->len = cpu_to_caam32(ses->digest_length);
 780                 length += ses->digest_length;
 781         } else {
 782                 sg->len -= ses->digest_length;
 783         }
 784
 785         /* last element*/
 786         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
 787
 788         SEC_JD_SET_IN_PTR(jobdescr,
 789                 (uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0, length);
 790         /* enabling sg list */
 791         (jobdescr)->seq_in.command.word  |= 0x01000000;
 792
 793         return ctx;
 794 }
 795
 796 static inline struct caam_jr_op_ctx *
 797 build_auth_only(struct rte_crypto_op *op, struct caam_jr_session *ses)
 798 {
 799         struct rte_crypto_sym_op *sym = op->sym;
 800         struct caam_jr_op_ctx *ctx;
 801         struct sec4_sg_entry *sg;
 802         rte_iova_t start_addr;
 803         struct sec_cdb *cdb;
 804         uint64_t sdesc_offset;
 805         struct sec_job_descriptor_t *jobdescr;
 806
 807         ctx = caam_jr_alloc_ctx(ses);
 808         if (!ctx)
 809                 return NULL;
 810
 811         ctx->op = op;
 812
 813         cdb = ses->cdb;
 814         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
 815
 816         start_addr = rte_pktmbuf_iova(sym->m_src);
 817
 818         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
 819
 820         SEC_JD_INIT(jobdescr);
 821         SEC_JD_SET_SD(jobdescr,
 822                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
 823                 cdb->sh_hdr.hi.field.idlen);
 824
 825         /* output */
 826         SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)sym->auth.digest.phys_addr,
 827                         0, ses->digest_length);
 828
 829         /*input */
 830         if (is_decode(ses)) {
 831                 sg = &ctx->sg[0];
 832                 SEC_JD_SET_IN_PTR(jobdescr,
 833                         (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0,
 834                         (sym->auth.data.length + ses->digest_length));
 835                 /* enabling sg list */
 836                 (jobdescr)->seq_in.command.word  |= 0x01000000;
 837
 838                 /* hash result or digest, save digest first */
 839                 rte_memcpy(ctx->digest, sym->auth.digest.data,
 840                            ses->digest_length);
 841                 sg->ptr = cpu_to_caam64(start_addr + sym->auth.data.offset);
 842                 sg->len = cpu_to_caam32(sym->auth.data.length);
 843
 844 #if CAAM_JR_DBG
 845                 rte_hexdump(stdout, "ICV", ctx->digest, ses->digest_length);
 846 #endif
 847                 /* let's check digest by hw */
 848                 sg++;
 849                 sg->ptr = cpu_to_caam64(caam_jr_vtop_ctx(ctx, ctx->digest));
 850                 sg->len = cpu_to_caam32(ses->digest_length);
 851                 /* last element*/
 852                 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
 853         } else {
 854                 SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)start_addr,
 855                         sym->auth.data.offset, sym->auth.data.length);
 856         }
 857         return ctx;
 858 }
 859
 860 static inline struct caam_jr_op_ctx *
 861 build_cipher_only_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
 862 {
 863         struct rte_crypto_sym_op *sym = op->sym;
 864         struct rte_mbuf *mbuf = sym->m_src;
 865         struct caam_jr_op_ctx *ctx;
 866         struct sec4_sg_entry *sg, *in_sg;
 867         int length;
 868         struct sec_cdb *cdb;
 869         uint64_t sdesc_offset;
 870         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 871                         ses->iv.offset);
 872         struct sec_job_descriptor_t *jobdescr;
 873         uint8_t reg_segs;
 874
 875         if (sym->m_dst) {
 876                 mbuf = sym->m_dst;
 877                 reg_segs = mbuf->nb_segs + sym->m_src->nb_segs + 2;
 878         } else {
 879                 mbuf = sym->m_src;
 880                 reg_segs = mbuf->nb_segs * 2 + 2;
 881         }
 882
 883         if (reg_segs > MAX_SG_ENTRIES) {
 884                 CAAM_JR_DP_ERR("Cipher: Max sec segs supported is %d",
 885                                 MAX_SG_ENTRIES);
 886                 return NULL;
 887         }
 888
 889         ctx = caam_jr_alloc_ctx(ses);
 890         if (!ctx)
 891                 return NULL;
 892
 893         ctx->op = op;
 894         cdb = ses->cdb;
 895         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
 896
 897         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
 898
 899         SEC_JD_INIT(jobdescr);
 900         SEC_JD_SET_SD(jobdescr,
 901                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
 902                 cdb->sh_hdr.hi.field.idlen);
 903
 904 #if CAAM_JR_DBG
 905         CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d",
 906                         sym->m_src->data_off, sym->cipher.data.offset,
 907                         sym->cipher.data.length, ses->iv.length);
 908 #endif
 909         /* output */
 910         if (sym->m_dst)
 911                 mbuf = sym->m_dst;
 912         else
 913                 mbuf = sym->m_src;
 914
 915         sg = &ctx->sg[0];
 916         length = sym->cipher.data.length;
 917
 918         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
 919                 + sym->cipher.data.offset);
 920         sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset);
 921
 922         /* Successive segs */
 923         mbuf = mbuf->next;
 924         while (mbuf) {
 925                 sg++;
 926                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
 927                 sg->len = cpu_to_caam32(mbuf->data_len);
 928                 mbuf = mbuf->next;
 929         }
 930         /* last element*/
 931         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
 932
 933         SEC_JD_SET_OUT_PTR(jobdescr,
 934                         (uint64_t)caam_jr_vtop_ctx(ctx, &ctx->sg[0]), 0,
 935                         length);
 936         /*enabling sg bit */
 937         (jobdescr)->seq_out.command.word  |= 0x01000000;
 938
 939         /*input */
 940         sg++;
 941         mbuf = sym->m_src;
 942         in_sg = sg;
 943
 944         length = sym->cipher.data.length + ses->iv.length;
 945
 946         /* IV */
 947         sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
 948         sg->len = cpu_to_caam32(ses->iv.length);
 949
 950         /* 1st seg */
 951         sg++;
 952         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
 953                                 + sym->cipher.data.offset);
 954         sg->len = cpu_to_caam32(mbuf->data_len - sym->cipher.data.offset);
 955
 956         /* Successive segs */
 957         mbuf = mbuf->next;
 958         while (mbuf) {
 959                 sg++;
 960                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
 961                 sg->len = cpu_to_caam32(mbuf->data_len);
 962                 mbuf = mbuf->next;
 963         }
 964         /* last element*/
 965         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
 966
 967
 968         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, in_sg), 0,
 969                                 length);
 970         /*enabling sg bit */
 971         (jobdescr)->seq_in.command.word  |= 0x01000000;
 972
 973         return ctx;
 974 }
 975
 976 static inline struct caam_jr_op_ctx *
 977 build_cipher_only(struct rte_crypto_op *op, struct caam_jr_session *ses)
 978 {
 979         struct rte_crypto_sym_op *sym = op->sym;
 980         struct caam_jr_op_ctx *ctx;
 981         struct sec4_sg_entry *sg;
 982         rte_iova_t src_start_addr, dst_start_addr;
 983         struct sec_cdb *cdb;
 984         uint64_t sdesc_offset;
 985         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 986                         ses->iv.offset);
 987         struct sec_job_descriptor_t *jobdescr;
 988
 989         ctx = caam_jr_alloc_ctx(ses);
 990         if (!ctx)
 991                 return NULL;
 992
 993         ctx->op = op;
 994         cdb = ses->cdb;
 995         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
 996
 997         src_start_addr = rte_pktmbuf_iova(sym->m_src);
 998         if (sym->m_dst)
 999                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
1000         else
1001                 dst_start_addr = src_start_addr;
1002
1003         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1004
1005         SEC_JD_INIT(jobdescr);
1006         SEC_JD_SET_SD(jobdescr,
1007                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1008                 cdb->sh_hdr.hi.field.idlen);
1009
1010 #if CAAM_JR_DBG
1011         CAAM_JR_INFO("mbuf offset =%d, cipher offset = %d, length =%d+%d",
1012                         sym->m_src->data_off, sym->cipher.data.offset,
1013                         sym->cipher.data.length, ses->iv.length);
1014 #endif
1015         /* output */
1016         SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr,
1017                         sym->cipher.data.offset,
1018                         sym->cipher.data.length + ses->iv.length);
1019
1020         /*input */
1021         sg = &ctx->sg[0];
1022         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_vtop_ctx(ctx, sg), 0,
1023                                 sym->cipher.data.length + ses->iv.length);
1024         /*enabling sg bit */
1025         (jobdescr)->seq_in.command.word  |= 0x01000000;
1026
1027         sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1028         sg->len = cpu_to_caam32(ses->iv.length);
1029
1030         sg = &ctx->sg[1];
1031         sg->ptr = cpu_to_caam64(src_start_addr + sym->cipher.data.offset);
1032         sg->len = cpu_to_caam32(sym->cipher.data.length);
1033         /* last element*/
1034         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1035
1036         return ctx;
1037 }
1038
1039 /* For decapsulation:
1040  *     Input:
1041  * +----+----------------+--------------------------------+-----+
1042  * | IV | Auth-only data | Authenticated & Encrypted data | ICV |
1043  * +----+----------------+--------------------------------+-----+
1044  *     Output:
1045  * +----+--------------------------+
1046  * | Decrypted & authenticated data |
1047  * +----+--------------------------+
1048  */
1049
1050 static inline struct caam_jr_op_ctx *
1051 build_cipher_auth_sg(struct rte_crypto_op *op, struct caam_jr_session *ses)
1052 {
1053         struct rte_crypto_sym_op *sym = op->sym;
1054         struct caam_jr_op_ctx *ctx;
1055         struct sec4_sg_entry *sg, *out_sg, *in_sg;
1056         struct rte_mbuf *mbuf;
1057         uint32_t length = 0;
1058         struct sec_cdb *cdb;
1059         uint64_t sdesc_offset;
1060         uint8_t req_segs;
1061         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1062                         ses->iv.offset);
1063         struct sec_job_descriptor_t *jobdescr;
1064         uint16_t auth_hdr_len = sym->cipher.data.offset -
1065                         sym->auth.data.offset;
1066         uint16_t auth_tail_len = sym->auth.data.length -
1067                         sym->cipher.data.length - auth_hdr_len;
1068         uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
1069
1070         if (sym->m_dst) {
1071                 mbuf = sym->m_dst;
1072                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3;
1073         } else {
1074                 mbuf = sym->m_src;
1075                 req_segs = mbuf->nb_segs * 2 + 3;
1076         }
1077
1078         if (req_segs > MAX_SG_ENTRIES) {
1079                 CAAM_JR_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
1080                                 MAX_SG_ENTRIES);
1081                 return NULL;
1082         }
1083
1084         ctx = caam_jr_alloc_ctx(ses);
1085         if (!ctx)
1086                 return NULL;
1087
1088         ctx->op = op;
1089         cdb = ses->cdb;
1090         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
1091
1092         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1093
1094         SEC_JD_INIT(jobdescr);
1095         SEC_JD_SET_SD(jobdescr,
1096                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1097                 cdb->sh_hdr.hi.field.idlen);
1098
1099         /* output */
1100         if (sym->m_dst)
1101                 mbuf = sym->m_dst;
1102         else
1103                 mbuf = sym->m_src;
1104
1105         out_sg = &ctx->sg[0];
1106         if (is_encode(ses))
1107                 length = sym->auth.data.length + ses->digest_length;
1108         else
1109                 length = sym->auth.data.length;
1110
1111         sg = &ctx->sg[0];
1112
1113         /* 1st seg */
1114         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
1115                 + sym->auth.data.offset);
1116         sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
1117
1118         /* Successive segs */
1119         mbuf = mbuf->next;
1120         while (mbuf) {
1121                 sg++;
1122                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
1123                 sg->len = cpu_to_caam32(mbuf->data_len);
1124                 mbuf = mbuf->next;
1125         }
1126
1127         if (is_encode(ses)) {
1128                 /* set auth output */
1129                 sg++;
1130                 sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr);
1131                 sg->len = cpu_to_caam32(ses->digest_length);
1132         }
1133         /* last element*/
1134         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1135
1136         SEC_JD_SET_OUT_PTR(jobdescr,
1137                            (uint64_t)caam_jr_dma_vtop(out_sg), 0, length);
1138         /* set sg bit */
1139         (jobdescr)->seq_out.command.word  |= 0x01000000;
1140
1141         /* input */
1142         sg++;
1143         mbuf = sym->m_src;
1144         in_sg = sg;
1145         if (is_encode(ses))
1146                 length = ses->iv.length + sym->auth.data.length;
1147         else
1148                 length = ses->iv.length + sym->auth.data.length
1149                                                 + ses->digest_length;
1150
1151         sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1152         sg->len = cpu_to_caam32(ses->iv.length);
1153
1154         sg++;
1155         /* 1st seg */
1156         sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf)
1157                 + sym->auth.data.offset);
1158         sg->len = cpu_to_caam32(mbuf->data_len - sym->auth.data.offset);
1159
1160         /* Successive segs */
1161         mbuf = mbuf->next;
1162         while (mbuf) {
1163                 sg++;
1164                 sg->ptr = cpu_to_caam64(rte_pktmbuf_iova(mbuf));
1165                 sg->len = cpu_to_caam32(mbuf->data_len);
1166                 mbuf = mbuf->next;
1167         }
1168
1169         if (is_decode(ses)) {
1170                 sg++;
1171                 rte_memcpy(ctx->digest, sym->auth.digest.data,
1172                        ses->digest_length);
1173                 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest));
1174                 sg->len = cpu_to_caam32(ses->digest_length);
1175         }
1176         /* last element*/
1177         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1178
1179         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(in_sg), 0,
1180                                 length);
1181         /* set sg bit */
1182         (jobdescr)->seq_in.command.word  |= 0x01000000;
1183         /* Auth_only_len is set as 0 in descriptor and it is
1184          * overwritten here in the jd which will update
1185          * the DPOVRD reg.
1186          */
1187         if (auth_only_len)
1188                 /* set sg bit */
1189                 (jobdescr)->dpovrd = 0x80000000 | auth_only_len;
1190
1191         return ctx;
1192 }
1193
1194 static inline struct caam_jr_op_ctx *
1195 build_cipher_auth(struct rte_crypto_op *op, struct caam_jr_session *ses)
1196 {
1197         struct rte_crypto_sym_op *sym = op->sym;
1198         struct caam_jr_op_ctx *ctx;
1199         struct sec4_sg_entry *sg;
1200         rte_iova_t src_start_addr, dst_start_addr;
1201         uint32_t length = 0;
1202         struct sec_cdb *cdb;
1203         uint64_t sdesc_offset;
1204         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1205                         ses->iv.offset);
1206         struct sec_job_descriptor_t *jobdescr;
1207         uint16_t auth_hdr_len = sym->cipher.data.offset -
1208                         sym->auth.data.offset;
1209         uint16_t auth_tail_len = sym->auth.data.length -
1210                         sym->cipher.data.length - auth_hdr_len;
1211         uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
1212
1213         src_start_addr = rte_pktmbuf_iova(sym->m_src);
1214         if (sym->m_dst)
1215                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
1216         else
1217                 dst_start_addr = src_start_addr;
1218
1219         ctx = caam_jr_alloc_ctx(ses);
1220         if (!ctx)
1221                 return NULL;
1222
1223         ctx->op = op;
1224         cdb = ses->cdb;
1225         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
1226
1227         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1228
1229         SEC_JD_INIT(jobdescr);
1230         SEC_JD_SET_SD(jobdescr,
1231                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1232                 cdb->sh_hdr.hi.field.idlen);
1233
1234         /* input */
1235         sg = &ctx->sg[0];
1236         if (is_encode(ses)) {
1237                 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1238                 sg->len = cpu_to_caam32(ses->iv.length);
1239                 length += ses->iv.length;
1240
1241                 sg++;
1242                 sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset);
1243                 sg->len = cpu_to_caam32(sym->auth.data.length);
1244                 length += sym->auth.data.length;
1245                 /* last element*/
1246                 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1247         } else {
1248                 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(IV_ptr));
1249                 sg->len = cpu_to_caam32(ses->iv.length);
1250                 length += ses->iv.length;
1251
1252                 sg++;
1253                 sg->ptr = cpu_to_caam64(src_start_addr + sym->auth.data.offset);
1254                 sg->len = cpu_to_caam32(sym->auth.data.length);
1255                 length += sym->auth.data.length;
1256
1257                 rte_memcpy(ctx->digest, sym->auth.digest.data,
1258                        ses->digest_length);
1259                 sg++;
1260                 sg->ptr = cpu_to_caam64(caam_jr_dma_vtop(ctx->digest));
1261                 sg->len = cpu_to_caam32(ses->digest_length);
1262                 length += ses->digest_length;
1263                 /* last element*/
1264                 sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1265         }
1266
1267         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)caam_jr_dma_vtop(&ctx->sg[0]), 0,
1268                                 length);
1269         /* set sg bit */
1270         (jobdescr)->seq_in.command.word  |= 0x01000000;
1271
1272         /* output */
1273         sg = &ctx->sg[6];
1274
1275         sg->ptr = cpu_to_caam64(dst_start_addr + sym->cipher.data.offset);
1276         sg->len = cpu_to_caam32(sym->cipher.data.length);
1277         length = sym->cipher.data.length;
1278
1279         if (is_encode(ses)) {
1280                 /* set auth output */
1281                 sg++;
1282                 sg->ptr = cpu_to_caam64(sym->auth.digest.phys_addr);
1283                 sg->len = cpu_to_caam32(ses->digest_length);
1284                 length += ses->digest_length;
1285         }
1286         /* last element*/
1287         sg->len |= cpu_to_caam32(SEC4_SG_LEN_FIN);
1288
1289         SEC_JD_SET_OUT_PTR(jobdescr,
1290                            (uint64_t)caam_jr_dma_vtop(&ctx->sg[6]), 0, length);
1291         /* set sg bit */
1292         (jobdescr)->seq_out.command.word  |= 0x01000000;
1293
1294         /* Auth_only_len is set as 0 in descriptor and it is
1295          * overwritten here in the jd which will update
1296          * the DPOVRD reg.
1297          */
1298         if (auth_only_len)
1299                 /* set sg bit */
1300                 (jobdescr)->dpovrd = 0x80000000 | auth_only_len;
1301
1302         return ctx;
1303 }
1304
1305 static inline struct caam_jr_op_ctx *
1306 build_proto(struct rte_crypto_op *op, struct caam_jr_session *ses)
1307 {
1308         struct rte_crypto_sym_op *sym = op->sym;
1309         struct caam_jr_op_ctx *ctx = NULL;
1310         phys_addr_t src_start_addr, dst_start_addr;
1311         struct sec_cdb *cdb;
1312         uint64_t sdesc_offset;
1313         struct sec_job_descriptor_t *jobdescr;
1314
1315         ctx = caam_jr_alloc_ctx(ses);
1316         if (!ctx)
1317                 return NULL;
1318         ctx->op = op;
1319
1320         src_start_addr = rte_pktmbuf_iova(sym->m_src);
1321         if (sym->m_dst)
1322                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
1323         else
1324                 dst_start_addr = src_start_addr;
1325
1326         cdb = ses->cdb;
1327         sdesc_offset = (size_t) ((char *)&cdb->sh_desc - (char *)cdb);
1328
1329         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1330
1331         SEC_JD_INIT(jobdescr);
1332         SEC_JD_SET_SD(jobdescr,
1333                 (phys_addr_t)(caam_jr_dma_vtop(cdb)) + sdesc_offset,
1334                         cdb->sh_hdr.hi.field.idlen);
1335
1336         /* output */
1337         SEC_JD_SET_OUT_PTR(jobdescr, (uint64_t)dst_start_addr, 0,
1338                         sym->m_src->buf_len - sym->m_src->data_off);
1339         /* input */
1340         SEC_JD_SET_IN_PTR(jobdescr, (uint64_t)src_start_addr, 0,
1341                         sym->m_src->pkt_len);
1342         sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
1343
1344         return ctx;
1345 }
1346
1347 static int
1348 caam_jr_enqueue_op(struct rte_crypto_op *op, struct caam_jr_qp *qp)
1349 {
1350         struct sec_job_ring_t *ring = qp->ring;
1351         struct caam_jr_session *ses;
1352         struct caam_jr_op_ctx *ctx = NULL;
1353         struct sec_job_descriptor_t *jobdescr __rte_unused;
1354
1355         switch (op->sess_type) {
1356         case RTE_CRYPTO_OP_WITH_SESSION:
1357                 ses = (struct caam_jr_session *)
1358                 get_sym_session_private_data(op->sym->session,
1359                                         cryptodev_driver_id);
1360                 break;
1361         case RTE_CRYPTO_OP_SECURITY_SESSION:
1362                 ses = (struct caam_jr_session *)
1363                         get_sec_session_private_data(
1364                                         op->sym->sec_session);
1365                 break;
1366         default:
1367                 CAAM_JR_DP_ERR("sessionless crypto op not supported");
1368                 qp->tx_errs++;
1369                 return -1;
1370         }
1371
1372         if (unlikely(!ses->qp || ses->qp != qp)) {
1373                 CAAM_JR_DP_DEBUG("Old:sess->qp=%p New qp = %p\n", ses->qp, qp);
1374                 ses->qp = qp;
1375                 caam_jr_prep_cdb(ses);
1376         }
1377
1378         if (rte_pktmbuf_is_contiguous(op->sym->m_src)) {
1379                 if (is_auth_cipher(ses))
1380                         ctx = build_cipher_auth(op, ses);
1381                 else if (is_aead(ses))
1382                         goto err1;
1383                 else if (is_auth_only(ses))
1384                         ctx = build_auth_only(op, ses);
1385                 else if (is_cipher_only(ses))
1386                         ctx = build_cipher_only(op, ses);
1387                 else if (is_proto_ipsec(ses))
1388                         ctx = build_proto(op, ses);
1389         } else {
1390                 if (is_auth_cipher(ses))
1391                         ctx = build_cipher_auth_sg(op, ses);
1392                 else if (is_aead(ses))
1393                         goto err1;
1394                 else if (is_auth_only(ses))
1395                         ctx = build_auth_only_sg(op, ses);
1396                 else if (is_cipher_only(ses))
1397                         ctx = build_cipher_only_sg(op, ses);
1398         }
1399 err1:
1400         if (unlikely(!ctx)) {
1401                 qp->tx_errs++;
1402                 CAAM_JR_ERR("not supported sec op");
1403                 return -1;
1404         }
1405 #if CAAM_JR_DBG
1406         if (is_decode(ses))
1407                 rte_hexdump(stdout, "DECODE",
1408                         rte_pktmbuf_mtod(op->sym->m_src, void *),
1409                         rte_pktmbuf_data_len(op->sym->m_src));
1410         else
1411                 rte_hexdump(stdout, "ENCODE",
1412                         rte_pktmbuf_mtod(op->sym->m_src, void *),
1413                         rte_pktmbuf_data_len(op->sym->m_src));
1414
1415         printf("\n JD before conversion\n");
1416         for (int i = 0; i < 12; i++)
1417                 printf("\n 0x%08x", ctx->jobdes.desc[i]);
1418 #endif
1419
1420         CAAM_JR_DP_DEBUG("Jr[%p] pi[%d] ci[%d].Before sending desc",
1421                       ring, ring->pidx, ring->cidx);
1422
1423         /* todo - do we want to retry */
1424         if (SEC_JOB_RING_IS_FULL(ring->pidx, ring->cidx,
1425                          SEC_JOB_RING_SIZE, SEC_JOB_RING_SIZE)) {
1426                 CAAM_JR_DP_DEBUG("Ring FULL Jr[%p] pi[%d] ci[%d].Size = %d",
1427                               ring, ring->pidx, ring->cidx, SEC_JOB_RING_SIZE);
1428                 caam_jr_op_ending(ctx);
1429                 qp->tx_ring_full++;
1430                 return -EBUSY;
1431         }
1432
1433 #if CORE_BYTE_ORDER != CAAM_BYTE_ORDER
1434         jobdescr = (struct sec_job_descriptor_t *) ctx->jobdes.desc;
1435
1436         jobdescr->deschdr.command.word =
1437                 cpu_to_caam32(jobdescr->deschdr.command.word);
1438         jobdescr->sd_ptr = cpu_to_caam64(jobdescr->sd_ptr);
1439         jobdescr->seq_out.command.word =
1440                 cpu_to_caam32(jobdescr->seq_out.command.word);
1441         jobdescr->seq_out_ptr = cpu_to_caam64(jobdescr->seq_out_ptr);
1442         jobdescr->out_ext_length = cpu_to_caam32(jobdescr->out_ext_length);
1443         jobdescr->seq_in.command.word =
1444                 cpu_to_caam32(jobdescr->seq_in.command.word);
1445         jobdescr->seq_in_ptr = cpu_to_caam64(jobdescr->seq_in_ptr);
1446         jobdescr->in_ext_length = cpu_to_caam32(jobdescr->in_ext_length);
1447         jobdescr->load_dpovrd.command.word =
1448                 cpu_to_caam32(jobdescr->load_dpovrd.command.word);
1449         jobdescr->dpovrd = cpu_to_caam32(jobdescr->dpovrd);
1450 #endif
1451
1452         /* Set ptr in input ring to current descriptor  */
1453         sec_write_addr(&ring->input_ring[ring->pidx],
1454                         (phys_addr_t)caam_jr_vtop_ctx(ctx, ctx->jobdes.desc));
1455         rte_smp_wmb();
1456
1457         /* Notify HW that a new job is enqueued */
1458         hw_enqueue_desc_on_job_ring(ring);
1459
1460         /* increment the producer index for the current job ring */
1461         ring->pidx = SEC_CIRCULAR_COUNTER(ring->pidx, SEC_JOB_RING_SIZE);
1462
1463         return 0;
1464 }
1465
1466 static uint16_t
1467 caam_jr_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1468                        uint16_t nb_ops)
1469 {
1470         /* Function to transmit the frames to given device and queuepair */
1471         uint32_t loop;
1472         int32_t ret;
1473         struct caam_jr_qp *jr_qp = (struct caam_jr_qp *)qp;
1474         uint16_t num_tx = 0;
1475         /*Prepare each packet which is to be sent*/
1476         for (loop = 0; loop < nb_ops; loop++) {
1477                 ret = caam_jr_enqueue_op(ops[loop], jr_qp);
1478                 if (!ret)
1479                         num_tx++;
1480         }
1481
1482         jr_qp->tx_pkts += num_tx;
1483
1484         return num_tx;
1485 }
1486
1487 /* Release queue pair */
1488 static int
1489 caam_jr_queue_pair_release(struct rte_cryptodev *dev,
1490                            uint16_t qp_id)
1491 {
1492         struct sec_job_ring_t *internals;
1493         struct caam_jr_qp *qp = NULL;
1494
1495         PMD_INIT_FUNC_TRACE();
1496         CAAM_JR_DEBUG("dev =%p, queue =%d", dev, qp_id);
1497
1498         internals = dev->data->dev_private;
1499         if (qp_id >= internals->max_nb_queue_pairs) {
1500                 CAAM_JR_ERR("Max supported qpid %d",
1501                              internals->max_nb_queue_pairs);
1502                 return -EINVAL;
1503         }
1504
1505         qp = &internals->qps[qp_id];
1506         qp->ring = NULL;
1507         dev->data->queue_pairs[qp_id] = NULL;
1508
1509         return 0;
1510 }
1511
1512 /* Setup a queue pair */
1513 static int
1514 caam_jr_queue_pair_setup(
1515                 struct rte_cryptodev *dev, uint16_t qp_id,
1516                 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
1517                 __rte_unused int socket_id)
1518 {
1519         struct sec_job_ring_t *internals;
1520         struct caam_jr_qp *qp = NULL;
1521
1522         PMD_INIT_FUNC_TRACE();
1523         CAAM_JR_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);
1524
1525         internals = dev->data->dev_private;
1526         if (qp_id >= internals->max_nb_queue_pairs) {
1527                 CAAM_JR_ERR("Max supported qpid %d",
1528                              internals->max_nb_queue_pairs);
1529                 return -EINVAL;
1530         }
1531
1532         qp = &internals->qps[qp_id];
1533         qp->ring = internals;
1534         dev->data->queue_pairs[qp_id] = qp;
1535
1536         return 0;
1537 }
1538
1539 /* Returns the size of the aesni gcm session structure */
1540 static unsigned int
1541 caam_jr_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
1542 {
1543         PMD_INIT_FUNC_TRACE();
1544
1545         return sizeof(struct caam_jr_session);
1546 }
1547
1548 static int
1549 caam_jr_cipher_init(struct rte_cryptodev *dev __rte_unused,
1550                     struct rte_crypto_sym_xform *xform,
1551                     struct caam_jr_session *session)
1552 {
1553         session->cipher_alg = xform->cipher.algo;
1554         session->iv.length = xform->cipher.iv.length;
1555         session->iv.offset = xform->cipher.iv.offset;
1556         session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
1557                                                RTE_CACHE_LINE_SIZE);
1558         if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
1559                 CAAM_JR_ERR("No Memory for cipher key\n");
1560                 return -ENOMEM;
1561         }
1562         session->cipher_key.length = xform->cipher.key.length;
1563
1564         memcpy(session->cipher_key.data, xform->cipher.key.data,
1565                xform->cipher.key.length);
1566         session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
1567                         DIR_ENC : DIR_DEC;
1568
1569         return 0;
1570 }
1571
1572 static int
1573 caam_jr_auth_init(struct rte_cryptodev *dev __rte_unused,
1574                   struct rte_crypto_sym_xform *xform,
1575                   struct caam_jr_session *session)
1576 {
1577         session->auth_alg = xform->auth.algo;
1578         session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
1579                                              RTE_CACHE_LINE_SIZE);
1580         if (session->auth_key.data == NULL && xform->auth.key.length > 0) {
1581                 CAAM_JR_ERR("No Memory for auth key\n");
1582                 return -ENOMEM;
1583         }
1584         session->auth_key.length = xform->auth.key.length;
1585         session->digest_length = xform->auth.digest_length;
1586
1587         memcpy(session->auth_key.data, xform->auth.key.data,
1588                xform->auth.key.length);
1589         session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
1590                         DIR_ENC : DIR_DEC;
1591
1592         return 0;
1593 }
1594
1595 static int
1596 caam_jr_aead_init(struct rte_cryptodev *dev __rte_unused,
1597                   struct rte_crypto_sym_xform *xform,
1598                   struct caam_jr_session *session)
1599 {
1600         session->aead_alg = xform->aead.algo;
1601         session->iv.length = xform->aead.iv.length;
1602         session->iv.offset = xform->aead.iv.offset;
1603         session->auth_only_len = xform->aead.aad_length;
1604         session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length,
1605                                              RTE_CACHE_LINE_SIZE);
1606         if (session->aead_key.data == NULL && xform->aead.key.length > 0) {
1607                 CAAM_JR_ERR("No Memory for aead key\n");
1608                 return -ENOMEM;
1609         }
1610         session->aead_key.length = xform->aead.key.length;
1611         session->digest_length = xform->aead.digest_length;
1612
1613         memcpy(session->aead_key.data, xform->aead.key.data,
1614                xform->aead.key.length);
1615         session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
1616                         DIR_ENC : DIR_DEC;
1617
1618         return 0;
1619 }
1620
1621 static int
1622 caam_jr_set_session_parameters(struct rte_cryptodev *dev,
1623                                struct rte_crypto_sym_xform *xform, void *sess)
1624 {
1625         struct sec_job_ring_t *internals = dev->data->dev_private;
1626         struct caam_jr_session *session = sess;
1627
1628         PMD_INIT_FUNC_TRACE();
1629
1630         if (unlikely(sess == NULL)) {
1631                 CAAM_JR_ERR("invalid session struct");
1632                 return -EINVAL;
1633         }
1634
1635         /* Default IV length = 0 */
1636         session->iv.length = 0;
1637
1638         /* Cipher Only */
1639         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
1640                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1641                 caam_jr_cipher_init(dev, xform, session);
1642
1643         /* Authentication Only */
1644         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1645                    xform->next == NULL) {
1646                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
1647                 caam_jr_auth_init(dev, xform, session);
1648
1649         /* Cipher then Authenticate */
1650         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1651                    xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
1652                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1653                         caam_jr_cipher_init(dev, xform, session);
1654                         caam_jr_auth_init(dev, xform->next, session);
1655                 } else {
1656                         CAAM_JR_ERR("Not supported: Auth then Cipher");
1657                         goto err1;
1658                 }
1659
1660         /* Authenticate then Cipher */
1661         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1662                    xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
1663                 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
1664                         caam_jr_auth_init(dev, xform, session);
1665                         caam_jr_cipher_init(dev, xform->next, session);
1666                 } else {
1667                         CAAM_JR_ERR("Not supported: Auth then Cipher");
1668                         goto err1;
1669                 }
1670
1671         /* AEAD operation for AES-GCM kind of Algorithms */
1672         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
1673                    xform->next == NULL) {
1674                 caam_jr_aead_init(dev, xform, session);
1675
1676         } else {
1677                 CAAM_JR_ERR("Invalid crypto type");
1678                 return -EINVAL;
1679         }
1680         session->ctx_pool = internals->ctx_pool;
1681
1682         return 0;
1683
1684 err1:
1685         rte_free(session->cipher_key.data);
1686         rte_free(session->auth_key.data);
1687         memset(session, 0, sizeof(struct caam_jr_session));
1688
1689         return -EINVAL;
1690 }
1691
1692 static int
1693 caam_jr_sym_session_configure(struct rte_cryptodev *dev,
1694                               struct rte_crypto_sym_xform *xform,
1695                               struct rte_cryptodev_sym_session *sess,
1696                               struct rte_mempool *mempool)
1697 {
1698         void *sess_private_data;
1699         int ret;
1700
1701         PMD_INIT_FUNC_TRACE();
1702
1703         if (rte_mempool_get(mempool, &sess_private_data)) {
1704                 CAAM_JR_ERR("Couldn't get object from session mempool");
1705                 return -ENOMEM;
1706         }
1707
1708         memset(sess_private_data, 0, sizeof(struct caam_jr_session));
1709         ret = caam_jr_set_session_parameters(dev, xform, sess_private_data);
1710         if (ret != 0) {
1711                 CAAM_JR_ERR("failed to configure session parameters");
1712                 /* Return session to mempool */
1713                 rte_mempool_put(mempool, sess_private_data);
1714                 return ret;
1715         }
1716
1717         set_sym_session_private_data(sess, dev->driver_id, sess_private_data);
1718
1719         return 0;
1720 }
1721
1722 /* Clear the memory of session so it doesn't leave key material behind */
1723 static void
1724 caam_jr_sym_session_clear(struct rte_cryptodev *dev,
1725                 struct rte_cryptodev_sym_session *sess)
1726 {
1727         uint8_t index = dev->driver_id;
1728         void *sess_priv = get_sym_session_private_data(sess, index);
1729         struct caam_jr_session *s = (struct caam_jr_session *)sess_priv;
1730
1731         PMD_INIT_FUNC_TRACE();
1732
1733         if (sess_priv) {
1734                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1735
1736                 rte_free(s->cipher_key.data);
1737                 rte_free(s->auth_key.data);
1738                 memset(s, 0, sizeof(struct caam_jr_session));
1739                 set_sym_session_private_data(sess, index, NULL);
1740                 rte_mempool_put(sess_mp, sess_priv);
1741         }
1742 }
1743
1744 static int
1745 caam_jr_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
1746                           struct rte_security_session_conf *conf,
1747                           void *sess)
1748 {
1749         struct sec_job_ring_t *internals = dev->data->dev_private;
1750         struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
1751         struct rte_crypto_auth_xform *auth_xform;
1752         struct rte_crypto_cipher_xform *cipher_xform;
1753         struct caam_jr_session *session = (struct caam_jr_session *)sess;
1754
1755         PMD_INIT_FUNC_TRACE();
1756
1757         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
1758                 cipher_xform = &conf->crypto_xform->cipher;
1759                 auth_xform = &conf->crypto_xform->next->auth;
1760         } else {
1761                 auth_xform = &conf->crypto_xform->auth;
1762                 cipher_xform = &conf->crypto_xform->next->cipher;
1763         }
1764         session->proto_alg = conf->protocol;
1765         session->cipher_key.data = rte_zmalloc(NULL,
1766                                                cipher_xform->key.length,
1767                                                RTE_CACHE_LINE_SIZE);
1768         if (session->cipher_key.data == NULL &&
1769                         cipher_xform->key.length > 0) {
1770                 CAAM_JR_ERR("No Memory for cipher key\n");
1771                 return -ENOMEM;
1772         }
1773
1774         session->cipher_key.length = cipher_xform->key.length;
1775         session->auth_key.data = rte_zmalloc(NULL,
1776                                         auth_xform->key.length,
1777                                         RTE_CACHE_LINE_SIZE);
1778         if (session->auth_key.data == NULL &&
1779                         auth_xform->key.length > 0) {
1780                 CAAM_JR_ERR("No Memory for auth key\n");
1781                 rte_free(session->cipher_key.data);
1782                 return -ENOMEM;
1783         }
1784         session->auth_key.length = auth_xform->key.length;
1785         memcpy(session->cipher_key.data, cipher_xform->key.data,
1786                         cipher_xform->key.length);
1787         memcpy(session->auth_key.data, auth_xform->key.data,
1788                         auth_xform->key.length);
1789
1790         switch (auth_xform->algo) {
1791         case RTE_CRYPTO_AUTH_SHA1_HMAC:
1792                 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
1793                 break;
1794         case RTE_CRYPTO_AUTH_MD5_HMAC:
1795                 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
1796                 break;
1797         case RTE_CRYPTO_AUTH_SHA256_HMAC:
1798                 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
1799                 break;
1800         case RTE_CRYPTO_AUTH_SHA384_HMAC:
1801                 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
1802                 break;
1803         case RTE_CRYPTO_AUTH_SHA512_HMAC:
1804                 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
1805                 break;
1806         case RTE_CRYPTO_AUTH_AES_CMAC:
1807                 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
1808                 break;
1809         case RTE_CRYPTO_AUTH_NULL:
1810                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1811                 break;
1812         case RTE_CRYPTO_AUTH_SHA224_HMAC:
1813         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
1814         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
1815         case RTE_CRYPTO_AUTH_SHA1:
1816         case RTE_CRYPTO_AUTH_SHA256:
1817         case RTE_CRYPTO_AUTH_SHA512:
1818         case RTE_CRYPTO_AUTH_SHA224:
1819         case RTE_CRYPTO_AUTH_SHA384:
1820         case RTE_CRYPTO_AUTH_MD5:
1821         case RTE_CRYPTO_AUTH_AES_GMAC:
1822         case RTE_CRYPTO_AUTH_KASUMI_F9:
1823         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
1824         case RTE_CRYPTO_AUTH_ZUC_EIA3:
1825                 CAAM_JR_ERR("Crypto: Unsupported auth alg %u\n",
1826                         auth_xform->algo);
1827                 goto out;
1828         default:
1829                 CAAM_JR_ERR("Crypto: Undefined Auth specified %u\n",
1830                         auth_xform->algo);
1831                 goto out;
1832         }
1833
1834         switch (cipher_xform->algo) {
1835         case RTE_CRYPTO_CIPHER_AES_CBC:
1836                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
1837                 break;
1838         case RTE_CRYPTO_CIPHER_3DES_CBC:
1839                 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
1840                 break;
1841         case RTE_CRYPTO_CIPHER_AES_CTR:
1842                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
1843                 break;
1844         case RTE_CRYPTO_CIPHER_NULL:
1845         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
1846         case RTE_CRYPTO_CIPHER_3DES_ECB:
1847         case RTE_CRYPTO_CIPHER_AES_ECB:
1848         case RTE_CRYPTO_CIPHER_KASUMI_F8:
1849                 CAAM_JR_ERR("Crypto: Unsupported Cipher alg %u\n",
1850                         cipher_xform->algo);
1851                 goto out;
1852         default:
1853                 CAAM_JR_ERR("Crypto: Undefined Cipher specified %u\n",
1854                         cipher_xform->algo);
1855                 goto out;
1856         }
1857
1858         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
1859                 memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) +
1860                                 sizeof(session->ip4_hdr));
1861                 session->ip4_hdr.ip_v = IPVERSION;
1862                 session->ip4_hdr.ip_hl = 5;
1863                 session->ip4_hdr.ip_len = rte_cpu_to_be_16(
1864                                                 sizeof(session->ip4_hdr));
1865                 session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
1866                 session->ip4_hdr.ip_id = 0;
1867                 session->ip4_hdr.ip_off = 0;
1868                 session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
1869                 session->ip4_hdr.ip_p = (ipsec_xform->proto ==
1870                                 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP
1871                                 : IPPROTO_AH;
1872                 session->ip4_hdr.ip_sum = 0;
1873                 session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
1874                 session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
1875                 session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
1876                                                 (void *)&session->ip4_hdr,
1877                                                 sizeof(struct ip));
1878
1879                 session->encap_pdb.options =
1880                         (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
1881                         PDBOPTS_ESP_OIHI_PDB_INL |
1882                         PDBOPTS_ESP_IVSRC |
1883                         PDBHMO_ESP_ENCAP_DTTL;
1884                 if (ipsec_xform->options.esn)
1885                         session->encap_pdb.options |= PDBOPTS_ESP_ESN;
1886                 session->encap_pdb.spi = ipsec_xform->spi;
1887                 session->encap_pdb.ip_hdr_len = sizeof(struct ip);
1888
1889                 session->dir = DIR_ENC;
1890         } else if (ipsec_xform->direction ==
1891                         RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
1892                 memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
1893                 session->decap_pdb.options = sizeof(struct ip) << 16;
1894                 if (ipsec_xform->options.esn)
1895                         session->decap_pdb.options |= PDBOPTS_ESP_ESN;
1896                 session->dir = DIR_DEC;
1897         } else
1898                 goto out;
1899         session->ctx_pool = internals->ctx_pool;
1900
1901         return 0;
1902 out:
1903         rte_free(session->auth_key.data);
1904         rte_free(session->cipher_key.data);
1905         memset(session, 0, sizeof(struct caam_jr_session));
1906         return -1;
1907 }
1908
1909 static int
1910 caam_jr_security_session_create(void *dev,
1911                                 struct rte_security_session_conf *conf,
1912                                 struct rte_security_session *sess,
1913                                 struct rte_mempool *mempool)
1914 {
1915         void *sess_private_data;
1916         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
1917         int ret;
1918
1919         if (rte_mempool_get(mempool, &sess_private_data)) {
1920                 CAAM_JR_ERR("Couldn't get object from session mempool");
1921                 return -ENOMEM;
1922         }
1923
1924         switch (conf->protocol) {
1925         case RTE_SECURITY_PROTOCOL_IPSEC:
1926                 ret = caam_jr_set_ipsec_session(cdev, conf,
1927                                 sess_private_data);
1928                 break;
1929         case RTE_SECURITY_PROTOCOL_MACSEC:
1930                 return -ENOTSUP;
1931         default:
1932                 return -EINVAL;
1933         }
1934         if (ret != 0) {
1935                 CAAM_JR_ERR("failed to configure session parameters");
1936                 /* Return session to mempool */
1937                 rte_mempool_put(mempool, sess_private_data);
1938                 return ret;
1939         }
1940
1941         set_sec_session_private_data(sess, sess_private_data);
1942
1943         return ret;
1944 }
1945
1946 /* Clear the memory of session so it doesn't leave key material behind */
1947 static int
1948 caam_jr_security_session_destroy(void *dev __rte_unused,
1949                                  struct rte_security_session *sess)
1950 {
1951         PMD_INIT_FUNC_TRACE();
1952         void *sess_priv = get_sec_session_private_data(sess);
1953
1954         struct caam_jr_session *s = (struct caam_jr_session *)sess_priv;
1955
1956         if (sess_priv) {
1957                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1958
1959                 rte_free(s->cipher_key.data);
1960                 rte_free(s->auth_key.data);
1961                 memset(sess, 0, sizeof(struct caam_jr_session));
1962                 set_sec_session_private_data(sess, NULL);
1963                 rte_mempool_put(sess_mp, sess_priv);
1964         }
1965         return 0;
1966 }
1967
1968
1969 static int
1970 caam_jr_dev_configure(struct rte_cryptodev *dev,
1971                        struct rte_cryptodev_config *config __rte_unused)
1972 {
1973         char str[20];
1974         struct sec_job_ring_t *internals;
1975
1976         PMD_INIT_FUNC_TRACE();
1977
1978         internals = dev->data->dev_private;
1979         snprintf(str, sizeof(str), "ctx_pool_%d", dev->data->dev_id);
1980         if (!internals->ctx_pool) {
1981                 internals->ctx_pool = rte_mempool_create((const char *)str,
1982                                                 CTX_POOL_NUM_BUFS,
1983                                                 sizeof(struct caam_jr_op_ctx),
1984                                                 CTX_POOL_CACHE_SIZE, 0,
1985                                                 NULL, NULL, NULL, NULL,
1986                                                 SOCKET_ID_ANY, 0);
1987                 if (!internals->ctx_pool) {
1988                         CAAM_JR_ERR("%s create failed\n", str);
1989                         return -ENOMEM;
1990                 }
1991         } else
1992                 CAAM_JR_INFO("mempool already created for dev_id : %d",
1993                                 dev->data->dev_id);
1994
1995         return 0;
1996 }
1997
1998 static int
1999 caam_jr_dev_start(struct rte_cryptodev *dev __rte_unused)
2000 {
2001         PMD_INIT_FUNC_TRACE();
2002         return 0;
2003 }
2004
2005 static void
2006 caam_jr_dev_stop(struct rte_cryptodev *dev __rte_unused)
2007 {
2008         PMD_INIT_FUNC_TRACE();
2009 }
2010
2011 static int
2012 caam_jr_dev_close(struct rte_cryptodev *dev)
2013 {
2014         struct sec_job_ring_t *internals;
2015
2016         PMD_INIT_FUNC_TRACE();
2017
2018         if (dev == NULL)
2019                 return -ENOMEM;
2020
2021         internals = dev->data->dev_private;
2022         rte_mempool_free(internals->ctx_pool);
2023         internals->ctx_pool = NULL;
2024
2025         return 0;
2026 }
2027
2028 static void
2029 caam_jr_dev_infos_get(struct rte_cryptodev *dev,
2030                        struct rte_cryptodev_info *info)
2031 {
2032         struct sec_job_ring_t *internals = dev->data->dev_private;
2033
2034         PMD_INIT_FUNC_TRACE();
2035         if (info != NULL) {
2036                 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
2037                 info->feature_flags = dev->feature_flags;
2038                 info->capabilities = caam_jr_get_cryptodev_capabilities();
2039                 info->sym.max_nb_sessions = internals->max_nb_sessions;
2040                 info->driver_id = cryptodev_driver_id;
2041         }
2042 }
2043
2044 static struct rte_cryptodev_ops caam_jr_ops = {
2045         .dev_configure        = caam_jr_dev_configure,
2046         .dev_start            = caam_jr_dev_start,
2047         .dev_stop             = caam_jr_dev_stop,
2048         .dev_close            = caam_jr_dev_close,
2049         .dev_infos_get        = caam_jr_dev_infos_get,
2050         .stats_get            = caam_jr_stats_get,
2051         .stats_reset          = caam_jr_stats_reset,
2052         .queue_pair_setup     = caam_jr_queue_pair_setup,
2053         .queue_pair_release   = caam_jr_queue_pair_release,
2054         .sym_session_get_size = caam_jr_sym_session_get_size,
2055         .sym_session_configure = caam_jr_sym_session_configure,
2056         .sym_session_clear    = caam_jr_sym_session_clear
2057 };
2058
2059 static struct rte_security_ops caam_jr_security_ops = {
2060         .session_create = caam_jr_security_session_create,
2061         .session_update = NULL,
2062         .session_stats_get = NULL,
2063         .session_destroy = caam_jr_security_session_destroy,
2064         .set_pkt_metadata = NULL,
2065         .capabilities_get = caam_jr_get_security_capabilities
2066 };
2067
2068 /* @brief Flush job rings of any processed descs.
2069  * The processed descs are silently dropped,
2070  * WITHOUT being notified to UA.
2071  */
2072 static void
2073 close_job_ring(struct sec_job_ring_t *job_ring)
2074 {
2075         if (job_ring->irq_fd != -1) {
2076                 /* Producer index is frozen. If consumer index is not equal
2077                  * with producer index, then we have descs to flush.
2078                  */
2079                 while (job_ring->pidx != job_ring->cidx)
2080                         hw_flush_job_ring(job_ring, false, NULL);
2081
2082                 /* free the uio job ring */
2083                 free_job_ring(job_ring->irq_fd);
2084                 job_ring->irq_fd = -1;
2085                 caam_jr_dma_free(job_ring->input_ring);
2086                 caam_jr_dma_free(job_ring->output_ring);
2087                 g_job_rings_no--;
2088         }
2089 }
2090
2091 /** @brief Release the software and hardware resources tied to a job ring.
2092  * @param [in] job_ring The job ring
2093  *
2094  * @retval  0 for success
2095  * @retval  -1 for error
2096  */
2097 static int
2098 shutdown_job_ring(struct sec_job_ring_t *job_ring)
2099 {
2100         int ret = 0;
2101
2102         PMD_INIT_FUNC_TRACE();
2103         ASSERT(job_ring != NULL);
2104         ret = hw_shutdown_job_ring(job_ring);
2105         SEC_ASSERT(ret == 0, ret,
2106                 "Failed to shutdown hardware job ring %p",
2107                 job_ring);
2108
2109         if (job_ring->coalescing_en)
2110                 hw_job_ring_disable_coalescing(job_ring);
2111
2112         if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL) {
2113                 ret = caam_jr_disable_irqs(job_ring->irq_fd);
2114                 SEC_ASSERT(ret == 0, ret,
2115                 "Failed to disable irqs for job ring %p",
2116                 job_ring);
2117         }
2118
2119         return ret;
2120 }
2121
2122 /*
2123  * @brief Release the resources used by the SEC user space driver.
2124  *
2125  * Reset and release SEC's job rings indicated by the User Application at
2126  * init_job_ring() and free any memory allocated internally.
2127  * Call once during application tear down.
2128  *
2129  * @note In case there are any descriptors in-flight (descriptors received by
2130  * SEC driver for processing and for which no response was yet provided to UA),
2131  * the descriptors are discarded without any notifications to User Application.
2132  *
2133  * @retval ::0                  is returned for a successful execution
2134  * @retval ::-1         is returned if SEC driver release is in progress
2135  */
2136 static int
2137 caam_jr_dev_uninit(struct rte_cryptodev *dev)
2138 {
2139         struct sec_job_ring_t *internals;
2140
2141         PMD_INIT_FUNC_TRACE();
2142         if (dev == NULL)
2143                 return -ENODEV;
2144
2145         internals = dev->data->dev_private;
2146         rte_free(dev->security_ctx);
2147
2148         /* If any descriptors in flight , poll and wait
2149          * until all descriptors are received and silently discarded.
2150          */
2151         if (internals) {
2152                 shutdown_job_ring(internals);
2153                 close_job_ring(internals);
2154                 rte_mempool_free(internals->ctx_pool);
2155         }
2156
2157         CAAM_JR_INFO("Closing crypto device %s", dev->data->name);
2158
2159         /* last caam jr instance) */
2160         if (g_job_rings_no == 0)
2161                 g_driver_state = SEC_DRIVER_STATE_IDLE;
2162
2163         return SEC_SUCCESS;
2164 }
2165
2166 /* @brief Initialize the software and hardware resources tied to a job ring.
2167  * @param [in] jr_mode;         Model to be used by SEC Driver to receive
2168  *                              notifications from SEC.  Can be either
2169  *                              of the three: #SEC_NOTIFICATION_TYPE_NAPI
2170  *                              #SEC_NOTIFICATION_TYPE_IRQ or
2171  *                              #SEC_NOTIFICATION_TYPE_POLL
2172  * @param [in] NAPI_mode        The NAPI work mode to configure a job ring at
2173  *                              startup. Used only when #SEC_NOTIFICATION_TYPE
2174  *                              is set to #SEC_NOTIFICATION_TYPE_NAPI.
2175  * @param [in] irq_coalescing_timer This value determines the maximum
2176  *                                      amount of time after processing a
2177  *                                      descriptor before raising an interrupt.
2178  * @param [in] irq_coalescing_count This value determines how many
2179  *                                      descriptors are completed before
2180  *                                      raising an interrupt.
2181  * @param [in] reg_base_addr,   The job ring base address register
2182  * @param [in] irq_id           The job ring interrupt identification number.
2183  * @retval  job_ring_handle for successful job ring configuration
2184  * @retval  NULL on error
2185  *
2186  */
2187 static void *
2188 init_job_ring(void *reg_base_addr, int irq_id)
2189 {
2190         struct sec_job_ring_t *job_ring = NULL;
2191         int i, ret = 0;
2192         int jr_mode = SEC_NOTIFICATION_TYPE_POLL;
2193         int napi_mode = 0;
2194         int irq_coalescing_timer = 0;
2195         int irq_coalescing_count = 0;
2196
2197         for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
2198                 if (g_job_rings[i].irq_fd == -1) {
2199                         job_ring = &g_job_rings[i];
2200                         g_job_rings_no++;
2201                         break;
2202                 }
2203         }
2204         if (job_ring == NULL) {
2205                 CAAM_JR_ERR("No free job ring\n");
2206                 return NULL;
2207         }
2208
2209         job_ring->register_base_addr = reg_base_addr;
2210         job_ring->jr_mode = jr_mode;
2211         job_ring->napi_mode = 0;
2212         job_ring->irq_fd = irq_id;
2213
2214         /* Allocate mem for input and output ring */
2215
2216         /* Allocate memory for input ring */
2217         job_ring->input_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
2218                                 SEC_DMA_MEM_INPUT_RING_SIZE);
2219         memset(job_ring->input_ring, 0, SEC_DMA_MEM_INPUT_RING_SIZE);
2220
2221         /* Allocate memory for output ring */
2222         job_ring->output_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
2223                                 SEC_DMA_MEM_OUTPUT_RING_SIZE);
2224         memset(job_ring->output_ring, 0, SEC_DMA_MEM_OUTPUT_RING_SIZE);
2225
2226         /* Reset job ring in SEC hw and configure job ring registers */
2227         ret = hw_reset_job_ring(job_ring);
2228         if (ret != 0) {
2229                 CAAM_JR_ERR("Failed to reset hardware job ring");
2230                 goto cleanup;
2231         }
2232
2233         if (jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
2234         /* When SEC US driver works in NAPI mode, the UA can select
2235          * if the driver starts with IRQs on or off.
2236          */
2237                 if (napi_mode == SEC_STARTUP_INTERRUPT_MODE) {
2238                         CAAM_JR_INFO("Enabling DONE IRQ generationon job ring - %p",
2239                                 job_ring);
2240                         ret = caam_jr_enable_irqs(job_ring->irq_fd);
2241                         if (ret != 0) {
2242                                 CAAM_JR_ERR("Failed to enable irqs for job ring");
2243                                 goto cleanup;
2244                         }
2245                 }
2246         } else if (jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
2247         /* When SEC US driver works in pure interrupt mode,
2248          * IRQ's are always enabled.
2249          */
2250                 CAAM_JR_INFO("Enabling DONE IRQ generation on job ring - %p",
2251                          job_ring);
2252                 ret = caam_jr_enable_irqs(job_ring->irq_fd);
2253                 if (ret != 0) {
2254                         CAAM_JR_ERR("Failed to enable irqs for job ring");
2255                         goto cleanup;
2256                 }
2257         }
2258         if (irq_coalescing_timer || irq_coalescing_count) {
2259                 hw_job_ring_set_coalescing_param(job_ring,
2260                          irq_coalescing_timer,
2261                          irq_coalescing_count);
2262
2263                 hw_job_ring_enable_coalescing(job_ring);
2264                 job_ring->coalescing_en = 1;
2265         }
2266
2267         job_ring->jr_state = SEC_JOB_RING_STATE_STARTED;
2268         job_ring->max_nb_queue_pairs = RTE_CAAM_MAX_NB_SEC_QPS;
2269         job_ring->max_nb_sessions = RTE_CAAM_JR_PMD_MAX_NB_SESSIONS;
2270
2271         return job_ring;
2272 cleanup:
2273         caam_jr_dma_free(job_ring->output_ring);
2274         caam_jr_dma_free(job_ring->input_ring);
2275         return NULL;
2276 }
2277
2278
2279 static int
2280 caam_jr_dev_init(const char *name,
2281                  struct rte_vdev_device *vdev,
2282                  struct rte_cryptodev_pmd_init_params *init_params)
2283 {
2284         struct rte_cryptodev *dev;
2285         struct rte_security_ctx *security_instance;
2286         struct uio_job_ring *job_ring;
2287         char str[RTE_CRYPTODEV_NAME_MAX_LEN];
2288
2289         PMD_INIT_FUNC_TRACE();
2290
2291         /* Validate driver state */
2292         if (g_driver_state == SEC_DRIVER_STATE_IDLE) {
2293                 g_job_rings_max = sec_configure();
2294                 if (!g_job_rings_max) {
2295                         CAAM_JR_ERR("No job ring detected on UIO !!!!");
2296                         return -1;
2297                 }
2298                 /* Update driver state */
2299                 g_driver_state = SEC_DRIVER_STATE_STARTED;
2300         }
2301
2302         if (g_job_rings_no >= g_job_rings_max) {
2303                 CAAM_JR_ERR("No more job rings available max=%d!!!!",
2304                                 g_job_rings_max);
2305                 return -1;
2306         }
2307
2308         job_ring = config_job_ring();
2309         if (job_ring == NULL) {
2310                 CAAM_JR_ERR("failed to create job ring");
2311                 goto init_error;
2312         }
2313
2314         snprintf(str, sizeof(str), "caam_jr%d", job_ring->jr_id);
2315
2316         dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
2317         if (dev == NULL) {
2318                 CAAM_JR_ERR("failed to create cryptodev vdev");
2319                 goto cleanup;
2320         }
2321         /*TODO free it during teardown*/
2322         dev->data->dev_private = init_job_ring(job_ring->register_base_addr,
2323                                                 job_ring->uio_fd);
2324
2325         if (!dev->data->dev_private) {
2326                 CAAM_JR_ERR("Ring memory allocation failed\n");
2327                 goto cleanup2;
2328         }
2329
2330         dev->driver_id = cryptodev_driver_id;
2331         dev->dev_ops = &caam_jr_ops;
2332
2333         /* register rx/tx burst functions for data path */
2334         dev->dequeue_burst = caam_jr_dequeue_burst;
2335         dev->enqueue_burst = caam_jr_enqueue_burst;
2336         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2337                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
2338                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2339                         RTE_CRYPTODEV_FF_SECURITY |
2340                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
2341                         RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
2342                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
2343                         RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
2344                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
2345
2346         /* For secondary processes, we don't initialise any further as primary
2347          * has already done this work. Only check we don't need a different
2348          * RX function
2349          */
2350         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2351                 CAAM_JR_WARN("Device already init by primary process");
2352                 return 0;
2353         }
2354
2355         /*TODO free it during teardown*/
2356         security_instance = rte_malloc("caam_jr",
2357                                 sizeof(struct rte_security_ctx), 0);
2358         if (security_instance == NULL) {
2359                 CAAM_JR_ERR("memory allocation failed\n");
2360                 //todo error handling.
2361                 goto cleanup2;
2362         }
2363
2364         security_instance->device = (void *)dev;
2365         security_instance->ops = &caam_jr_security_ops;
2366         security_instance->sess_cnt = 0;
2367         dev->security_ctx = security_instance;
2368
2369         RTE_LOG(INFO, PMD, "%s cryptodev init\n", dev->data->name);
2370
2371         return 0;
2372
2373 cleanup2:
2374         caam_jr_dev_uninit(dev);
2375         rte_cryptodev_pmd_release_device(dev);
2376 cleanup:
2377         free_job_ring(job_ring->uio_fd);
2378 init_error:
2379         CAAM_JR_ERR("driver %s: cryptodev_caam_jr_create failed",
2380                         init_params->name);
2381
2382         return -ENXIO;
2383 }
2384
2385 /** Initialise CAAM JR crypto device */
2386 static int
2387 cryptodev_caam_jr_probe(struct rte_vdev_device *vdev)
2388 {
2389         struct rte_cryptodev_pmd_init_params init_params = {
2390                 "",
2391                 sizeof(struct sec_job_ring_t),
2392                 rte_socket_id(),
2393                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
2394         };
2395         const char *name;
2396         const char *input_args;
2397
2398         name = rte_vdev_device_name(vdev);
2399         if (name == NULL)
2400                 return -EINVAL;
2401
2402         input_args = rte_vdev_device_args(vdev);
2403         rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
2404
2405         /* if sec device version is not configured */
2406         if (!rta_get_sec_era()) {
2407                 const struct device_node *caam_node;
2408
2409                 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
2410                         const uint32_t *prop = of_get_property(caam_node,
2411                                         "fsl,sec-era",
2412                                         NULL);
2413                         if (prop) {
2414                                 rta_set_sec_era(
2415                                         INTL_SEC_ERA(cpu_to_caam32(*prop)));
2416                                 break;
2417                         }
2418                 }
2419         }
2420 #ifdef RTE_LIBRTE_PMD_CAAM_JR_BE
2421         if (rta_get_sec_era() > RTA_SEC_ERA_8) {
2422                 RTE_LOG(ERR, PMD,
2423                 "CAAM is compiled in BE mode for device with sec era > 8???\n");
2424                 return -EINVAL;
2425         }
2426 #endif
2427
2428         return caam_jr_dev_init(name, vdev, &init_params);
2429 }
2430
2431 /** Uninitialise CAAM JR crypto device */
2432 static int
2433 cryptodev_caam_jr_remove(struct rte_vdev_device *vdev)
2434 {
2435         struct rte_cryptodev *cryptodev;
2436         const char *name;
2437
2438         name = rte_vdev_device_name(vdev);
2439         if (name == NULL)
2440                 return -EINVAL;
2441
2442         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
2443         if (cryptodev == NULL)
2444                 return -ENODEV;
2445
2446         caam_jr_dev_uninit(cryptodev);
2447
2448         return rte_cryptodev_pmd_destroy(cryptodev);
2449 }
2450
2451 static void
2452 sec_job_rings_init(void)
2453 {
2454         int i;
2455
2456         for (i = 0; i < MAX_SEC_JOB_RINGS; i++)
2457                 g_job_rings[i].irq_fd = -1;
2458 }
2459
2460 static struct rte_vdev_driver cryptodev_caam_jr_drv = {
2461         .probe = cryptodev_caam_jr_probe,
2462         .remove = cryptodev_caam_jr_remove
2463 };
2464
2465 static struct cryptodev_driver caam_jr_crypto_drv;
2466
2467 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CAAM_JR_PMD, cryptodev_caam_jr_drv);
2468 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CAAM_JR_PMD,
2469         "max_nb_queue_pairs=<int>"
2470         "socket_id=<int>");
2471 RTE_PMD_REGISTER_CRYPTO_DRIVER(caam_jr_crypto_drv, cryptodev_caam_jr_drv.driver,
2472                 cryptodev_driver_id);
2473
2474 RTE_INIT(caam_jr_init)
2475 {
2476         sec_uio_job_rings_init();
2477         sec_job_rings_init();
2478 }
2479
2480 RTE_INIT(caam_jr_init_log)
2481 {
2482         caam_jr_logtype = rte_log_register("pmd.crypto.caam");
2483         if (caam_jr_logtype >= 0)
2484                 rte_log_set_level(caam_jr_logtype, RTE_LOG_NOTICE);
2485 }