drivers/crypto/dpaa_sec/dpaa_sec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2017-2018 NXP
   5  *
   6  */
   7
   8 #include <fcntl.h>
   9 #include <unistd.h>
  10 #include <sched.h>
  11 #include <net/if.h>
  12
  13 #include <rte_byteorder.h>
  14 #include <rte_common.h>
  15 #include <rte_cryptodev_pmd.h>
  16 #include <rte_crypto.h>
  17 #include <rte_cryptodev.h>
  18 #include <rte_security_driver.h>
  19 #include <rte_cycles.h>
  20 #include <rte_dev.h>
  21 #include <rte_kvargs.h>
  22 #include <rte_malloc.h>
  23 #include <rte_mbuf.h>
  24 #include <rte_memcpy.h>
  25 #include <rte_string_fns.h>
  26 #include <rte_spinlock.h>
  27
  28 #include <fsl_usd.h>
  29 #include <fsl_qman.h>
  30 #include <of.h>
  31
  32 /* RTA header files */
  33 #include <hw/desc/common.h>
  34 #include <hw/desc/algo.h>
  35 #include <hw/desc/ipsec.h>
  36
  37 #include <rte_dpaa_bus.h>
  38 #include <dpaa_sec.h>
  39 #include <dpaa_sec_log.h>
  40
  41 enum rta_sec_era rta_sec_era;
  42
  43 int dpaa_logtype_sec;
  44
  45 static uint8_t cryptodev_driver_id;
  46
  47 static __thread struct rte_crypto_op **dpaa_sec_ops;
  48 static __thread int dpaa_sec_op_nb;
  49
  50 static int
  51 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess);
  52
  53 static inline void
  54 dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx)
  55 {
  56         if (!ctx->fd_status) {
  57                 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
  58         } else {
  59                 DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status);
  60                 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
  61         }
  62
  63         /* report op status to sym->op and then free the ctx memeory  */
  64         rte_mempool_put(ctx->ctx_pool, (void *)ctx);
  65 }
  66
  67 static inline struct dpaa_sec_op_ctx *
  68 dpaa_sec_alloc_ctx(dpaa_sec_session *ses)
  69 {
  70         struct dpaa_sec_op_ctx *ctx;
  71         int retval;
  72
  73         retval = rte_mempool_get(ses->ctx_pool, (void **)(&ctx));
  74         if (!ctx || retval) {
  75                 DPAA_SEC_DP_WARN("Alloc sec descriptor failed!");
  76                 return NULL;
  77         }
  78         /*
  79          * Clear SG memory. There are 16 SG entries of 16 Bytes each.
  80          * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
  81          * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for
  82          * each packet, memset is costlier than dcbz_64().
  83          */
  84         dcbz_64(&ctx->job.sg[SG_CACHELINE_0]);
  85         dcbz_64(&ctx->job.sg[SG_CACHELINE_1]);
  86         dcbz_64(&ctx->job.sg[SG_CACHELINE_2]);
  87         dcbz_64(&ctx->job.sg[SG_CACHELINE_3]);
  88
  89         ctx->ctx_pool = ses->ctx_pool;
  90         ctx->vtop_offset = (size_t) ctx
  91                                 - rte_mempool_virt2iova(ctx);
  92
  93         return ctx;
  94 }
  95
  96 static inline rte_iova_t
  97 dpaa_mem_vtop(void *vaddr)
  98 {
  99         const struct rte_memseg *ms;
 100
 101         ms = rte_mem_virt2memseg(vaddr, NULL);
 102         if (ms)
 103                 return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr);
 104         return (size_t)NULL;
 105 }
 106
 107 static inline void *
 108 dpaa_mem_ptov(rte_iova_t paddr)
 109 {
 110         void *va;
 111
 112         va = (void *)dpaax_iova_table_get_va(paddr);
 113         if (likely(va))
 114                 return va;
 115
 116         return rte_mem_iova2virt(paddr);
 117 }
 118
 119 static void
 120 ern_sec_fq_handler(struct qman_portal *qm __rte_unused,
 121                    struct qman_fq *fq,
 122                    const struct qm_mr_entry *msg)
 123 {
 124         DPAA_SEC_DP_ERR("sec fq %d error, RC = %x, seqnum = %x\n",
 125                         fq->fqid, msg->ern.rc, msg->ern.seqnum);
 126 }
 127
 128 /* initialize the queue with dest chan as caam chan so that
 129  * all the packets in this queue could be dispatched into caam
 130  */
 131 static int
 132 dpaa_sec_init_rx(struct qman_fq *fq_in, rte_iova_t hwdesc,
 133                  uint32_t fqid_out)
 134 {
 135         struct qm_mcc_initfq fq_opts;
 136         uint32_t flags;
 137         int ret = -1;
 138
 139         /* Clear FQ options */
 140         memset(&fq_opts, 0x00, sizeof(struct qm_mcc_initfq));
 141
 142         flags = QMAN_INITFQ_FLAG_SCHED;
 143         fq_opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_CONTEXTA |
 144                           QM_INITFQ_WE_CONTEXTB;
 145
 146         qm_fqd_context_a_set64(&fq_opts.fqd, hwdesc);
 147         fq_opts.fqd.context_b = fqid_out;
 148         fq_opts.fqd.dest.channel = qm_channel_caam;
 149         fq_opts.fqd.dest.wq = 0;
 150
 151         fq_in->cb.ern  = ern_sec_fq_handler;
 152
 153         DPAA_SEC_DEBUG("in-%x out-%x", fq_in->fqid, fqid_out);
 154
 155         ret = qman_init_fq(fq_in, flags, &fq_opts);
 156         if (unlikely(ret != 0))
 157                 DPAA_SEC_ERR("qman_init_fq failed %d", ret);
 158
 159         return ret;
 160 }
 161
 162 /* something is put into in_fq and caam put the crypto result into out_fq */
 163 static enum qman_cb_dqrr_result
 164 dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused,
 165                   struct qman_fq *fq __always_unused,
 166                   const struct qm_dqrr_entry *dqrr)
 167 {
 168         const struct qm_fd *fd;
 169         struct dpaa_sec_job *job;
 170         struct dpaa_sec_op_ctx *ctx;
 171
 172         if (dpaa_sec_op_nb >= DPAA_SEC_BURST)
 173                 return qman_cb_dqrr_defer;
 174
 175         if (!(dqrr->stat & QM_DQRR_STAT_FD_VALID))
 176                 return qman_cb_dqrr_consume;
 177
 178         fd = &dqrr->fd;
 179         /* sg is embedded in an op ctx,
 180          * sg[0] is for output
 181          * sg[1] for input
 182          */
 183         job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
 184
 185         ctx = container_of(job, struct dpaa_sec_op_ctx, job);
 186         ctx->fd_status = fd->status;
 187         if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 188                 struct qm_sg_entry *sg_out;
 189                 uint32_t len;
 190
 191                 sg_out = &job->sg[0];
 192                 hw_sg_to_cpu(sg_out);
 193                 len = sg_out->length;
 194                 ctx->op->sym->m_src->pkt_len = len;
 195                 ctx->op->sym->m_src->data_len = len;
 196         }
 197         dpaa_sec_ops[dpaa_sec_op_nb++] = ctx->op;
 198         dpaa_sec_op_ending(ctx);
 199
 200         return qman_cb_dqrr_consume;
 201 }
 202
 203 /* caam result is put into this queue */
 204 static int
 205 dpaa_sec_init_tx(struct qman_fq *fq)
 206 {
 207         int ret;
 208         struct qm_mcc_initfq opts;
 209         uint32_t flags;
 210
 211         flags = QMAN_FQ_FLAG_NO_ENQUEUE | QMAN_FQ_FLAG_LOCKED |
 212                 QMAN_FQ_FLAG_DYNAMIC_FQID;
 213
 214         ret = qman_create_fq(0, flags, fq);
 215         if (unlikely(ret)) {
 216                 DPAA_SEC_ERR("qman_create_fq failed");
 217                 return ret;
 218         }
 219
 220         memset(&opts, 0, sizeof(opts));
 221         opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
 222                        QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB;
 223
 224         /* opts.fqd.dest.channel = dpaa_sec_pool_chan; */
 225
 226         fq->cb.dqrr = dqrr_out_fq_cb_rx;
 227         fq->cb.ern  = ern_sec_fq_handler;
 228
 229         ret = qman_init_fq(fq, 0, &opts);
 230         if (unlikely(ret)) {
 231                 DPAA_SEC_ERR("unable to init caam source fq!");
 232                 return ret;
 233         }
 234
 235         return ret;
 236 }
 237
 238 static inline int is_cipher_only(dpaa_sec_session *ses)
 239 {
 240         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 241                 (ses->auth_alg == RTE_CRYPTO_AUTH_NULL));
 242 }
 243
 244 static inline int is_auth_only(dpaa_sec_session *ses)
 245 {
 246         return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) &&
 247                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL));
 248 }
 249
 250 static inline int is_aead(dpaa_sec_session *ses)
 251 {
 252         return ((ses->cipher_alg == 0) &&
 253                 (ses->auth_alg == 0) &&
 254                 (ses->aead_alg != 0));
 255 }
 256
 257 static inline int is_auth_cipher(dpaa_sec_session *ses)
 258 {
 259         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 260                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) &&
 261                 (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC));
 262 }
 263
 264 static inline int is_proto_ipsec(dpaa_sec_session *ses)
 265 {
 266         return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC);
 267 }
 268
 269 static inline int is_encode(dpaa_sec_session *ses)
 270 {
 271         return ses->dir == DIR_ENC;
 272 }
 273
 274 static inline int is_decode(dpaa_sec_session *ses)
 275 {
 276         return ses->dir == DIR_DEC;
 277 }
 278
 279 static inline void
 280 caam_auth_alg(dpaa_sec_session *ses, struct alginfo *alginfo_a)
 281 {
 282         switch (ses->auth_alg) {
 283         case RTE_CRYPTO_AUTH_NULL:
 284                 alginfo_a->algtype =
 285                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 286                         OP_PCL_IPSEC_HMAC_NULL : 0;
 287                 ses->digest_length = 0;
 288                 break;
 289         case RTE_CRYPTO_AUTH_MD5_HMAC:
 290                 alginfo_a->algtype =
 291                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 292                         OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5;
 293                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 294                 break;
 295         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 296                 alginfo_a->algtype =
 297                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 298                         OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1;
 299                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 300                 break;
 301         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 302                 alginfo_a->algtype =
 303                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 304                         OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224;
 305                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 306                 break;
 307         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 308                 alginfo_a->algtype =
 309                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 310                         OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256;
 311                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 312                 break;
 313         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 314                 alginfo_a->algtype =
 315                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 316                         OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384;
 317                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 318                 break;
 319         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 320                 alginfo_a->algtype =
 321                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 322                         OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512;
 323                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 324                 break;
 325         default:
 326                 DPAA_SEC_ERR("unsupported auth alg %u", ses->auth_alg);
 327         }
 328 }
 329
 330 static inline void
 331 caam_cipher_alg(dpaa_sec_session *ses, struct alginfo *alginfo_c)
 332 {
 333         switch (ses->cipher_alg) {
 334         case RTE_CRYPTO_CIPHER_NULL:
 335                 alginfo_c->algtype =
 336                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 337                         OP_PCL_IPSEC_NULL : 0;
 338                 break;
 339         case RTE_CRYPTO_CIPHER_AES_CBC:
 340                 alginfo_c->algtype =
 341                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 342                         OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES;
 343                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 344                 break;
 345         case RTE_CRYPTO_CIPHER_3DES_CBC:
 346                 alginfo_c->algtype =
 347                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 348                         OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES;
 349                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 350                 break;
 351         case RTE_CRYPTO_CIPHER_AES_CTR:
 352                 alginfo_c->algtype =
 353                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 354                         OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES;
 355                 alginfo_c->algmode = OP_ALG_AAI_CTR;
 356                 break;
 357         default:
 358                 DPAA_SEC_ERR("unsupported cipher alg %d", ses->cipher_alg);
 359         }
 360 }
 361
 362 static inline void
 363 caam_aead_alg(dpaa_sec_session *ses, struct alginfo *alginfo)
 364 {
 365         switch (ses->aead_alg) {
 366         case RTE_CRYPTO_AEAD_AES_GCM:
 367                 alginfo->algtype = OP_ALG_ALGSEL_AES;
 368                 alginfo->algmode = OP_ALG_AAI_GCM;
 369                 break;
 370         default:
 371                 DPAA_SEC_ERR("unsupported AEAD alg %d", ses->aead_alg);
 372         }
 373 }
 374
 375 /* prepare ipsec proto command block of the session */
 376 static int
 377 dpaa_sec_prep_ipsec_cdb(dpaa_sec_session *ses)
 378 {
 379         struct alginfo cipherdata = {0}, authdata = {0};
 380         struct sec_cdb *cdb = &ses->cdb;
 381         int32_t shared_desc_len = 0;
 382         int err;
 383 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 384         int swap = false;
 385 #else
 386         int swap = true;
 387 #endif
 388
 389         caam_cipher_alg(ses, &cipherdata);
 390         if (cipherdata.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 391                 DPAA_SEC_ERR("not supported cipher alg");
 392                 return -ENOTSUP;
 393         }
 394
 395         cipherdata.key = (size_t)ses->cipher_key.data;
 396         cipherdata.keylen = ses->cipher_key.length;
 397         cipherdata.key_enc_flags = 0;
 398         cipherdata.key_type = RTA_DATA_IMM;
 399
 400         caam_auth_alg(ses, &authdata);
 401         if (authdata.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 402                 DPAA_SEC_ERR("not supported auth alg");
 403                 return -ENOTSUP;
 404         }
 405
 406         authdata.key = (size_t)ses->auth_key.data;
 407         authdata.keylen = ses->auth_key.length;
 408         authdata.key_enc_flags = 0;
 409         authdata.key_type = RTA_DATA_IMM;
 410
 411         cdb->sh_desc[0] = cipherdata.keylen;
 412         cdb->sh_desc[1] = authdata.keylen;
 413         err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
 414                                MIN_JOB_DESC_SIZE,
 415                                (unsigned int *)cdb->sh_desc,
 416                                &cdb->sh_desc[2], 2);
 417
 418         if (err < 0) {
 419                 DPAA_SEC_ERR("Crypto: Incorrect key lengths");
 420                 return err;
 421         }
 422         if (cdb->sh_desc[2] & 1)
 423                 cipherdata.key_type = RTA_DATA_IMM;
 424         else {
 425                 cipherdata.key = (size_t)dpaa_mem_vtop(
 426                                         (void *)(size_t)cipherdata.key);
 427                 cipherdata.key_type = RTA_DATA_PTR;
 428         }
 429         if (cdb->sh_desc[2] & (1<<1))
 430                 authdata.key_type = RTA_DATA_IMM;
 431         else {
 432                 authdata.key = (size_t)dpaa_mem_vtop(
 433                                         (void *)(size_t)authdata.key);
 434                 authdata.key_type = RTA_DATA_PTR;
 435         }
 436
 437         cdb->sh_desc[0] = 0;
 438         cdb->sh_desc[1] = 0;
 439         cdb->sh_desc[2] = 0;
 440         if (ses->dir == DIR_ENC) {
 441                 shared_desc_len = cnstr_shdsc_ipsec_new_encap(
 442                                 cdb->sh_desc,
 443                                 true, swap, SHR_SERIAL,
 444                                 &ses->encap_pdb,
 445                                 (uint8_t *)&ses->ip4_hdr,
 446                                 &cipherdata, &authdata);
 447         } else if (ses->dir == DIR_DEC) {
 448                 shared_desc_len = cnstr_shdsc_ipsec_new_decap(
 449                                 cdb->sh_desc,
 450                                 true, swap, SHR_SERIAL,
 451                                 &ses->decap_pdb,
 452                                 &cipherdata, &authdata);
 453         }
 454         return shared_desc_len;
 455 }
 456
 457 /* prepare command block of the session */
 458 static int
 459 dpaa_sec_prep_cdb(dpaa_sec_session *ses)
 460 {
 461         struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0};
 462         int32_t shared_desc_len = 0;
 463         struct sec_cdb *cdb = &ses->cdb;
 464         int err;
 465 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 466         int swap = false;
 467 #else
 468         int swap = true;
 469 #endif
 470
 471         memset(cdb, 0, sizeof(struct sec_cdb));
 472
 473         if (is_proto_ipsec(ses)) {
 474                 shared_desc_len = dpaa_sec_prep_ipsec_cdb(ses);
 475         } else if (is_cipher_only(ses)) {
 476                 caam_cipher_alg(ses, &alginfo_c);
 477                 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 478                         DPAA_SEC_ERR("not supported cipher alg");
 479                         return -ENOTSUP;
 480                 }
 481
 482                 alginfo_c.key = (size_t)ses->cipher_key.data;
 483                 alginfo_c.keylen = ses->cipher_key.length;
 484                 alginfo_c.key_enc_flags = 0;
 485                 alginfo_c.key_type = RTA_DATA_IMM;
 486
 487                 shared_desc_len = cnstr_shdsc_blkcipher(
 488                                                 cdb->sh_desc, true,
 489                                                 swap, &alginfo_c,
 490                                                 NULL,
 491                                                 ses->iv.length,
 492                                                 ses->dir);
 493         } else if (is_auth_only(ses)) {
 494                 caam_auth_alg(ses, &alginfo_a);
 495                 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 496                         DPAA_SEC_ERR("not supported auth alg");
 497                         return -ENOTSUP;
 498                 }
 499
 500                 alginfo_a.key = (size_t)ses->auth_key.data;
 501                 alginfo_a.keylen = ses->auth_key.length;
 502                 alginfo_a.key_enc_flags = 0;
 503                 alginfo_a.key_type = RTA_DATA_IMM;
 504
 505                 shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true,
 506                                                    swap, &alginfo_a,
 507                                                    !ses->dir,
 508                                                    ses->digest_length);
 509         } else if (is_aead(ses)) {
 510                 caam_aead_alg(ses, &alginfo);
 511                 if (alginfo.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 512                         DPAA_SEC_ERR("not supported aead alg");
 513                         return -ENOTSUP;
 514                 }
 515                 alginfo.key = (size_t)ses->aead_key.data;
 516                 alginfo.keylen = ses->aead_key.length;
 517                 alginfo.key_enc_flags = 0;
 518                 alginfo.key_type = RTA_DATA_IMM;
 519
 520                 if (ses->dir == DIR_ENC)
 521                         shared_desc_len = cnstr_shdsc_gcm_encap(
 522                                         cdb->sh_desc, true, swap,
 523                                         &alginfo,
 524                                         ses->iv.length,
 525                                         ses->digest_length);
 526                 else
 527                         shared_desc_len = cnstr_shdsc_gcm_decap(
 528                                         cdb->sh_desc, true, swap,
 529                                         &alginfo,
 530                                         ses->iv.length,
 531                                         ses->digest_length);
 532         } else {
 533                 caam_cipher_alg(ses, &alginfo_c);
 534                 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 535                         DPAA_SEC_ERR("not supported cipher alg");
 536                         return -ENOTSUP;
 537                 }
 538
 539                 alginfo_c.key = (size_t)ses->cipher_key.data;
 540                 alginfo_c.keylen = ses->cipher_key.length;
 541                 alginfo_c.key_enc_flags = 0;
 542                 alginfo_c.key_type = RTA_DATA_IMM;
 543
 544                 caam_auth_alg(ses, &alginfo_a);
 545                 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 546                         DPAA_SEC_ERR("not supported auth alg");
 547                         return -ENOTSUP;
 548                 }
 549
 550                 alginfo_a.key = (size_t)ses->auth_key.data;
 551                 alginfo_a.keylen = ses->auth_key.length;
 552                 alginfo_a.key_enc_flags = 0;
 553                 alginfo_a.key_type = RTA_DATA_IMM;
 554
 555                 cdb->sh_desc[0] = alginfo_c.keylen;
 556                 cdb->sh_desc[1] = alginfo_a.keylen;
 557                 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
 558                                        MIN_JOB_DESC_SIZE,
 559                                        (unsigned int *)cdb->sh_desc,
 560                                        &cdb->sh_desc[2], 2);
 561
 562                 if (err < 0) {
 563                         DPAA_SEC_ERR("Crypto: Incorrect key lengths");
 564                         return err;
 565                 }
 566                 if (cdb->sh_desc[2] & 1)
 567                         alginfo_c.key_type = RTA_DATA_IMM;
 568                 else {
 569                         alginfo_c.key = (size_t)dpaa_mem_vtop(
 570                                                 (void *)(size_t)alginfo_c.key);
 571                         alginfo_c.key_type = RTA_DATA_PTR;
 572                 }
 573                 if (cdb->sh_desc[2] & (1<<1))
 574                         alginfo_a.key_type = RTA_DATA_IMM;
 575                 else {
 576                         alginfo_a.key = (size_t)dpaa_mem_vtop(
 577                                                 (void *)(size_t)alginfo_a.key);
 578                         alginfo_a.key_type = RTA_DATA_PTR;
 579                 }
 580                 cdb->sh_desc[0] = 0;
 581                 cdb->sh_desc[1] = 0;
 582                 cdb->sh_desc[2] = 0;
 583                 /* Auth_only_len is set as 0 here and it will be
 584                  * overwritten in fd for each packet.
 585                  */
 586                 shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
 587                                 true, swap, &alginfo_c, &alginfo_a,
 588                                 ses->iv.length, 0,
 589                                 ses->digest_length, ses->dir);
 590         }
 591
 592         if (shared_desc_len < 0) {
 593                 DPAA_SEC_ERR("error in preparing command block");
 594                 return shared_desc_len;
 595         }
 596
 597         cdb->sh_hdr.hi.field.idlen = shared_desc_len;
 598         cdb->sh_hdr.hi.word = rte_cpu_to_be_32(cdb->sh_hdr.hi.word);
 599         cdb->sh_hdr.lo.word = rte_cpu_to_be_32(cdb->sh_hdr.lo.word);
 600
 601         return 0;
 602 }
 603
 604 /* qp is lockless, should be accessed by only one thread */
 605 static int
 606 dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops)
 607 {
 608         struct qman_fq *fq;
 609         unsigned int pkts = 0;
 610         int num_rx_bufs, ret;
 611         struct qm_dqrr_entry *dq;
 612         uint32_t vdqcr_flags = 0;
 613
 614         fq = &qp->outq;
 615         /*
 616          * Until request for four buffers, we provide exact number of buffers.
 617          * Otherwise we do not set the QM_VDQCR_EXACT flag.
 618          * Not setting QM_VDQCR_EXACT flag can provide two more buffers than
 619          * requested, so we request two less in this case.
 620          */
 621         if (nb_ops < 4) {
 622                 vdqcr_flags = QM_VDQCR_EXACT;
 623                 num_rx_bufs = nb_ops;
 624         } else {
 625                 num_rx_bufs = nb_ops > DPAA_MAX_DEQUEUE_NUM_FRAMES ?
 626                         (DPAA_MAX_DEQUEUE_NUM_FRAMES - 2) : (nb_ops - 2);
 627         }
 628         ret = qman_set_vdq(fq, num_rx_bufs, vdqcr_flags);
 629         if (ret)
 630                 return 0;
 631
 632         do {
 633                 const struct qm_fd *fd;
 634                 struct dpaa_sec_job *job;
 635                 struct dpaa_sec_op_ctx *ctx;
 636                 struct rte_crypto_op *op;
 637
 638                 dq = qman_dequeue(fq);
 639                 if (!dq)
 640                         continue;
 641
 642                 fd = &dq->fd;
 643                 /* sg is embedded in an op ctx,
 644                  * sg[0] is for output
 645                  * sg[1] for input
 646                  */
 647                 job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
 648
 649                 ctx = container_of(job, struct dpaa_sec_op_ctx, job);
 650                 ctx->fd_status = fd->status;
 651                 op = ctx->op;
 652                 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 653                         struct qm_sg_entry *sg_out;
 654                         uint32_t len;
 655
 656                         sg_out = &job->sg[0];
 657                         hw_sg_to_cpu(sg_out);
 658                         len = sg_out->length;
 659                         op->sym->m_src->pkt_len = len;
 660                         op->sym->m_src->data_len = len;
 661                 }
 662                 if (!ctx->fd_status) {
 663                         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 664                 } else {
 665                         DPAA_SEC_DP_WARN("SEC return err:0x%x", ctx->fd_status);
 666                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 667                 }
 668                 ops[pkts++] = op;
 669
 670                 /* report op status to sym->op and then free the ctx memeory */
 671                 rte_mempool_put(ctx->ctx_pool, (void *)ctx);
 672
 673                 qman_dqrr_consume(fq, dq);
 674         } while (fq->flags & QMAN_FQ_STATE_VDQCR);
 675
 676         return pkts;
 677 }
 678
 679 static inline struct dpaa_sec_job *
 680 build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 681 {
 682         struct rte_crypto_sym_op *sym = op->sym;
 683         struct rte_mbuf *mbuf = sym->m_src;
 684         struct dpaa_sec_job *cf;
 685         struct dpaa_sec_op_ctx *ctx;
 686         struct qm_sg_entry *sg, *out_sg, *in_sg;
 687         phys_addr_t start_addr;
 688         uint8_t *old_digest, extra_segs;
 689
 690         if (is_decode(ses))
 691                 extra_segs = 3;
 692         else
 693                 extra_segs = 2;
 694
 695         if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) {
 696                 DPAA_SEC_DP_ERR("Auth: Max sec segs supported is %d",
 697                                 MAX_SG_ENTRIES);
 698                 return NULL;
 699         }
 700         ctx = dpaa_sec_alloc_ctx(ses);
 701         if (!ctx)
 702                 return NULL;
 703
 704         cf = &ctx->job;
 705         ctx->op = op;
 706         old_digest = ctx->digest;
 707
 708         /* output */
 709         out_sg = &cf->sg[0];
 710         qm_sg_entry_set64(out_sg, sym->auth.digest.phys_addr);
 711         out_sg->length = ses->digest_length;
 712         cpu_to_hw_sg(out_sg);
 713
 714         /* input */
 715         in_sg = &cf->sg[1];
 716         /* need to extend the input to a compound frame */
 717         in_sg->extension = 1;
 718         in_sg->final = 1;
 719         in_sg->length = sym->auth.data.length;
 720         qm_sg_entry_set64(in_sg, dpaa_mem_vtop(&cf->sg[2]));
 721
 722         /* 1st seg */
 723         sg = in_sg + 1;
 724         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 725         sg->length = mbuf->data_len - sym->auth.data.offset;
 726         sg->offset = sym->auth.data.offset;
 727
 728         /* Successive segs */
 729         mbuf = mbuf->next;
 730         while (mbuf) {
 731                 cpu_to_hw_sg(sg);
 732                 sg++;
 733                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 734                 sg->length = mbuf->data_len;
 735                 mbuf = mbuf->next;
 736         }
 737
 738         if (is_decode(ses)) {
 739                 /* Digest verification case */
 740                 cpu_to_hw_sg(sg);
 741                 sg++;
 742                 rte_memcpy(old_digest, sym->auth.digest.data,
 743                                 ses->digest_length);
 744                 start_addr = dpaa_mem_vtop(old_digest);
 745                 qm_sg_entry_set64(sg, start_addr);
 746                 sg->length = ses->digest_length;
 747                 in_sg->length += ses->digest_length;
 748         } else {
 749                 /* Digest calculation case */
 750                 sg->length -= ses->digest_length;
 751         }
 752         sg->final = 1;
 753         cpu_to_hw_sg(sg);
 754         cpu_to_hw_sg(in_sg);
 755
 756         return cf;
 757 }
 758
 759 /**
 760  * packet looks like:
 761  *              |<----data_len------->|
 762  *    |ip_header|ah_header|icv|payload|
 763  *              ^
 764  *              |
 765  *         mbuf->pkt.data
 766  */
 767 static inline struct dpaa_sec_job *
 768 build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
 769 {
 770         struct rte_crypto_sym_op *sym = op->sym;
 771         struct rte_mbuf *mbuf = sym->m_src;
 772         struct dpaa_sec_job *cf;
 773         struct dpaa_sec_op_ctx *ctx;
 774         struct qm_sg_entry *sg;
 775         rte_iova_t start_addr;
 776         uint8_t *old_digest;
 777
 778         ctx = dpaa_sec_alloc_ctx(ses);
 779         if (!ctx)
 780                 return NULL;
 781
 782         cf = &ctx->job;
 783         ctx->op = op;
 784         old_digest = ctx->digest;
 785
 786         start_addr = rte_pktmbuf_iova(mbuf);
 787         /* output */
 788         sg = &cf->sg[0];
 789         qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
 790         sg->length = ses->digest_length;
 791         cpu_to_hw_sg(sg);
 792
 793         /* input */
 794         sg = &cf->sg[1];
 795         if (is_decode(ses)) {
 796                 /* need to extend the input to a compound frame */
 797                 sg->extension = 1;
 798                 qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2]));
 799                 sg->length = sym->auth.data.length + ses->digest_length;
 800                 sg->final = 1;
 801                 cpu_to_hw_sg(sg);
 802
 803                 sg = &cf->sg[2];
 804                 /* hash result or digest, save digest first */
 805                 rte_memcpy(old_digest, sym->auth.digest.data,
 806                            ses->digest_length);
 807                 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset);
 808                 sg->length = sym->auth.data.length;
 809                 cpu_to_hw_sg(sg);
 810
 811                 /* let's check digest by hw */
 812                 start_addr = dpaa_mem_vtop(old_digest);
 813                 sg++;
 814                 qm_sg_entry_set64(sg, start_addr);
 815                 sg->length = ses->digest_length;
 816                 sg->final = 1;
 817                 cpu_to_hw_sg(sg);
 818         } else {
 819                 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset);
 820                 sg->length = sym->auth.data.length;
 821                 sg->final = 1;
 822                 cpu_to_hw_sg(sg);
 823         }
 824
 825         return cf;
 826 }
 827
 828 static inline struct dpaa_sec_job *
 829 build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 830 {
 831         struct rte_crypto_sym_op *sym = op->sym;
 832         struct dpaa_sec_job *cf;
 833         struct dpaa_sec_op_ctx *ctx;
 834         struct qm_sg_entry *sg, *out_sg, *in_sg;
 835         struct rte_mbuf *mbuf;
 836         uint8_t req_segs;
 837         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 838                         ses->iv.offset);
 839
 840         if (sym->m_dst) {
 841                 mbuf = sym->m_dst;
 842                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3;
 843         } else {
 844                 mbuf = sym->m_src;
 845                 req_segs = mbuf->nb_segs * 2 + 3;
 846         }
 847
 848         if (req_segs > MAX_SG_ENTRIES) {
 849                 DPAA_SEC_DP_ERR("Cipher: Max sec segs supported is %d",
 850                                 MAX_SG_ENTRIES);
 851                 return NULL;
 852         }
 853
 854         ctx = dpaa_sec_alloc_ctx(ses);
 855         if (!ctx)
 856                 return NULL;
 857
 858         cf = &ctx->job;
 859         ctx->op = op;
 860
 861         /* output */
 862         out_sg = &cf->sg[0];
 863         out_sg->extension = 1;
 864         out_sg->length = sym->cipher.data.length;
 865         qm_sg_entry_set64(out_sg, dpaa_mem_vtop(&cf->sg[2]));
 866         cpu_to_hw_sg(out_sg);
 867
 868         /* 1st seg */
 869         sg = &cf->sg[2];
 870         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 871         sg->length = mbuf->data_len - sym->cipher.data.offset;
 872         sg->offset = sym->cipher.data.offset;
 873
 874         /* Successive segs */
 875         mbuf = mbuf->next;
 876         while (mbuf) {
 877                 cpu_to_hw_sg(sg);
 878                 sg++;
 879                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 880                 sg->length = mbuf->data_len;
 881                 mbuf = mbuf->next;
 882         }
 883         sg->final = 1;
 884         cpu_to_hw_sg(sg);
 885
 886         /* input */
 887         mbuf = sym->m_src;
 888         in_sg = &cf->sg[1];
 889         in_sg->extension = 1;
 890         in_sg->final = 1;
 891         in_sg->length = sym->cipher.data.length + ses->iv.length;
 892
 893         sg++;
 894         qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
 895         cpu_to_hw_sg(in_sg);
 896
 897         /* IV */
 898         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
 899         sg->length = ses->iv.length;
 900         cpu_to_hw_sg(sg);
 901
 902         /* 1st seg */
 903         sg++;
 904         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 905         sg->length = mbuf->data_len - sym->cipher.data.offset;
 906         sg->offset = sym->cipher.data.offset;
 907
 908         /* Successive segs */
 909         mbuf = mbuf->next;
 910         while (mbuf) {
 911                 cpu_to_hw_sg(sg);
 912                 sg++;
 913                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 914                 sg->length = mbuf->data_len;
 915                 mbuf = mbuf->next;
 916         }
 917         sg->final = 1;
 918         cpu_to_hw_sg(sg);
 919
 920         return cf;
 921 }
 922
 923 static inline struct dpaa_sec_job *
 924 build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
 925 {
 926         struct rte_crypto_sym_op *sym = op->sym;
 927         struct dpaa_sec_job *cf;
 928         struct dpaa_sec_op_ctx *ctx;
 929         struct qm_sg_entry *sg;
 930         rte_iova_t src_start_addr, dst_start_addr;
 931         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 932                         ses->iv.offset);
 933
 934         ctx = dpaa_sec_alloc_ctx(ses);
 935         if (!ctx)
 936                 return NULL;
 937
 938         cf = &ctx->job;
 939         ctx->op = op;
 940
 941         src_start_addr = rte_pktmbuf_iova(sym->m_src);
 942
 943         if (sym->m_dst)
 944                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
 945         else
 946                 dst_start_addr = src_start_addr;
 947
 948         /* output */
 949         sg = &cf->sg[0];
 950         qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset);
 951         sg->length = sym->cipher.data.length + ses->iv.length;
 952         cpu_to_hw_sg(sg);
 953
 954         /* input */
 955         sg = &cf->sg[1];
 956
 957         /* need to extend the input to a compound frame */
 958         sg->extension = 1;
 959         sg->final = 1;
 960         sg->length = sym->cipher.data.length + ses->iv.length;
 961         qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2]));
 962         cpu_to_hw_sg(sg);
 963
 964         sg = &cf->sg[2];
 965         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
 966         sg->length = ses->iv.length;
 967         cpu_to_hw_sg(sg);
 968
 969         sg++;
 970         qm_sg_entry_set64(sg, src_start_addr + sym->cipher.data.offset);
 971         sg->length = sym->cipher.data.length;
 972         sg->final = 1;
 973         cpu_to_hw_sg(sg);
 974
 975         return cf;
 976 }
 977
 978 static inline struct dpaa_sec_job *
 979 build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 980 {
 981         struct rte_crypto_sym_op *sym = op->sym;
 982         struct dpaa_sec_job *cf;
 983         struct dpaa_sec_op_ctx *ctx;
 984         struct qm_sg_entry *sg, *out_sg, *in_sg;
 985         struct rte_mbuf *mbuf;
 986         uint8_t req_segs;
 987         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 988                         ses->iv.offset);
 989
 990         if (sym->m_dst) {
 991                 mbuf = sym->m_dst;
 992                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4;
 993         } else {
 994                 mbuf = sym->m_src;
 995                 req_segs = mbuf->nb_segs * 2 + 4;
 996         }
 997
 998         if (ses->auth_only_len)
 999                 req_segs++;
1000
1001         if (req_segs > MAX_SG_ENTRIES) {
1002                 DPAA_SEC_DP_ERR("AEAD: Max sec segs supported is %d",
1003                                 MAX_SG_ENTRIES);
1004                 return NULL;
1005         }
1006
1007         ctx = dpaa_sec_alloc_ctx(ses);
1008         if (!ctx)
1009                 return NULL;
1010
1011         cf = &ctx->job;
1012         ctx->op = op;
1013
1014         rte_prefetch0(cf->sg);
1015
1016         /* output */
1017         out_sg = &cf->sg[0];
1018         out_sg->extension = 1;
1019         if (is_encode(ses))
1020                 out_sg->length = sym->aead.data.length + ses->auth_only_len
1021                                                 + ses->digest_length;
1022         else
1023                 out_sg->length = sym->aead.data.length + ses->auth_only_len;
1024
1025         /* output sg entries */
1026         sg = &cf->sg[2];
1027         qm_sg_entry_set64(out_sg, dpaa_mem_vtop(sg));
1028         cpu_to_hw_sg(out_sg);
1029
1030         /* 1st seg */
1031         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1032         sg->length = mbuf->data_len - sym->aead.data.offset +
1033                                         ses->auth_only_len;
1034         sg->offset = sym->aead.data.offset - ses->auth_only_len;
1035
1036         /* Successive segs */
1037         mbuf = mbuf->next;
1038         while (mbuf) {
1039                 cpu_to_hw_sg(sg);
1040                 sg++;
1041                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1042                 sg->length = mbuf->data_len;
1043                 mbuf = mbuf->next;
1044         }
1045         sg->length -= ses->digest_length;
1046
1047         if (is_encode(ses)) {
1048                 cpu_to_hw_sg(sg);
1049                 /* set auth output */
1050                 sg++;
1051                 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr);
1052                 sg->length = ses->digest_length;
1053         }
1054         sg->final = 1;
1055         cpu_to_hw_sg(sg);
1056
1057         /* input */
1058         mbuf = sym->m_src;
1059         in_sg = &cf->sg[1];
1060         in_sg->extension = 1;
1061         in_sg->final = 1;
1062         if (is_encode(ses))
1063                 in_sg->length = ses->iv.length + sym->aead.data.length
1064                                                         + ses->auth_only_len;
1065         else
1066                 in_sg->length = ses->iv.length + sym->aead.data.length
1067                                 + ses->auth_only_len + ses->digest_length;
1068
1069         /* input sg entries */
1070         sg++;
1071         qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
1072         cpu_to_hw_sg(in_sg);
1073
1074         /* 1st seg IV */
1075         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1076         sg->length = ses->iv.length;
1077         cpu_to_hw_sg(sg);
1078
1079         /* 2nd seg auth only */
1080         if (ses->auth_only_len) {
1081                 sg++;
1082                 qm_sg_entry_set64(sg, dpaa_mem_vtop(sym->aead.aad.data));
1083                 sg->length = ses->auth_only_len;
1084                 cpu_to_hw_sg(sg);
1085         }
1086
1087         /* 3rd seg */
1088         sg++;
1089         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1090         sg->length = mbuf->data_len - sym->aead.data.offset;
1091         sg->offset = sym->aead.data.offset;
1092
1093         /* Successive segs */
1094         mbuf = mbuf->next;
1095         while (mbuf) {
1096                 cpu_to_hw_sg(sg);
1097                 sg++;
1098                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1099                 sg->length = mbuf->data_len;
1100                 mbuf = mbuf->next;
1101         }
1102
1103         if (is_decode(ses)) {
1104                 cpu_to_hw_sg(sg);
1105                 sg++;
1106                 memcpy(ctx->digest, sym->aead.digest.data,
1107                         ses->digest_length);
1108                 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
1109                 sg->length = ses->digest_length;
1110         }
1111         sg->final = 1;
1112         cpu_to_hw_sg(sg);
1113
1114         return cf;
1115 }
1116
1117 static inline struct dpaa_sec_job *
1118 build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses)
1119 {
1120         struct rte_crypto_sym_op *sym = op->sym;
1121         struct dpaa_sec_job *cf;
1122         struct dpaa_sec_op_ctx *ctx;
1123         struct qm_sg_entry *sg;
1124         uint32_t length = 0;
1125         rte_iova_t src_start_addr, dst_start_addr;
1126         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1127                         ses->iv.offset);
1128
1129         src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
1130
1131         if (sym->m_dst)
1132                 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
1133         else
1134                 dst_start_addr = src_start_addr;
1135
1136         ctx = dpaa_sec_alloc_ctx(ses);
1137         if (!ctx)
1138                 return NULL;
1139
1140         cf = &ctx->job;
1141         ctx->op = op;
1142
1143         /* input */
1144         rte_prefetch0(cf->sg);
1145         sg = &cf->sg[2];
1146         qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg));
1147         if (is_encode(ses)) {
1148                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1149                 sg->length = ses->iv.length;
1150                 length += sg->length;
1151                 cpu_to_hw_sg(sg);
1152
1153                 sg++;
1154                 if (ses->auth_only_len) {
1155                         qm_sg_entry_set64(sg,
1156                                           dpaa_mem_vtop(sym->aead.aad.data));
1157                         sg->length = ses->auth_only_len;
1158                         length += sg->length;
1159                         cpu_to_hw_sg(sg);
1160                         sg++;
1161                 }
1162                 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
1163                 sg->length = sym->aead.data.length;
1164                 length += sg->length;
1165                 sg->final = 1;
1166                 cpu_to_hw_sg(sg);
1167         } else {
1168                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1169                 sg->length = ses->iv.length;
1170                 length += sg->length;
1171                 cpu_to_hw_sg(sg);
1172
1173                 sg++;
1174                 if (ses->auth_only_len) {
1175                         qm_sg_entry_set64(sg,
1176                                           dpaa_mem_vtop(sym->aead.aad.data));
1177                         sg->length = ses->auth_only_len;
1178                         length += sg->length;
1179                         cpu_to_hw_sg(sg);
1180                         sg++;
1181                 }
1182                 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
1183                 sg->length = sym->aead.data.length;
1184                 length += sg->length;
1185                 cpu_to_hw_sg(sg);
1186
1187                 memcpy(ctx->digest, sym->aead.digest.data,
1188                        ses->digest_length);
1189                 sg++;
1190
1191                 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
1192                 sg->length = ses->digest_length;
1193                 length += sg->length;
1194                 sg->final = 1;
1195                 cpu_to_hw_sg(sg);
1196         }
1197         /* input compound frame */
1198         cf->sg[1].length = length;
1199         cf->sg[1].extension = 1;
1200         cf->sg[1].final = 1;
1201         cpu_to_hw_sg(&cf->sg[1]);
1202
1203         /* output */
1204         sg++;
1205         qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg));
1206         qm_sg_entry_set64(sg,
1207                 dst_start_addr + sym->aead.data.offset - ses->auth_only_len);
1208         sg->length = sym->aead.data.length + ses->auth_only_len;
1209         length = sg->length;
1210         if (is_encode(ses)) {
1211                 cpu_to_hw_sg(sg);
1212                 /* set auth output */
1213                 sg++;
1214                 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr);
1215                 sg->length = ses->digest_length;
1216                 length += sg->length;
1217         }
1218         sg->final = 1;
1219         cpu_to_hw_sg(sg);
1220
1221         /* output compound frame */
1222         cf->sg[0].length = length;
1223         cf->sg[0].extension = 1;
1224         cpu_to_hw_sg(&cf->sg[0]);
1225
1226         return cf;
1227 }
1228
1229 static inline struct dpaa_sec_job *
1230 build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
1231 {
1232         struct rte_crypto_sym_op *sym = op->sym;
1233         struct dpaa_sec_job *cf;
1234         struct dpaa_sec_op_ctx *ctx;
1235         struct qm_sg_entry *sg, *out_sg, *in_sg;
1236         struct rte_mbuf *mbuf;
1237         uint8_t req_segs;
1238         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1239                         ses->iv.offset);
1240
1241         if (sym->m_dst) {
1242                 mbuf = sym->m_dst;
1243                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4;
1244         } else {
1245                 mbuf = sym->m_src;
1246                 req_segs = mbuf->nb_segs * 2 + 4;
1247         }
1248
1249         if (req_segs > MAX_SG_ENTRIES) {
1250                 DPAA_SEC_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
1251                                 MAX_SG_ENTRIES);
1252                 return NULL;
1253         }
1254
1255         ctx = dpaa_sec_alloc_ctx(ses);
1256         if (!ctx)
1257                 return NULL;
1258
1259         cf = &ctx->job;
1260         ctx->op = op;
1261
1262         rte_prefetch0(cf->sg);
1263
1264         /* output */
1265         out_sg = &cf->sg[0];
1266         out_sg->extension = 1;
1267         if (is_encode(ses))
1268                 out_sg->length = sym->auth.data.length + ses->digest_length;
1269         else
1270                 out_sg->length = sym->auth.data.length;
1271
1272         /* output sg entries */
1273         sg = &cf->sg[2];
1274         qm_sg_entry_set64(out_sg, dpaa_mem_vtop(sg));
1275         cpu_to_hw_sg(out_sg);
1276
1277         /* 1st seg */
1278         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1279         sg->length = mbuf->data_len - sym->auth.data.offset;
1280         sg->offset = sym->auth.data.offset;
1281
1282         /* Successive segs */
1283         mbuf = mbuf->next;
1284         while (mbuf) {
1285                 cpu_to_hw_sg(sg);
1286                 sg++;
1287                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1288                 sg->length = mbuf->data_len;
1289                 mbuf = mbuf->next;
1290         }
1291         sg->length -= ses->digest_length;
1292
1293         if (is_encode(ses)) {
1294                 cpu_to_hw_sg(sg);
1295                 /* set auth output */
1296                 sg++;
1297                 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
1298                 sg->length = ses->digest_length;
1299         }
1300         sg->final = 1;
1301         cpu_to_hw_sg(sg);
1302
1303         /* input */
1304         mbuf = sym->m_src;
1305         in_sg = &cf->sg[1];
1306         in_sg->extension = 1;
1307         in_sg->final = 1;
1308         if (is_encode(ses))
1309                 in_sg->length = ses->iv.length + sym->auth.data.length;
1310         else
1311                 in_sg->length = ses->iv.length + sym->auth.data.length
1312                                                 + ses->digest_length;
1313
1314         /* input sg entries */
1315         sg++;
1316         qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
1317         cpu_to_hw_sg(in_sg);
1318
1319         /* 1st seg IV */
1320         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1321         sg->length = ses->iv.length;
1322         cpu_to_hw_sg(sg);
1323
1324         /* 2nd seg */
1325         sg++;
1326         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1327         sg->length = mbuf->data_len - sym->auth.data.offset;
1328         sg->offset = sym->auth.data.offset;
1329
1330         /* Successive segs */
1331         mbuf = mbuf->next;
1332         while (mbuf) {
1333                 cpu_to_hw_sg(sg);
1334                 sg++;
1335                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1336                 sg->length = mbuf->data_len;
1337                 mbuf = mbuf->next;
1338         }
1339
1340         sg->length -= ses->digest_length;
1341         if (is_decode(ses)) {
1342                 cpu_to_hw_sg(sg);
1343                 sg++;
1344                 memcpy(ctx->digest, sym->auth.digest.data,
1345                         ses->digest_length);
1346                 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
1347                 sg->length = ses->digest_length;
1348         }
1349         sg->final = 1;
1350         cpu_to_hw_sg(sg);
1351
1352         return cf;
1353 }
1354
1355 static inline struct dpaa_sec_job *
1356 build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses)
1357 {
1358         struct rte_crypto_sym_op *sym = op->sym;
1359         struct dpaa_sec_job *cf;
1360         struct dpaa_sec_op_ctx *ctx;
1361         struct qm_sg_entry *sg;
1362         rte_iova_t src_start_addr, dst_start_addr;
1363         uint32_t length = 0;
1364         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1365                         ses->iv.offset);
1366
1367         src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
1368         if (sym->m_dst)
1369                 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
1370         else
1371                 dst_start_addr = src_start_addr;
1372
1373         ctx = dpaa_sec_alloc_ctx(ses);
1374         if (!ctx)
1375                 return NULL;
1376
1377         cf = &ctx->job;
1378         ctx->op = op;
1379
1380         /* input */
1381         rte_prefetch0(cf->sg);
1382         sg = &cf->sg[2];
1383         qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg));
1384         if (is_encode(ses)) {
1385                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1386                 sg->length = ses->iv.length;
1387                 length += sg->length;
1388                 cpu_to_hw_sg(sg);
1389
1390                 sg++;
1391                 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
1392                 sg->length = sym->auth.data.length;
1393                 length += sg->length;
1394                 sg->final = 1;
1395                 cpu_to_hw_sg(sg);
1396         } else {
1397                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1398                 sg->length = ses->iv.length;
1399                 length += sg->length;
1400                 cpu_to_hw_sg(sg);
1401
1402                 sg++;
1403
1404                 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
1405                 sg->length = sym->auth.data.length;
1406                 length += sg->length;
1407                 cpu_to_hw_sg(sg);
1408
1409                 memcpy(ctx->digest, sym->auth.digest.data,
1410                        ses->digest_length);
1411                 sg++;
1412
1413                 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
1414                 sg->length = ses->digest_length;
1415                 length += sg->length;
1416                 sg->final = 1;
1417                 cpu_to_hw_sg(sg);
1418         }
1419         /* input compound frame */
1420         cf->sg[1].length = length;
1421         cf->sg[1].extension = 1;
1422         cf->sg[1].final = 1;
1423         cpu_to_hw_sg(&cf->sg[1]);
1424
1425         /* output */
1426         sg++;
1427         qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg));
1428         qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset);
1429         sg->length = sym->cipher.data.length;
1430         length = sg->length;
1431         if (is_encode(ses)) {
1432                 cpu_to_hw_sg(sg);
1433                 /* set auth output */
1434                 sg++;
1435                 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
1436                 sg->length = ses->digest_length;
1437                 length += sg->length;
1438         }
1439         sg->final = 1;
1440         cpu_to_hw_sg(sg);
1441
1442         /* output compound frame */
1443         cf->sg[0].length = length;
1444         cf->sg[0].extension = 1;
1445         cpu_to_hw_sg(&cf->sg[0]);
1446
1447         return cf;
1448 }
1449
1450 static inline struct dpaa_sec_job *
1451 build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses)
1452 {
1453         struct rte_crypto_sym_op *sym = op->sym;
1454         struct dpaa_sec_job *cf;
1455         struct dpaa_sec_op_ctx *ctx;
1456         struct qm_sg_entry *sg;
1457         phys_addr_t src_start_addr, dst_start_addr;
1458
1459         ctx = dpaa_sec_alloc_ctx(ses);
1460         if (!ctx)
1461                 return NULL;
1462         cf = &ctx->job;
1463         ctx->op = op;
1464
1465         src_start_addr = rte_pktmbuf_mtophys(sym->m_src);
1466
1467         if (sym->m_dst)
1468                 dst_start_addr = rte_pktmbuf_mtophys(sym->m_dst);
1469         else
1470                 dst_start_addr = src_start_addr;
1471
1472         /* input */
1473         sg = &cf->sg[1];
1474         qm_sg_entry_set64(sg, src_start_addr);
1475         sg->length = sym->m_src->pkt_len;
1476         sg->final = 1;
1477         cpu_to_hw_sg(sg);
1478
1479         sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
1480         /* output */
1481         sg = &cf->sg[0];
1482         qm_sg_entry_set64(sg, dst_start_addr);
1483         sg->length = sym->m_src->buf_len - sym->m_src->data_off;
1484         cpu_to_hw_sg(sg);
1485
1486         return cf;
1487 }
1488
1489 static uint16_t
1490 dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1491                        uint16_t nb_ops)
1492 {
1493         /* Function to transmit the frames to given device and queuepair */
1494         uint32_t loop;
1495         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp;
1496         uint16_t num_tx = 0;
1497         struct qm_fd fds[DPAA_SEC_BURST], *fd;
1498         uint32_t frames_to_send;
1499         struct rte_crypto_op *op;
1500         struct dpaa_sec_job *cf;
1501         dpaa_sec_session *ses;
1502         uint32_t auth_only_len;
1503         struct qman_fq *inq[DPAA_SEC_BURST];
1504
1505         while (nb_ops) {
1506                 frames_to_send = (nb_ops > DPAA_SEC_BURST) ?
1507                                 DPAA_SEC_BURST : nb_ops;
1508                 for (loop = 0; loop < frames_to_send; loop++) {
1509                         op = *(ops++);
1510                         switch (op->sess_type) {
1511                         case RTE_CRYPTO_OP_WITH_SESSION:
1512                                 ses = (dpaa_sec_session *)
1513                                         get_sym_session_private_data(
1514                                                         op->sym->session,
1515                                                         cryptodev_driver_id);
1516                                 break;
1517                         case RTE_CRYPTO_OP_SECURITY_SESSION:
1518                                 ses = (dpaa_sec_session *)
1519                                         get_sec_session_private_data(
1520                                                         op->sym->sec_session);
1521                                 break;
1522                         default:
1523                                 DPAA_SEC_DP_ERR(
1524                                         "sessionless crypto op not supported");
1525                                 frames_to_send = loop;
1526                                 nb_ops = loop;
1527                                 goto send_pkts;
1528                         }
1529                         if (unlikely(!ses->qp)) {
1530                                 if (dpaa_sec_attach_sess_q(qp, ses)) {
1531                                         frames_to_send = loop;
1532                                         nb_ops = loop;
1533                                         goto send_pkts;
1534                                 }
1535                         } else if (unlikely(ses->qp != qp)) {
1536                                 DPAA_SEC_DP_ERR("Old:sess->qp = %p"
1537                                         " New qp = %p\n", ses->qp, qp);
1538                                 frames_to_send = loop;
1539                                 nb_ops = loop;
1540                                 goto send_pkts;
1541                         }
1542
1543                         auth_only_len = op->sym->auth.data.length -
1544                                                 op->sym->cipher.data.length;
1545                         if (rte_pktmbuf_is_contiguous(op->sym->m_src)) {
1546                                 if (is_proto_ipsec(ses)) {
1547                                         cf = build_proto(op, ses);
1548                                 } else if (is_auth_only(ses)) {
1549                                         cf = build_auth_only(op, ses);
1550                                 } else if (is_cipher_only(ses)) {
1551                                         cf = build_cipher_only(op, ses);
1552                                 } else if (is_aead(ses)) {
1553                                         cf = build_cipher_auth_gcm(op, ses);
1554                                         auth_only_len = ses->auth_only_len;
1555                                 } else if (is_auth_cipher(ses)) {
1556                                         cf = build_cipher_auth(op, ses);
1557                                 } else {
1558                                         DPAA_SEC_DP_ERR("not supported ops");
1559                                         frames_to_send = loop;
1560                                         nb_ops = loop;
1561                                         goto send_pkts;
1562                                 }
1563                         } else {
1564                                 if (is_auth_only(ses)) {
1565                                         cf = build_auth_only_sg(op, ses);
1566                                 } else if (is_cipher_only(ses)) {
1567                                         cf = build_cipher_only_sg(op, ses);
1568                                 } else if (is_aead(ses)) {
1569                                         cf = build_cipher_auth_gcm_sg(op, ses);
1570                                         auth_only_len = ses->auth_only_len;
1571                                 } else if (is_auth_cipher(ses)) {
1572                                         cf = build_cipher_auth_sg(op, ses);
1573                                 } else {
1574                                         DPAA_SEC_DP_ERR("not supported ops");
1575                                         frames_to_send = loop;
1576                                         nb_ops = loop;
1577                                         goto send_pkts;
1578                                 }
1579                         }
1580                         if (unlikely(!cf)) {
1581                                 frames_to_send = loop;
1582                                 nb_ops = loop;
1583                                 goto send_pkts;
1584                         }
1585
1586                         fd = &fds[loop];
1587                         inq[loop] = ses->inq;
1588                         fd->opaque_addr = 0;
1589                         fd->cmd = 0;
1590                         qm_fd_addr_set64(fd, dpaa_mem_vtop(cf->sg));
1591                         fd->_format1 = qm_fd_compound;
1592                         fd->length29 = 2 * sizeof(struct qm_sg_entry);
1593                         /* Auth_only_len is set as 0 in descriptor and it is
1594                          * overwritten here in the fd.cmd which will update
1595                          * the DPOVRD reg.
1596                          */
1597                         if (auth_only_len)
1598                                 fd->cmd = 0x80000000 | auth_only_len;
1599
1600                 }
1601 send_pkts:
1602                 loop = 0;
1603                 while (loop < frames_to_send) {
1604                         loop += qman_enqueue_multi_fq(&inq[loop], &fds[loop],
1605                                         frames_to_send - loop);
1606                 }
1607                 nb_ops -= frames_to_send;
1608                 num_tx += frames_to_send;
1609         }
1610
1611         dpaa_qp->tx_pkts += num_tx;
1612         dpaa_qp->tx_errs += nb_ops - num_tx;
1613
1614         return num_tx;
1615 }
1616
1617 static uint16_t
1618 dpaa_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops,
1619                        uint16_t nb_ops)
1620 {
1621         uint16_t num_rx;
1622         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp;
1623
1624         num_rx = dpaa_sec_deq(dpaa_qp, ops, nb_ops);
1625
1626         dpaa_qp->rx_pkts += num_rx;
1627         dpaa_qp->rx_errs += nb_ops - num_rx;
1628
1629         DPAA_SEC_DP_DEBUG("SEC Received %d Packets\n", num_rx);
1630
1631         return num_rx;
1632 }
1633
1634 /** Release queue pair */
1635 static int
1636 dpaa_sec_queue_pair_release(struct rte_cryptodev *dev,
1637                             uint16_t qp_id)
1638 {
1639         struct dpaa_sec_dev_private *internals;
1640         struct dpaa_sec_qp *qp = NULL;
1641
1642         PMD_INIT_FUNC_TRACE();
1643
1644         DPAA_SEC_DEBUG("dev =%p, queue =%d", dev, qp_id);
1645
1646         internals = dev->data->dev_private;
1647         if (qp_id >= internals->max_nb_queue_pairs) {
1648                 DPAA_SEC_ERR("Max supported qpid %d",
1649                              internals->max_nb_queue_pairs);
1650                 return -EINVAL;
1651         }
1652
1653         qp = &internals->qps[qp_id];
1654         qp->internals = NULL;
1655         dev->data->queue_pairs[qp_id] = NULL;
1656
1657         return 0;
1658 }
1659
1660 /** Setup a queue pair */
1661 static int
1662 dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
1663                 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
1664                 __rte_unused int socket_id)
1665 {
1666         struct dpaa_sec_dev_private *internals;
1667         struct dpaa_sec_qp *qp = NULL;
1668
1669         DPAA_SEC_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);
1670
1671         internals = dev->data->dev_private;
1672         if (qp_id >= internals->max_nb_queue_pairs) {
1673                 DPAA_SEC_ERR("Max supported qpid %d",
1674                              internals->max_nb_queue_pairs);
1675                 return -EINVAL;
1676         }
1677
1678         qp = &internals->qps[qp_id];
1679         qp->internals = internals;
1680         dev->data->queue_pairs[qp_id] = qp;
1681
1682         return 0;
1683 }
1684
1685 /** Return the number of allocated queue pairs */
1686 static uint32_t
1687 dpaa_sec_queue_pair_count(struct rte_cryptodev *dev)
1688 {
1689         PMD_INIT_FUNC_TRACE();
1690
1691         return dev->data->nb_queue_pairs;
1692 }
1693
1694 /** Returns the size of session structure */
1695 static unsigned int
1696 dpaa_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
1697 {
1698         PMD_INIT_FUNC_TRACE();
1699
1700         return sizeof(dpaa_sec_session);
1701 }
1702
1703 static int
1704 dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused,
1705                      struct rte_crypto_sym_xform *xform,
1706                      dpaa_sec_session *session)
1707 {
1708         session->cipher_alg = xform->cipher.algo;
1709         session->iv.length = xform->cipher.iv.length;
1710         session->iv.offset = xform->cipher.iv.offset;
1711         session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
1712                                                RTE_CACHE_LINE_SIZE);
1713         if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
1714                 DPAA_SEC_ERR("No Memory for cipher key");
1715                 return -ENOMEM;
1716         }
1717         session->cipher_key.length = xform->cipher.key.length;
1718
1719         memcpy(session->cipher_key.data, xform->cipher.key.data,
1720                xform->cipher.key.length);
1721         session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
1722                         DIR_ENC : DIR_DEC;
1723
1724         return 0;
1725 }
1726
1727 static int
1728 dpaa_sec_auth_init(struct rte_cryptodev *dev __rte_unused,
1729                    struct rte_crypto_sym_xform *xform,
1730                    dpaa_sec_session *session)
1731 {
1732         session->auth_alg = xform->auth.algo;
1733         session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
1734                                              RTE_CACHE_LINE_SIZE);
1735         if (session->auth_key.data == NULL && xform->auth.key.length > 0) {
1736                 DPAA_SEC_ERR("No Memory for auth key");
1737                 return -ENOMEM;
1738         }
1739         session->auth_key.length = xform->auth.key.length;
1740         session->digest_length = xform->auth.digest_length;
1741
1742         memcpy(session->auth_key.data, xform->auth.key.data,
1743                xform->auth.key.length);
1744         session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
1745                         DIR_ENC : DIR_DEC;
1746
1747         return 0;
1748 }
1749
1750 static int
1751 dpaa_sec_aead_init(struct rte_cryptodev *dev __rte_unused,
1752                    struct rte_crypto_sym_xform *xform,
1753                    dpaa_sec_session *session)
1754 {
1755         session->aead_alg = xform->aead.algo;
1756         session->iv.length = xform->aead.iv.length;
1757         session->iv.offset = xform->aead.iv.offset;
1758         session->auth_only_len = xform->aead.aad_length;
1759         session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length,
1760                                              RTE_CACHE_LINE_SIZE);
1761         if (session->aead_key.data == NULL && xform->aead.key.length > 0) {
1762                 DPAA_SEC_ERR("No Memory for aead key\n");
1763                 return -ENOMEM;
1764         }
1765         session->aead_key.length = xform->aead.key.length;
1766         session->digest_length = xform->aead.digest_length;
1767
1768         memcpy(session->aead_key.data, xform->aead.key.data,
1769                xform->aead.key.length);
1770         session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
1771                         DIR_ENC : DIR_DEC;
1772
1773         return 0;
1774 }
1775
1776 static struct qman_fq *
1777 dpaa_sec_attach_rxq(struct dpaa_sec_dev_private *qi)
1778 {
1779         unsigned int i;
1780
1781         for (i = 0; i < qi->max_nb_sessions; i++) {
1782                 if (qi->inq_attach[i] == 0) {
1783                         qi->inq_attach[i] = 1;
1784                         return &qi->inq[i];
1785                 }
1786         }
1787         DPAA_SEC_WARN("All ses session in use %x", qi->max_nb_sessions);
1788
1789         return NULL;
1790 }
1791
1792 static int
1793 dpaa_sec_detach_rxq(struct dpaa_sec_dev_private *qi, struct qman_fq *fq)
1794 {
1795         unsigned int i;
1796
1797         for (i = 0; i < qi->max_nb_sessions; i++) {
1798                 if (&qi->inq[i] == fq) {
1799                         qman_retire_fq(fq, NULL);
1800                         qman_oos_fq(fq);
1801                         qi->inq_attach[i] = 0;
1802                         return 0;
1803                 }
1804         }
1805         return -1;
1806 }
1807
1808 static int
1809 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess)
1810 {
1811         int ret;
1812
1813         sess->qp = qp;
1814         ret = dpaa_sec_prep_cdb(sess);
1815         if (ret) {
1816                 DPAA_SEC_ERR("Unable to prepare sec cdb");
1817                 return -1;
1818         }
1819         if (unlikely(!RTE_PER_LCORE(dpaa_io))) {
1820                 ret = rte_dpaa_portal_init((void *)0);
1821                 if (ret) {
1822                         DPAA_SEC_ERR("Failure in affining portal");
1823                         return ret;
1824                 }
1825         }
1826         ret = dpaa_sec_init_rx(sess->inq, dpaa_mem_vtop(&sess->cdb),
1827                                qman_fq_fqid(&qp->outq));
1828         if (ret)
1829                 DPAA_SEC_ERR("Unable to init sec queue");
1830
1831         return ret;
1832 }
1833
1834 static int
1835 dpaa_sec_set_session_parameters(struct rte_cryptodev *dev,
1836                             struct rte_crypto_sym_xform *xform, void *sess)
1837 {
1838         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
1839         dpaa_sec_session *session = sess;
1840
1841         PMD_INIT_FUNC_TRACE();
1842
1843         if (unlikely(sess == NULL)) {
1844                 DPAA_SEC_ERR("invalid session struct");
1845                 return -EINVAL;
1846         }
1847         memset(session, 0, sizeof(dpaa_sec_session));
1848
1849         /* Default IV length = 0 */
1850         session->iv.length = 0;
1851
1852         /* Cipher Only */
1853         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
1854                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1855                 dpaa_sec_cipher_init(dev, xform, session);
1856
1857         /* Authentication Only */
1858         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1859                    xform->next == NULL) {
1860                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
1861                 dpaa_sec_auth_init(dev, xform, session);
1862
1863         /* Cipher then Authenticate */
1864         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1865                    xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
1866                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1867                         dpaa_sec_cipher_init(dev, xform, session);
1868                         dpaa_sec_auth_init(dev, xform->next, session);
1869                 } else {
1870                         DPAA_SEC_ERR("Not supported: Auth then Cipher");
1871                         return -EINVAL;
1872                 }
1873
1874         /* Authenticate then Cipher */
1875         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1876                    xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
1877                 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
1878                         dpaa_sec_auth_init(dev, xform, session);
1879                         dpaa_sec_cipher_init(dev, xform->next, session);
1880                 } else {
1881                         DPAA_SEC_ERR("Not supported: Auth then Cipher");
1882                         return -EINVAL;
1883                 }
1884
1885         /* AEAD operation for AES-GCM kind of Algorithms */
1886         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
1887                    xform->next == NULL) {
1888                 dpaa_sec_aead_init(dev, xform, session);
1889
1890         } else {
1891                 DPAA_SEC_ERR("Invalid crypto type");
1892                 return -EINVAL;
1893         }
1894         session->ctx_pool = internals->ctx_pool;
1895         rte_spinlock_lock(&internals->lock);
1896         session->inq = dpaa_sec_attach_rxq(internals);
1897         rte_spinlock_unlock(&internals->lock);
1898         if (session->inq == NULL) {
1899                 DPAA_SEC_ERR("unable to attach sec queue");
1900                 goto err1;
1901         }
1902
1903         return 0;
1904
1905 err1:
1906         rte_free(session->cipher_key.data);
1907         rte_free(session->auth_key.data);
1908         memset(session, 0, sizeof(dpaa_sec_session));
1909
1910         return -EINVAL;
1911 }
1912
1913 static int
1914 dpaa_sec_sym_session_configure(struct rte_cryptodev *dev,
1915                 struct rte_crypto_sym_xform *xform,
1916                 struct rte_cryptodev_sym_session *sess,
1917                 struct rte_mempool *mempool)
1918 {
1919         void *sess_private_data;
1920         int ret;
1921
1922         PMD_INIT_FUNC_TRACE();
1923
1924         if (rte_mempool_get(mempool, &sess_private_data)) {
1925                 DPAA_SEC_ERR("Couldn't get object from session mempool");
1926                 return -ENOMEM;
1927         }
1928
1929         ret = dpaa_sec_set_session_parameters(dev, xform, sess_private_data);
1930         if (ret != 0) {
1931                 DPAA_SEC_ERR("failed to configure session parameters");
1932
1933                 /* Return session to mempool */
1934                 rte_mempool_put(mempool, sess_private_data);
1935                 return ret;
1936         }
1937
1938         set_sym_session_private_data(sess, dev->driver_id,
1939                         sess_private_data);
1940
1941
1942         return 0;
1943 }
1944
1945 /** Clear the memory of session so it doesn't leave key material behind */
1946 static void
1947 dpaa_sec_sym_session_clear(struct rte_cryptodev *dev,
1948                 struct rte_cryptodev_sym_session *sess)
1949 {
1950         struct dpaa_sec_dev_private *qi = dev->data->dev_private;
1951         uint8_t index = dev->driver_id;
1952         void *sess_priv = get_sym_session_private_data(sess, index);
1953
1954         PMD_INIT_FUNC_TRACE();
1955
1956         dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
1957
1958         if (sess_priv) {
1959                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1960
1961                 if (s->inq)
1962                         dpaa_sec_detach_rxq(qi, s->inq);
1963                 rte_free(s->cipher_key.data);
1964                 rte_free(s->auth_key.data);
1965                 memset(s, 0, sizeof(dpaa_sec_session));
1966                 set_sym_session_private_data(sess, index, NULL);
1967                 rte_mempool_put(sess_mp, sess_priv);
1968         }
1969 }
1970
1971 static int
1972 dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
1973                            struct rte_security_session_conf *conf,
1974                            void *sess)
1975 {
1976         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
1977         struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
1978         struct rte_crypto_auth_xform *auth_xform = NULL;
1979         struct rte_crypto_cipher_xform *cipher_xform = NULL;
1980         dpaa_sec_session *session = (dpaa_sec_session *)sess;
1981
1982         PMD_INIT_FUNC_TRACE();
1983
1984         memset(session, 0, sizeof(dpaa_sec_session));
1985         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
1986                 cipher_xform = &conf->crypto_xform->cipher;
1987                 if (conf->crypto_xform->next)
1988                         auth_xform = &conf->crypto_xform->next->auth;
1989         } else {
1990                 auth_xform = &conf->crypto_xform->auth;
1991                 if (conf->crypto_xform->next)
1992                         cipher_xform = &conf->crypto_xform->next->cipher;
1993         }
1994         session->proto_alg = conf->protocol;
1995
1996         if (cipher_xform && cipher_xform->algo != RTE_CRYPTO_CIPHER_NULL) {
1997                 session->cipher_key.data = rte_zmalloc(NULL,
1998                                                        cipher_xform->key.length,
1999                                                        RTE_CACHE_LINE_SIZE);
2000                 if (session->cipher_key.data == NULL &&
2001                                 cipher_xform->key.length > 0) {
2002                         DPAA_SEC_ERR("No Memory for cipher key");
2003                         return -ENOMEM;
2004                 }
2005                 memcpy(session->cipher_key.data, cipher_xform->key.data,
2006                                 cipher_xform->key.length);
2007                 session->cipher_key.length = cipher_xform->key.length;
2008
2009                 switch (cipher_xform->algo) {
2010                 case RTE_CRYPTO_CIPHER_AES_CBC:
2011                 case RTE_CRYPTO_CIPHER_3DES_CBC:
2012                 case RTE_CRYPTO_CIPHER_AES_CTR:
2013                         break;
2014                 default:
2015                         DPAA_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2016                                 cipher_xform->algo);
2017                         goto out;
2018                 }
2019                 session->cipher_alg = cipher_xform->algo;
2020         } else {
2021                 session->cipher_key.data = NULL;
2022                 session->cipher_key.length = 0;
2023                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
2024         }
2025
2026         if (auth_xform && auth_xform->algo != RTE_CRYPTO_AUTH_NULL) {
2027                 session->auth_key.data = rte_zmalloc(NULL,
2028                                                 auth_xform->key.length,
2029                                                 RTE_CACHE_LINE_SIZE);
2030                 if (session->auth_key.data == NULL &&
2031                                 auth_xform->key.length > 0) {
2032                         DPAA_SEC_ERR("No Memory for auth key");
2033                         rte_free(session->cipher_key.data);
2034                         return -ENOMEM;
2035                 }
2036                 memcpy(session->auth_key.data, auth_xform->key.data,
2037                                 auth_xform->key.length);
2038                 session->auth_key.length = auth_xform->key.length;
2039
2040                 switch (auth_xform->algo) {
2041                 case RTE_CRYPTO_AUTH_SHA1_HMAC:
2042                 case RTE_CRYPTO_AUTH_MD5_HMAC:
2043                 case RTE_CRYPTO_AUTH_SHA256_HMAC:
2044                 case RTE_CRYPTO_AUTH_SHA384_HMAC:
2045                 case RTE_CRYPTO_AUTH_SHA512_HMAC:
2046                 case RTE_CRYPTO_AUTH_AES_CMAC:
2047                         break;
2048                 default:
2049                         DPAA_SEC_ERR("Crypto: Unsupported auth alg %u",
2050                                 auth_xform->algo);
2051                         goto out;
2052                 }
2053                 session->auth_alg = auth_xform->algo;
2054         } else {
2055                 session->auth_key.data = NULL;
2056                 session->auth_key.length = 0;
2057                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
2058         }
2059
2060         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
2061                 memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) +
2062                                 sizeof(session->ip4_hdr));
2063                 session->ip4_hdr.ip_v = IPVERSION;
2064                 session->ip4_hdr.ip_hl = 5;
2065                 session->ip4_hdr.ip_len = rte_cpu_to_be_16(
2066                                                 sizeof(session->ip4_hdr));
2067                 session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
2068                 session->ip4_hdr.ip_id = 0;
2069                 session->ip4_hdr.ip_off = 0;
2070                 session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
2071                 session->ip4_hdr.ip_p = (ipsec_xform->proto ==
2072                                 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP
2073                                 : IPPROTO_AH;
2074                 session->ip4_hdr.ip_sum = 0;
2075                 session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
2076                 session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
2077                 session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
2078                                                 (void *)&session->ip4_hdr,
2079                                                 sizeof(struct ip));
2080
2081                 session->encap_pdb.options =
2082                         (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
2083                         PDBOPTS_ESP_OIHI_PDB_INL |
2084                         PDBOPTS_ESP_IVSRC |
2085                         PDBHMO_ESP_ENCAP_DTTL |
2086                         PDBHMO_ESP_SNR;
2087                 session->encap_pdb.spi = ipsec_xform->spi;
2088                 session->encap_pdb.ip_hdr_len = sizeof(struct ip);
2089
2090                 session->dir = DIR_ENC;
2091         } else if (ipsec_xform->direction ==
2092                         RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
2093                 memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
2094                 session->decap_pdb.options = sizeof(struct ip) << 16;
2095                 session->dir = DIR_DEC;
2096         } else
2097                 goto out;
2098         session->ctx_pool = internals->ctx_pool;
2099         rte_spinlock_lock(&internals->lock);
2100         session->inq = dpaa_sec_attach_rxq(internals);
2101         rte_spinlock_unlock(&internals->lock);
2102         if (session->inq == NULL) {
2103                 DPAA_SEC_ERR("unable to attach sec queue");
2104                 goto out;
2105         }
2106
2107
2108         return 0;
2109 out:
2110         rte_free(session->auth_key.data);
2111         rte_free(session->cipher_key.data);
2112         memset(session, 0, sizeof(dpaa_sec_session));
2113         return -1;
2114 }
2115
2116 static int
2117 dpaa_sec_security_session_create(void *dev,
2118                                  struct rte_security_session_conf *conf,
2119                                  struct rte_security_session *sess,
2120                                  struct rte_mempool *mempool)
2121 {
2122         void *sess_private_data;
2123         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
2124         int ret;
2125
2126         if (rte_mempool_get(mempool, &sess_private_data)) {
2127                 DPAA_SEC_ERR("Couldn't get object from session mempool");
2128                 return -ENOMEM;
2129         }
2130
2131         switch (conf->protocol) {
2132         case RTE_SECURITY_PROTOCOL_IPSEC:
2133                 ret = dpaa_sec_set_ipsec_session(cdev, conf,
2134                                 sess_private_data);
2135                 break;
2136         case RTE_SECURITY_PROTOCOL_MACSEC:
2137                 return -ENOTSUP;
2138         default:
2139                 return -EINVAL;
2140         }
2141         if (ret != 0) {
2142                 DPAA_SEC_ERR("failed to configure session parameters");
2143                 /* Return session to mempool */
2144                 rte_mempool_put(mempool, sess_private_data);
2145                 return ret;
2146         }
2147
2148         set_sec_session_private_data(sess, sess_private_data);
2149
2150         return ret;
2151 }
2152
2153 /** Clear the memory of session so it doesn't leave key material behind */
2154 static int
2155 dpaa_sec_security_session_destroy(void *dev __rte_unused,
2156                 struct rte_security_session *sess)
2157 {
2158         PMD_INIT_FUNC_TRACE();
2159         void *sess_priv = get_sec_session_private_data(sess);
2160
2161         dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
2162
2163         if (sess_priv) {
2164                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
2165
2166                 rte_free(s->cipher_key.data);
2167                 rte_free(s->auth_key.data);
2168                 memset(sess, 0, sizeof(dpaa_sec_session));
2169                 set_sec_session_private_data(sess, NULL);
2170                 rte_mempool_put(sess_mp, sess_priv);
2171         }
2172         return 0;
2173 }
2174
2175
2176 static int
2177 dpaa_sec_dev_configure(struct rte_cryptodev *dev,
2178                        struct rte_cryptodev_config *config __rte_unused)
2179 {
2180
2181         char str[20];
2182         struct dpaa_sec_dev_private *internals;
2183
2184         PMD_INIT_FUNC_TRACE();
2185
2186         internals = dev->data->dev_private;
2187         sprintf(str, "ctx_pool_%d", dev->data->dev_id);
2188         if (!internals->ctx_pool) {
2189                 internals->ctx_pool = rte_mempool_create((const char *)str,
2190                                                         CTX_POOL_NUM_BUFS,
2191                                                         CTX_POOL_BUF_SIZE,
2192                                                         CTX_POOL_CACHE_SIZE, 0,
2193                                                         NULL, NULL, NULL, NULL,
2194                                                         SOCKET_ID_ANY, 0);
2195                 if (!internals->ctx_pool) {
2196                         DPAA_SEC_ERR("%s create failed\n", str);
2197                         return -ENOMEM;
2198                 }
2199         } else
2200                 DPAA_SEC_INFO("mempool already created for dev_id : %d",
2201                                 dev->data->dev_id);
2202
2203         return 0;
2204 }
2205
2206 static int
2207 dpaa_sec_dev_start(struct rte_cryptodev *dev __rte_unused)
2208 {
2209         PMD_INIT_FUNC_TRACE();
2210         return 0;
2211 }
2212
2213 static void
2214 dpaa_sec_dev_stop(struct rte_cryptodev *dev __rte_unused)
2215 {
2216         PMD_INIT_FUNC_TRACE();
2217 }
2218
2219 static int
2220 dpaa_sec_dev_close(struct rte_cryptodev *dev)
2221 {
2222         struct dpaa_sec_dev_private *internals;
2223
2224         PMD_INIT_FUNC_TRACE();
2225
2226         if (dev == NULL)
2227                 return -ENOMEM;
2228
2229         internals = dev->data->dev_private;
2230         rte_mempool_free(internals->ctx_pool);
2231         internals->ctx_pool = NULL;
2232
2233         return 0;
2234 }
2235
2236 static void
2237 dpaa_sec_dev_infos_get(struct rte_cryptodev *dev,
2238                        struct rte_cryptodev_info *info)
2239 {
2240         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
2241
2242         PMD_INIT_FUNC_TRACE();
2243         if (info != NULL) {
2244                 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
2245                 info->feature_flags = dev->feature_flags;
2246                 info->capabilities = dpaa_sec_capabilities;
2247                 info->sym.max_nb_sessions = internals->max_nb_sessions;
2248                 info->driver_id = cryptodev_driver_id;
2249         }
2250 }
2251
2252 static struct rte_cryptodev_ops crypto_ops = {
2253         .dev_configure        = dpaa_sec_dev_configure,
2254         .dev_start            = dpaa_sec_dev_start,
2255         .dev_stop             = dpaa_sec_dev_stop,
2256         .dev_close            = dpaa_sec_dev_close,
2257         .dev_infos_get        = dpaa_sec_dev_infos_get,
2258         .queue_pair_setup     = dpaa_sec_queue_pair_setup,
2259         .queue_pair_release   = dpaa_sec_queue_pair_release,
2260         .queue_pair_count     = dpaa_sec_queue_pair_count,
2261         .sym_session_get_size     = dpaa_sec_sym_session_get_size,
2262         .sym_session_configure    = dpaa_sec_sym_session_configure,
2263         .sym_session_clear        = dpaa_sec_sym_session_clear
2264 };
2265
2266 static const struct rte_security_capability *
2267 dpaa_sec_capabilities_get(void *device __rte_unused)
2268 {
2269         return dpaa_sec_security_cap;
2270 }
2271
2272 static const struct rte_security_ops dpaa_sec_security_ops = {
2273         .session_create = dpaa_sec_security_session_create,
2274         .session_update = NULL,
2275         .session_stats_get = NULL,
2276         .session_destroy = dpaa_sec_security_session_destroy,
2277         .set_pkt_metadata = NULL,
2278         .capabilities_get = dpaa_sec_capabilities_get
2279 };
2280
2281 static int
2282 dpaa_sec_uninit(struct rte_cryptodev *dev)
2283 {
2284         struct dpaa_sec_dev_private *internals;
2285
2286         if (dev == NULL)
2287                 return -ENODEV;
2288
2289         internals = dev->data->dev_private;
2290         rte_free(dev->security_ctx);
2291
2292         /* In case close has been called, internals->ctx_pool would be NULL */
2293         rte_mempool_free(internals->ctx_pool);
2294         rte_free(internals);
2295
2296         DPAA_SEC_INFO("Closing DPAA_SEC device %s on numa socket %u",
2297                       dev->data->name, rte_socket_id());
2298
2299         return 0;
2300 }
2301
2302 static int
2303 dpaa_sec_dev_init(struct rte_cryptodev *cryptodev)
2304 {
2305         struct dpaa_sec_dev_private *internals;
2306         struct rte_security_ctx *security_instance;
2307         struct dpaa_sec_qp *qp;
2308         uint32_t i, flags;
2309         int ret;
2310
2311         PMD_INIT_FUNC_TRACE();
2312
2313         cryptodev->driver_id = cryptodev_driver_id;
2314         cryptodev->dev_ops = &crypto_ops;
2315
2316         cryptodev->enqueue_burst = dpaa_sec_enqueue_burst;
2317         cryptodev->dequeue_burst = dpaa_sec_dequeue_burst;
2318         cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2319                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
2320                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2321                         RTE_CRYPTODEV_FF_SECURITY |
2322                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
2323                         RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
2324                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
2325                         RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
2326                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
2327
2328         internals = cryptodev->data->dev_private;
2329         internals->max_nb_queue_pairs = RTE_DPAA_MAX_NB_SEC_QPS;
2330         internals->max_nb_sessions = RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS;
2331
2332         /*
2333          * For secondary processes, we don't initialise any further as primary
2334          * has already done this work. Only check we don't need a different
2335          * RX function
2336          */
2337         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2338                 DPAA_SEC_WARN("Device already init by primary process");
2339                 return 0;
2340         }
2341
2342         /* Initialize security_ctx only for primary process*/
2343         security_instance = rte_malloc("rte_security_instances_ops",
2344                                 sizeof(struct rte_security_ctx), 0);
2345         if (security_instance == NULL)
2346                 return -ENOMEM;
2347         security_instance->device = (void *)cryptodev;
2348         security_instance->ops = &dpaa_sec_security_ops;
2349         security_instance->sess_cnt = 0;
2350         cryptodev->security_ctx = security_instance;
2351
2352         rte_spinlock_init(&internals->lock);
2353         for (i = 0; i < internals->max_nb_queue_pairs; i++) {
2354                 /* init qman fq for queue pair */
2355                 qp = &internals->qps[i];
2356                 ret = dpaa_sec_init_tx(&qp->outq);
2357                 if (ret) {
2358                         DPAA_SEC_ERR("config tx of queue pair  %d", i);
2359                         goto init_error;
2360                 }
2361         }
2362
2363         flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID |
2364                 QMAN_FQ_FLAG_TO_DCPORTAL;
2365         for (i = 0; i < internals->max_nb_sessions; i++) {
2366                 /* create rx qman fq for sessions*/
2367                 ret = qman_create_fq(0, flags, &internals->inq[i]);
2368                 if (unlikely(ret != 0)) {
2369                         DPAA_SEC_ERR("sec qman_create_fq failed");
2370                         goto init_error;
2371                 }
2372         }
2373
2374         RTE_LOG(INFO, PMD, "%s cryptodev init\n", cryptodev->data->name);
2375         return 0;
2376
2377 init_error:
2378         DPAA_SEC_ERR("driver %s: create failed\n", cryptodev->data->name);
2379
2380         dpaa_sec_uninit(cryptodev);
2381         return -EFAULT;
2382 }
2383
2384 static int
2385 cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv __rte_unused,
2386                                 struct rte_dpaa_device *dpaa_dev)
2387 {
2388         struct rte_cryptodev *cryptodev;
2389         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
2390
2391         int retval;
2392
2393         sprintf(cryptodev_name, "dpaa_sec-%d", dpaa_dev->id.dev_id);
2394
2395         cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id());
2396         if (cryptodev == NULL)
2397                 return -ENOMEM;
2398
2399         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
2400                 cryptodev->data->dev_private = rte_zmalloc_socket(
2401                                         "cryptodev private structure",
2402                                         sizeof(struct dpaa_sec_dev_private),
2403                                         RTE_CACHE_LINE_SIZE,
2404                                         rte_socket_id());
2405
2406                 if (cryptodev->data->dev_private == NULL)
2407                         rte_panic("Cannot allocate memzone for private "
2408                                         "device data");
2409         }
2410
2411         dpaa_dev->crypto_dev = cryptodev;
2412         cryptodev->device = &dpaa_dev->device;
2413
2414         /* init user callbacks */
2415         TAILQ_INIT(&(cryptodev->link_intr_cbs));
2416
2417         /* if sec device version is not configured */
2418         if (!rta_get_sec_era()) {
2419                 const struct device_node *caam_node;
2420
2421                 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
2422                         const uint32_t *prop = of_get_property(caam_node,
2423                                         "fsl,sec-era",
2424                                         NULL);
2425                         if (prop) {
2426                                 rta_set_sec_era(
2427                                         INTL_SEC_ERA(rte_cpu_to_be_32(*prop)));
2428                                 break;
2429                         }
2430                 }
2431         }
2432
2433         /* Invoke PMD device initialization function */
2434         retval = dpaa_sec_dev_init(cryptodev);
2435         if (retval == 0)
2436                 return 0;
2437
2438         /* In case of error, cleanup is done */
2439         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
2440                 rte_free(cryptodev->data->dev_private);
2441
2442         rte_cryptodev_pmd_release_device(cryptodev);
2443
2444         return -ENXIO;
2445 }
2446
2447 static int
2448 cryptodev_dpaa_sec_remove(struct rte_dpaa_device *dpaa_dev)
2449 {
2450         struct rte_cryptodev *cryptodev;
2451         int ret;
2452
2453         cryptodev = dpaa_dev->crypto_dev;
2454         if (cryptodev == NULL)
2455                 return -ENODEV;
2456
2457         ret = dpaa_sec_uninit(cryptodev);
2458         if (ret)
2459                 return ret;
2460
2461         return rte_cryptodev_pmd_destroy(cryptodev);
2462 }
2463
2464 static struct rte_dpaa_driver rte_dpaa_sec_driver = {
2465         .drv_type = FSL_DPAA_CRYPTO,
2466         .driver = {
2467                 .name = "DPAA SEC PMD"
2468         },
2469         .probe = cryptodev_dpaa_sec_probe,
2470         .remove = cryptodev_dpaa_sec_remove,
2471 };
2472
2473 static struct cryptodev_driver dpaa_sec_crypto_drv;
2474
2475 RTE_PMD_REGISTER_DPAA(CRYPTODEV_NAME_DPAA_SEC_PMD, rte_dpaa_sec_driver);
2476 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa_sec_crypto_drv, rte_dpaa_sec_driver.driver,
2477                 cryptodev_driver_id);
2478
2479 RTE_INIT(dpaa_sec_init_log)
2480 {
2481         dpaa_logtype_sec = rte_log_register("pmd.crypto.dpaa");
2482         if (dpaa_logtype_sec >= 0)
2483                 rte_log_set_level(dpaa_logtype_sec, RTE_LOG_NOTICE);
2484 }