drivers/crypto/dpaa_sec/dpaa_sec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2017-2018 NXP
   5  *
   6  */
   7
   8 #include <fcntl.h>
   9 #include <unistd.h>
  10 #include <sched.h>
  11 #include <net/if.h>
  12
  13 #include <rte_byteorder.h>
  14 #include <rte_common.h>
  15 #include <rte_cryptodev_pmd.h>
  16 #include <rte_crypto.h>
  17 #include <rte_cryptodev.h>
  18 #include <rte_security_driver.h>
  19 #include <rte_cycles.h>
  20 #include <rte_dev.h>
  21 #include <rte_kvargs.h>
  22 #include <rte_malloc.h>
  23 #include <rte_mbuf.h>
  24 #include <rte_memcpy.h>
  25 #include <rte_string_fns.h>
  26
  27 #include <fsl_usd.h>
  28 #include <fsl_qman.h>
  29 #include <of.h>
  30
  31 /* RTA header files */
  32 #include <hw/desc/common.h>
  33 #include <hw/desc/algo.h>
  34 #include <hw/desc/ipsec.h>
  35
  36 #include <rte_dpaa_bus.h>
  37 #include <dpaa_sec.h>
  38 #include <dpaa_sec_log.h>
  39
  40 enum rta_sec_era rta_sec_era;
  41
  42 int dpaa_logtype_sec;
  43
  44 static uint8_t cryptodev_driver_id;
  45
  46 static __thread struct rte_crypto_op **dpaa_sec_ops;
  47 static __thread int dpaa_sec_op_nb;
  48
  49 static int
  50 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess);
  51
  52 static inline void
  53 dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx)
  54 {
  55         if (!ctx->fd_status) {
  56                 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
  57         } else {
  58                 DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status);
  59                 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
  60         }
  61
  62         /* report op status to sym->op and then free the ctx memeory  */
  63         rte_mempool_put(ctx->ctx_pool, (void *)ctx);
  64 }
  65
  66 static inline struct dpaa_sec_op_ctx *
  67 dpaa_sec_alloc_ctx(dpaa_sec_session *ses)
  68 {
  69         struct dpaa_sec_op_ctx *ctx;
  70         int retval;
  71
  72         retval = rte_mempool_get(ses->ctx_pool, (void **)(&ctx));
  73         if (!ctx || retval) {
  74                 DPAA_SEC_DP_WARN("Alloc sec descriptor failed!");
  75                 return NULL;
  76         }
  77         /*
  78          * Clear SG memory. There are 16 SG entries of 16 Bytes each.
  79          * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
  80          * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for
  81          * each packet, memset is costlier than dcbz_64().
  82          */
  83         dcbz_64(&ctx->job.sg[SG_CACHELINE_0]);
  84         dcbz_64(&ctx->job.sg[SG_CACHELINE_1]);
  85         dcbz_64(&ctx->job.sg[SG_CACHELINE_2]);
  86         dcbz_64(&ctx->job.sg[SG_CACHELINE_3]);
  87
  88         ctx->ctx_pool = ses->ctx_pool;
  89         ctx->vtop_offset = (size_t) ctx
  90                                 - rte_mempool_virt2iova(ctx);
  91
  92         return ctx;
  93 }
  94
  95 static inline rte_iova_t
  96 dpaa_mem_vtop(void *vaddr)
  97 {
  98         const struct rte_memseg *ms;
  99
 100         ms = rte_mem_virt2memseg(vaddr, NULL);
 101         if (ms)
 102                 return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr);
 103         return (size_t)NULL;
 104 }
 105
 106 /* virtual address conversin when mempool support is available for ctx */
 107 static inline phys_addr_t
 108 dpaa_mem_vtop_ctx(struct dpaa_sec_op_ctx *ctx, void *vaddr)
 109 {
 110         return (size_t)vaddr - ctx->vtop_offset;
 111 }
 112
 113 static inline void *
 114 dpaa_mem_ptov(rte_iova_t paddr)
 115 {
 116         return rte_mem_iova2virt(paddr);
 117 }
 118
 119 static void
 120 ern_sec_fq_handler(struct qman_portal *qm __rte_unused,
 121                    struct qman_fq *fq,
 122                    const struct qm_mr_entry *msg)
 123 {
 124         DPAA_SEC_DP_ERR("sec fq %d error, RC = %x, seqnum = %x\n",
 125                         fq->fqid, msg->ern.rc, msg->ern.seqnum);
 126 }
 127
 128 /* initialize the queue with dest chan as caam chan so that
 129  * all the packets in this queue could be dispatched into caam
 130  */
 131 static int
 132 dpaa_sec_init_rx(struct qman_fq *fq_in, rte_iova_t hwdesc,
 133                  uint32_t fqid_out)
 134 {
 135         struct qm_mcc_initfq fq_opts;
 136         uint32_t flags;
 137         int ret = -1;
 138
 139         /* Clear FQ options */
 140         memset(&fq_opts, 0x00, sizeof(struct qm_mcc_initfq));
 141
 142         flags = QMAN_INITFQ_FLAG_SCHED;
 143         fq_opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_CONTEXTA |
 144                           QM_INITFQ_WE_CONTEXTB;
 145
 146         qm_fqd_context_a_set64(&fq_opts.fqd, hwdesc);
 147         fq_opts.fqd.context_b = fqid_out;
 148         fq_opts.fqd.dest.channel = qm_channel_caam;
 149         fq_opts.fqd.dest.wq = 0;
 150
 151         fq_in->cb.ern  = ern_sec_fq_handler;
 152
 153         DPAA_SEC_DEBUG("in-%x out-%x", fq_in->fqid, fqid_out);
 154
 155         ret = qman_init_fq(fq_in, flags, &fq_opts);
 156         if (unlikely(ret != 0))
 157                 DPAA_SEC_ERR("qman_init_fq failed %d", ret);
 158
 159         return ret;
 160 }
 161
 162 /* something is put into in_fq and caam put the crypto result into out_fq */
 163 static enum qman_cb_dqrr_result
 164 dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused,
 165                   struct qman_fq *fq __always_unused,
 166                   const struct qm_dqrr_entry *dqrr)
 167 {
 168         const struct qm_fd *fd;
 169         struct dpaa_sec_job *job;
 170         struct dpaa_sec_op_ctx *ctx;
 171
 172         if (dpaa_sec_op_nb >= DPAA_SEC_BURST)
 173                 return qman_cb_dqrr_defer;
 174
 175         if (!(dqrr->stat & QM_DQRR_STAT_FD_VALID))
 176                 return qman_cb_dqrr_consume;
 177
 178         fd = &dqrr->fd;
 179         /* sg is embedded in an op ctx,
 180          * sg[0] is for output
 181          * sg[1] for input
 182          */
 183         job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
 184
 185         ctx = container_of(job, struct dpaa_sec_op_ctx, job);
 186         ctx->fd_status = fd->status;
 187         if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 188                 struct qm_sg_entry *sg_out;
 189                 uint32_t len;
 190
 191                 sg_out = &job->sg[0];
 192                 hw_sg_to_cpu(sg_out);
 193                 len = sg_out->length;
 194                 ctx->op->sym->m_src->pkt_len = len;
 195                 ctx->op->sym->m_src->data_len = len;
 196         }
 197         dpaa_sec_ops[dpaa_sec_op_nb++] = ctx->op;
 198         dpaa_sec_op_ending(ctx);
 199
 200         return qman_cb_dqrr_consume;
 201 }
 202
 203 /* caam result is put into this queue */
 204 static int
 205 dpaa_sec_init_tx(struct qman_fq *fq)
 206 {
 207         int ret;
 208         struct qm_mcc_initfq opts;
 209         uint32_t flags;
 210
 211         flags = QMAN_FQ_FLAG_NO_ENQUEUE | QMAN_FQ_FLAG_LOCKED |
 212                 QMAN_FQ_FLAG_DYNAMIC_FQID;
 213
 214         ret = qman_create_fq(0, flags, fq);
 215         if (unlikely(ret)) {
 216                 DPAA_SEC_ERR("qman_create_fq failed");
 217                 return ret;
 218         }
 219
 220         memset(&opts, 0, sizeof(opts));
 221         opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
 222                        QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB;
 223
 224         /* opts.fqd.dest.channel = dpaa_sec_pool_chan; */
 225
 226         fq->cb.dqrr = dqrr_out_fq_cb_rx;
 227         fq->cb.ern  = ern_sec_fq_handler;
 228
 229         ret = qman_init_fq(fq, 0, &opts);
 230         if (unlikely(ret)) {
 231                 DPAA_SEC_ERR("unable to init caam source fq!");
 232                 return ret;
 233         }
 234
 235         return ret;
 236 }
 237
 238 static inline int is_cipher_only(dpaa_sec_session *ses)
 239 {
 240         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 241                 (ses->auth_alg == RTE_CRYPTO_AUTH_NULL));
 242 }
 243
 244 static inline int is_auth_only(dpaa_sec_session *ses)
 245 {
 246         return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) &&
 247                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL));
 248 }
 249
 250 static inline int is_aead(dpaa_sec_session *ses)
 251 {
 252         return ((ses->cipher_alg == 0) &&
 253                 (ses->auth_alg == 0) &&
 254                 (ses->aead_alg != 0));
 255 }
 256
 257 static inline int is_auth_cipher(dpaa_sec_session *ses)
 258 {
 259         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 260                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) &&
 261                 (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC));
 262 }
 263
 264 static inline int is_proto_ipsec(dpaa_sec_session *ses)
 265 {
 266         return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC);
 267 }
 268
 269 static inline int is_encode(dpaa_sec_session *ses)
 270 {
 271         return ses->dir == DIR_ENC;
 272 }
 273
 274 static inline int is_decode(dpaa_sec_session *ses)
 275 {
 276         return ses->dir == DIR_DEC;
 277 }
 278
 279 static inline void
 280 caam_auth_alg(dpaa_sec_session *ses, struct alginfo *alginfo_a)
 281 {
 282         switch (ses->auth_alg) {
 283         case RTE_CRYPTO_AUTH_NULL:
 284                 ses->digest_length = 0;
 285                 break;
 286         case RTE_CRYPTO_AUTH_MD5_HMAC:
 287                 alginfo_a->algtype =
 288                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 289                         OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5;
 290                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 291                 break;
 292         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 293                 alginfo_a->algtype =
 294                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 295                         OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1;
 296                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 297                 break;
 298         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 299                 alginfo_a->algtype =
 300                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 301                         OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224;
 302                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 303                 break;
 304         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 305                 alginfo_a->algtype =
 306                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 307                         OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256;
 308                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 309                 break;
 310         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 311                 alginfo_a->algtype =
 312                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 313                         OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384;
 314                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 315                 break;
 316         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 317                 alginfo_a->algtype =
 318                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 319                         OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512;
 320                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 321                 break;
 322         default:
 323                 DPAA_SEC_ERR("unsupported auth alg %u", ses->auth_alg);
 324         }
 325 }
 326
 327 static inline void
 328 caam_cipher_alg(dpaa_sec_session *ses, struct alginfo *alginfo_c)
 329 {
 330         switch (ses->cipher_alg) {
 331         case RTE_CRYPTO_CIPHER_NULL:
 332                 break;
 333         case RTE_CRYPTO_CIPHER_AES_CBC:
 334                 alginfo_c->algtype =
 335                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 336                         OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES;
 337                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 338                 break;
 339         case RTE_CRYPTO_CIPHER_3DES_CBC:
 340                 alginfo_c->algtype =
 341                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 342                         OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES;
 343                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 344                 break;
 345         case RTE_CRYPTO_CIPHER_AES_CTR:
 346                 alginfo_c->algtype =
 347                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 348                         OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES;
 349                 alginfo_c->algmode = OP_ALG_AAI_CTR;
 350                 break;
 351         default:
 352                 DPAA_SEC_ERR("unsupported cipher alg %d", ses->cipher_alg);
 353         }
 354 }
 355
 356 static inline void
 357 caam_aead_alg(dpaa_sec_session *ses, struct alginfo *alginfo)
 358 {
 359         switch (ses->aead_alg) {
 360         case RTE_CRYPTO_AEAD_AES_GCM:
 361                 alginfo->algtype = OP_ALG_ALGSEL_AES;
 362                 alginfo->algmode = OP_ALG_AAI_GCM;
 363                 break;
 364         default:
 365                 DPAA_SEC_ERR("unsupported AEAD alg %d", ses->aead_alg);
 366         }
 367 }
 368
 369
 370 /* prepare command block of the session */
 371 static int
 372 dpaa_sec_prep_cdb(dpaa_sec_session *ses)
 373 {
 374         struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0};
 375         int32_t shared_desc_len = 0;
 376         struct sec_cdb *cdb = &ses->cdb;
 377         int err;
 378 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 379         int swap = false;
 380 #else
 381         int swap = true;
 382 #endif
 383
 384         memset(cdb, 0, sizeof(struct sec_cdb));
 385
 386         if (is_cipher_only(ses)) {
 387                 caam_cipher_alg(ses, &alginfo_c);
 388                 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 389                         DPAA_SEC_ERR("not supported cipher alg");
 390                         return -ENOTSUP;
 391                 }
 392
 393                 alginfo_c.key = (size_t)ses->cipher_key.data;
 394                 alginfo_c.keylen = ses->cipher_key.length;
 395                 alginfo_c.key_enc_flags = 0;
 396                 alginfo_c.key_type = RTA_DATA_IMM;
 397
 398                 shared_desc_len = cnstr_shdsc_blkcipher(
 399                                                 cdb->sh_desc, true,
 400                                                 swap, &alginfo_c,
 401                                                 NULL,
 402                                                 ses->iv.length,
 403                                                 ses->dir);
 404         } else if (is_auth_only(ses)) {
 405                 caam_auth_alg(ses, &alginfo_a);
 406                 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 407                         DPAA_SEC_ERR("not supported auth alg");
 408                         return -ENOTSUP;
 409                 }
 410
 411                 alginfo_a.key = (size_t)ses->auth_key.data;
 412                 alginfo_a.keylen = ses->auth_key.length;
 413                 alginfo_a.key_enc_flags = 0;
 414                 alginfo_a.key_type = RTA_DATA_IMM;
 415
 416                 shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true,
 417                                                    swap, &alginfo_a,
 418                                                    !ses->dir,
 419                                                    ses->digest_length);
 420         } else if (is_aead(ses)) {
 421                 caam_aead_alg(ses, &alginfo);
 422                 if (alginfo.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 423                         DPAA_SEC_ERR("not supported aead alg");
 424                         return -ENOTSUP;
 425                 }
 426                 alginfo.key = (size_t)ses->aead_key.data;
 427                 alginfo.keylen = ses->aead_key.length;
 428                 alginfo.key_enc_flags = 0;
 429                 alginfo.key_type = RTA_DATA_IMM;
 430
 431                 if (ses->dir == DIR_ENC)
 432                         shared_desc_len = cnstr_shdsc_gcm_encap(
 433                                         cdb->sh_desc, true, swap,
 434                                         &alginfo,
 435                                         ses->iv.length,
 436                                         ses->digest_length);
 437                 else
 438                         shared_desc_len = cnstr_shdsc_gcm_decap(
 439                                         cdb->sh_desc, true, swap,
 440                                         &alginfo,
 441                                         ses->iv.length,
 442                                         ses->digest_length);
 443         } else {
 444                 caam_cipher_alg(ses, &alginfo_c);
 445                 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 446                         DPAA_SEC_ERR("not supported cipher alg");
 447                         return -ENOTSUP;
 448                 }
 449
 450                 alginfo_c.key = (size_t)ses->cipher_key.data;
 451                 alginfo_c.keylen = ses->cipher_key.length;
 452                 alginfo_c.key_enc_flags = 0;
 453                 alginfo_c.key_type = RTA_DATA_IMM;
 454
 455                 caam_auth_alg(ses, &alginfo_a);
 456                 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 457                         DPAA_SEC_ERR("not supported auth alg");
 458                         return -ENOTSUP;
 459                 }
 460
 461                 alginfo_a.key = (size_t)ses->auth_key.data;
 462                 alginfo_a.keylen = ses->auth_key.length;
 463                 alginfo_a.key_enc_flags = 0;
 464                 alginfo_a.key_type = RTA_DATA_IMM;
 465
 466                 cdb->sh_desc[0] = alginfo_c.keylen;
 467                 cdb->sh_desc[1] = alginfo_a.keylen;
 468                 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
 469                                        MIN_JOB_DESC_SIZE,
 470                                        (unsigned int *)cdb->sh_desc,
 471                                        &cdb->sh_desc[2], 2);
 472
 473                 if (err < 0) {
 474                         DPAA_SEC_ERR("Crypto: Incorrect key lengths");
 475                         return err;
 476                 }
 477                 if (cdb->sh_desc[2] & 1)
 478                         alginfo_c.key_type = RTA_DATA_IMM;
 479                 else {
 480                         alginfo_c.key = (size_t)dpaa_mem_vtop(
 481                                                 (void *)(size_t)alginfo_c.key);
 482                         alginfo_c.key_type = RTA_DATA_PTR;
 483                 }
 484                 if (cdb->sh_desc[2] & (1<<1))
 485                         alginfo_a.key_type = RTA_DATA_IMM;
 486                 else {
 487                         alginfo_a.key = (size_t)dpaa_mem_vtop(
 488                                                 (void *)(size_t)alginfo_a.key);
 489                         alginfo_a.key_type = RTA_DATA_PTR;
 490                 }
 491                 cdb->sh_desc[0] = 0;
 492                 cdb->sh_desc[1] = 0;
 493                 cdb->sh_desc[2] = 0;
 494                 if (is_proto_ipsec(ses)) {
 495                         if (ses->dir == DIR_ENC) {
 496                                 shared_desc_len = cnstr_shdsc_ipsec_new_encap(
 497                                                 cdb->sh_desc,
 498                                                 true, swap, &ses->encap_pdb,
 499                                                 (uint8_t *)&ses->ip4_hdr,
 500                                                 &alginfo_c, &alginfo_a);
 501                         } else if (ses->dir == DIR_DEC) {
 502                                 shared_desc_len = cnstr_shdsc_ipsec_new_decap(
 503                                                 cdb->sh_desc,
 504                                                 true, swap, &ses->decap_pdb,
 505                                                 &alginfo_c, &alginfo_a);
 506                         }
 507                 } else {
 508                         /* Auth_only_len is set as 0 here and it will be
 509                          * overwritten in fd for each packet.
 510                          */
 511                         shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
 512                                         true, swap, &alginfo_c, &alginfo_a,
 513                                         ses->iv.length, 0,
 514                                         ses->digest_length, ses->dir);
 515                 }
 516         }
 517
 518         if (shared_desc_len < 0) {
 519                 DPAA_SEC_ERR("error in preparing command block");
 520                 return shared_desc_len;
 521         }
 522
 523         cdb->sh_hdr.hi.field.idlen = shared_desc_len;
 524         cdb->sh_hdr.hi.word = rte_cpu_to_be_32(cdb->sh_hdr.hi.word);
 525         cdb->sh_hdr.lo.word = rte_cpu_to_be_32(cdb->sh_hdr.lo.word);
 526
 527         return 0;
 528 }
 529
 530 /* qp is lockless, should be accessed by only one thread */
 531 static int
 532 dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops)
 533 {
 534         struct qman_fq *fq;
 535         unsigned int pkts = 0;
 536         int ret;
 537         struct qm_dqrr_entry *dq;
 538
 539         fq = &qp->outq;
 540         ret = qman_set_vdq(fq, (nb_ops > DPAA_MAX_DEQUEUE_NUM_FRAMES) ?
 541                                 DPAA_MAX_DEQUEUE_NUM_FRAMES : nb_ops);
 542         if (ret)
 543                 return 0;
 544
 545         do {
 546                 const struct qm_fd *fd;
 547                 struct dpaa_sec_job *job;
 548                 struct dpaa_sec_op_ctx *ctx;
 549                 struct rte_crypto_op *op;
 550
 551                 dq = qman_dequeue(fq);
 552                 if (!dq)
 553                         continue;
 554
 555                 fd = &dq->fd;
 556                 /* sg is embedded in an op ctx,
 557                  * sg[0] is for output
 558                  * sg[1] for input
 559                  */
 560                 job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
 561
 562                 ctx = container_of(job, struct dpaa_sec_op_ctx, job);
 563                 ctx->fd_status = fd->status;
 564                 op = ctx->op;
 565                 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 566                         struct qm_sg_entry *sg_out;
 567                         uint32_t len;
 568
 569                         sg_out = &job->sg[0];
 570                         hw_sg_to_cpu(sg_out);
 571                         len = sg_out->length;
 572                         op->sym->m_src->pkt_len = len;
 573                         op->sym->m_src->data_len = len;
 574                 }
 575                 if (!ctx->fd_status) {
 576                         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 577                 } else {
 578                         DPAA_SEC_DP_WARN("SEC return err:0x%x", ctx->fd_status);
 579                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 580                 }
 581                 ops[pkts++] = op;
 582
 583                 /* report op status to sym->op and then free the ctx memeory */
 584                 rte_mempool_put(ctx->ctx_pool, (void *)ctx);
 585
 586                 qman_dqrr_consume(fq, dq);
 587         } while (fq->flags & QMAN_FQ_STATE_VDQCR);
 588
 589         return pkts;
 590 }
 591
 592 static inline struct dpaa_sec_job *
 593 build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 594 {
 595         struct rte_crypto_sym_op *sym = op->sym;
 596         struct rte_mbuf *mbuf = sym->m_src;
 597         struct dpaa_sec_job *cf;
 598         struct dpaa_sec_op_ctx *ctx;
 599         struct qm_sg_entry *sg, *out_sg, *in_sg;
 600         phys_addr_t start_addr;
 601         uint8_t *old_digest, extra_segs;
 602
 603         if (is_decode(ses))
 604                 extra_segs = 3;
 605         else
 606                 extra_segs = 2;
 607
 608         if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) {
 609                 DPAA_SEC_DP_ERR("Auth: Max sec segs supported is %d",
 610                                 MAX_SG_ENTRIES);
 611                 return NULL;
 612         }
 613         ctx = dpaa_sec_alloc_ctx(ses);
 614         if (!ctx)
 615                 return NULL;
 616
 617         cf = &ctx->job;
 618         ctx->op = op;
 619         old_digest = ctx->digest;
 620
 621         /* output */
 622         out_sg = &cf->sg[0];
 623         qm_sg_entry_set64(out_sg, sym->auth.digest.phys_addr);
 624         out_sg->length = ses->digest_length;
 625         cpu_to_hw_sg(out_sg);
 626
 627         /* input */
 628         in_sg = &cf->sg[1];
 629         /* need to extend the input to a compound frame */
 630         in_sg->extension = 1;
 631         in_sg->final = 1;
 632         in_sg->length = sym->auth.data.length;
 633         qm_sg_entry_set64(in_sg, dpaa_mem_vtop_ctx(ctx, &cf->sg[2]));
 634
 635         /* 1st seg */
 636         sg = in_sg + 1;
 637         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 638         sg->length = mbuf->data_len - sym->auth.data.offset;
 639         sg->offset = sym->auth.data.offset;
 640
 641         /* Successive segs */
 642         mbuf = mbuf->next;
 643         while (mbuf) {
 644                 cpu_to_hw_sg(sg);
 645                 sg++;
 646                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 647                 sg->length = mbuf->data_len;
 648                 mbuf = mbuf->next;
 649         }
 650
 651         if (is_decode(ses)) {
 652                 /* Digest verification case */
 653                 cpu_to_hw_sg(sg);
 654                 sg++;
 655                 rte_memcpy(old_digest, sym->auth.digest.data,
 656                                 ses->digest_length);
 657                 start_addr = dpaa_mem_vtop_ctx(ctx, old_digest);
 658                 qm_sg_entry_set64(sg, start_addr);
 659                 sg->length = ses->digest_length;
 660                 in_sg->length += ses->digest_length;
 661         } else {
 662                 /* Digest calculation case */
 663                 sg->length -= ses->digest_length;
 664         }
 665         sg->final = 1;
 666         cpu_to_hw_sg(sg);
 667         cpu_to_hw_sg(in_sg);
 668
 669         return cf;
 670 }
 671
 672 /**
 673  * packet looks like:
 674  *              |<----data_len------->|
 675  *    |ip_header|ah_header|icv|payload|
 676  *              ^
 677  *              |
 678  *         mbuf->pkt.data
 679  */
 680 static inline struct dpaa_sec_job *
 681 build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
 682 {
 683         struct rte_crypto_sym_op *sym = op->sym;
 684         struct rte_mbuf *mbuf = sym->m_src;
 685         struct dpaa_sec_job *cf;
 686         struct dpaa_sec_op_ctx *ctx;
 687         struct qm_sg_entry *sg;
 688         rte_iova_t start_addr;
 689         uint8_t *old_digest;
 690
 691         ctx = dpaa_sec_alloc_ctx(ses);
 692         if (!ctx)
 693                 return NULL;
 694
 695         cf = &ctx->job;
 696         ctx->op = op;
 697         old_digest = ctx->digest;
 698
 699         start_addr = rte_pktmbuf_iova(mbuf);
 700         /* output */
 701         sg = &cf->sg[0];
 702         qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
 703         sg->length = ses->digest_length;
 704         cpu_to_hw_sg(sg);
 705
 706         /* input */
 707         sg = &cf->sg[1];
 708         if (is_decode(ses)) {
 709                 /* need to extend the input to a compound frame */
 710                 sg->extension = 1;
 711                 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, &cf->sg[2]));
 712                 sg->length = sym->auth.data.length + ses->digest_length;
 713                 sg->final = 1;
 714                 cpu_to_hw_sg(sg);
 715
 716                 sg = &cf->sg[2];
 717                 /* hash result or digest, save digest first */
 718                 rte_memcpy(old_digest, sym->auth.digest.data,
 719                            ses->digest_length);
 720                 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset);
 721                 sg->length = sym->auth.data.length;
 722                 cpu_to_hw_sg(sg);
 723
 724                 /* let's check digest by hw */
 725                 start_addr = dpaa_mem_vtop_ctx(ctx, old_digest);
 726                 sg++;
 727                 qm_sg_entry_set64(sg, start_addr);
 728                 sg->length = ses->digest_length;
 729                 sg->final = 1;
 730                 cpu_to_hw_sg(sg);
 731         } else {
 732                 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset);
 733                 sg->length = sym->auth.data.length;
 734                 sg->final = 1;
 735                 cpu_to_hw_sg(sg);
 736         }
 737
 738         return cf;
 739 }
 740
 741 static inline struct dpaa_sec_job *
 742 build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 743 {
 744         struct rte_crypto_sym_op *sym = op->sym;
 745         struct dpaa_sec_job *cf;
 746         struct dpaa_sec_op_ctx *ctx;
 747         struct qm_sg_entry *sg, *out_sg, *in_sg;
 748         struct rte_mbuf *mbuf;
 749         uint8_t req_segs;
 750         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 751                         ses->iv.offset);
 752
 753         if (sym->m_dst) {
 754                 mbuf = sym->m_dst;
 755                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3;
 756         } else {
 757                 mbuf = sym->m_src;
 758                 req_segs = mbuf->nb_segs * 2 + 3;
 759         }
 760
 761         if (req_segs > MAX_SG_ENTRIES) {
 762                 DPAA_SEC_DP_ERR("Cipher: Max sec segs supported is %d",
 763                                 MAX_SG_ENTRIES);
 764                 return NULL;
 765         }
 766
 767         ctx = dpaa_sec_alloc_ctx(ses);
 768         if (!ctx)
 769                 return NULL;
 770
 771         cf = &ctx->job;
 772         ctx->op = op;
 773
 774         /* output */
 775         out_sg = &cf->sg[0];
 776         out_sg->extension = 1;
 777         out_sg->length = sym->cipher.data.length;
 778         qm_sg_entry_set64(out_sg, dpaa_mem_vtop_ctx(ctx, &cf->sg[2]));
 779         cpu_to_hw_sg(out_sg);
 780
 781         /* 1st seg */
 782         sg = &cf->sg[2];
 783         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 784         sg->length = mbuf->data_len - sym->cipher.data.offset;
 785         sg->offset = sym->cipher.data.offset;
 786
 787         /* Successive segs */
 788         mbuf = mbuf->next;
 789         while (mbuf) {
 790                 cpu_to_hw_sg(sg);
 791                 sg++;
 792                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 793                 sg->length = mbuf->data_len;
 794                 mbuf = mbuf->next;
 795         }
 796         sg->final = 1;
 797         cpu_to_hw_sg(sg);
 798
 799         /* input */
 800         mbuf = sym->m_src;
 801         in_sg = &cf->sg[1];
 802         in_sg->extension = 1;
 803         in_sg->final = 1;
 804         in_sg->length = sym->cipher.data.length + ses->iv.length;
 805
 806         sg++;
 807         qm_sg_entry_set64(in_sg, dpaa_mem_vtop_ctx(ctx, sg));
 808         cpu_to_hw_sg(in_sg);
 809
 810         /* IV */
 811         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
 812         sg->length = ses->iv.length;
 813         cpu_to_hw_sg(sg);
 814
 815         /* 1st seg */
 816         sg++;
 817         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 818         sg->length = mbuf->data_len - sym->cipher.data.offset;
 819         sg->offset = sym->cipher.data.offset;
 820
 821         /* Successive segs */
 822         mbuf = mbuf->next;
 823         while (mbuf) {
 824                 cpu_to_hw_sg(sg);
 825                 sg++;
 826                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 827                 sg->length = mbuf->data_len;
 828                 mbuf = mbuf->next;
 829         }
 830         sg->final = 1;
 831         cpu_to_hw_sg(sg);
 832
 833         return cf;
 834 }
 835
 836 static inline struct dpaa_sec_job *
 837 build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
 838 {
 839         struct rte_crypto_sym_op *sym = op->sym;
 840         struct dpaa_sec_job *cf;
 841         struct dpaa_sec_op_ctx *ctx;
 842         struct qm_sg_entry *sg;
 843         rte_iova_t src_start_addr, dst_start_addr;
 844         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 845                         ses->iv.offset);
 846
 847         ctx = dpaa_sec_alloc_ctx(ses);
 848         if (!ctx)
 849                 return NULL;
 850
 851         cf = &ctx->job;
 852         ctx->op = op;
 853
 854         src_start_addr = rte_pktmbuf_iova(sym->m_src);
 855
 856         if (sym->m_dst)
 857                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
 858         else
 859                 dst_start_addr = src_start_addr;
 860
 861         /* output */
 862         sg = &cf->sg[0];
 863         qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset);
 864         sg->length = sym->cipher.data.length + ses->iv.length;
 865         cpu_to_hw_sg(sg);
 866
 867         /* input */
 868         sg = &cf->sg[1];
 869
 870         /* need to extend the input to a compound frame */
 871         sg->extension = 1;
 872         sg->final = 1;
 873         sg->length = sym->cipher.data.length + ses->iv.length;
 874         qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, &cf->sg[2]));
 875         cpu_to_hw_sg(sg);
 876
 877         sg = &cf->sg[2];
 878         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
 879         sg->length = ses->iv.length;
 880         cpu_to_hw_sg(sg);
 881
 882         sg++;
 883         qm_sg_entry_set64(sg, src_start_addr + sym->cipher.data.offset);
 884         sg->length = sym->cipher.data.length;
 885         sg->final = 1;
 886         cpu_to_hw_sg(sg);
 887
 888         return cf;
 889 }
 890
 891 static inline struct dpaa_sec_job *
 892 build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 893 {
 894         struct rte_crypto_sym_op *sym = op->sym;
 895         struct dpaa_sec_job *cf;
 896         struct dpaa_sec_op_ctx *ctx;
 897         struct qm_sg_entry *sg, *out_sg, *in_sg;
 898         struct rte_mbuf *mbuf;
 899         uint8_t req_segs;
 900         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 901                         ses->iv.offset);
 902
 903         if (sym->m_dst) {
 904                 mbuf = sym->m_dst;
 905                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4;
 906         } else {
 907                 mbuf = sym->m_src;
 908                 req_segs = mbuf->nb_segs * 2 + 4;
 909         }
 910
 911         if (ses->auth_only_len)
 912                 req_segs++;
 913
 914         if (req_segs > MAX_SG_ENTRIES) {
 915                 DPAA_SEC_DP_ERR("AEAD: Max sec segs supported is %d",
 916                                 MAX_SG_ENTRIES);
 917                 return NULL;
 918         }
 919
 920         ctx = dpaa_sec_alloc_ctx(ses);
 921         if (!ctx)
 922                 return NULL;
 923
 924         cf = &ctx->job;
 925         ctx->op = op;
 926
 927         rte_prefetch0(cf->sg);
 928
 929         /* output */
 930         out_sg = &cf->sg[0];
 931         out_sg->extension = 1;
 932         if (is_encode(ses))
 933                 out_sg->length = sym->aead.data.length + ses->auth_only_len
 934                                                 + ses->digest_length;
 935         else
 936                 out_sg->length = sym->aead.data.length + ses->auth_only_len;
 937
 938         /* output sg entries */
 939         sg = &cf->sg[2];
 940         qm_sg_entry_set64(out_sg, dpaa_mem_vtop_ctx(ctx, sg));
 941         cpu_to_hw_sg(out_sg);
 942
 943         /* 1st seg */
 944         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 945         sg->length = mbuf->data_len - sym->aead.data.offset +
 946                                         ses->auth_only_len;
 947         sg->offset = sym->aead.data.offset - ses->auth_only_len;
 948
 949         /* Successive segs */
 950         mbuf = mbuf->next;
 951         while (mbuf) {
 952                 cpu_to_hw_sg(sg);
 953                 sg++;
 954                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 955                 sg->length = mbuf->data_len;
 956                 mbuf = mbuf->next;
 957         }
 958         sg->length -= ses->digest_length;
 959
 960         if (is_encode(ses)) {
 961                 cpu_to_hw_sg(sg);
 962                 /* set auth output */
 963                 sg++;
 964                 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr);
 965                 sg->length = ses->digest_length;
 966         }
 967         sg->final = 1;
 968         cpu_to_hw_sg(sg);
 969
 970         /* input */
 971         mbuf = sym->m_src;
 972         in_sg = &cf->sg[1];
 973         in_sg->extension = 1;
 974         in_sg->final = 1;
 975         if (is_encode(ses))
 976                 in_sg->length = ses->iv.length + sym->aead.data.length
 977                                                         + ses->auth_only_len;
 978         else
 979                 in_sg->length = ses->iv.length + sym->aead.data.length
 980                                 + ses->auth_only_len + ses->digest_length;
 981
 982         /* input sg entries */
 983         sg++;
 984         qm_sg_entry_set64(in_sg, dpaa_mem_vtop_ctx(ctx, sg));
 985         cpu_to_hw_sg(in_sg);
 986
 987         /* 1st seg IV */
 988         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
 989         sg->length = ses->iv.length;
 990         cpu_to_hw_sg(sg);
 991
 992         /* 2nd seg auth only */
 993         if (ses->auth_only_len) {
 994                 sg++;
 995                 qm_sg_entry_set64(sg, dpaa_mem_vtop(sym->aead.aad.data));
 996                 sg->length = ses->auth_only_len;
 997                 cpu_to_hw_sg(sg);
 998         }
 999
1000         /* 3rd seg */
1001         sg++;
1002         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1003         sg->length = mbuf->data_len - sym->aead.data.offset;
1004         sg->offset = sym->aead.data.offset;
1005
1006         /* Successive segs */
1007         mbuf = mbuf->next;
1008         while (mbuf) {
1009                 cpu_to_hw_sg(sg);
1010                 sg++;
1011                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1012                 sg->length = mbuf->data_len;
1013                 mbuf = mbuf->next;
1014         }
1015
1016         if (is_decode(ses)) {
1017                 cpu_to_hw_sg(sg);
1018                 sg++;
1019                 memcpy(ctx->digest, sym->aead.digest.data,
1020                         ses->digest_length);
1021                 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, ctx->digest));
1022                 sg->length = ses->digest_length;
1023         }
1024         sg->final = 1;
1025         cpu_to_hw_sg(sg);
1026
1027         return cf;
1028 }
1029
1030 static inline struct dpaa_sec_job *
1031 build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses)
1032 {
1033         struct rte_crypto_sym_op *sym = op->sym;
1034         struct dpaa_sec_job *cf;
1035         struct dpaa_sec_op_ctx *ctx;
1036         struct qm_sg_entry *sg;
1037         uint32_t length = 0;
1038         rte_iova_t src_start_addr, dst_start_addr;
1039         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1040                         ses->iv.offset);
1041
1042         src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
1043
1044         if (sym->m_dst)
1045                 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
1046         else
1047                 dst_start_addr = src_start_addr;
1048
1049         ctx = dpaa_sec_alloc_ctx(ses);
1050         if (!ctx)
1051                 return NULL;
1052
1053         cf = &ctx->job;
1054         ctx->op = op;
1055
1056         /* input */
1057         rte_prefetch0(cf->sg);
1058         sg = &cf->sg[2];
1059         qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop_ctx(ctx, sg));
1060         if (is_encode(ses)) {
1061                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1062                 sg->length = ses->iv.length;
1063                 length += sg->length;
1064                 cpu_to_hw_sg(sg);
1065
1066                 sg++;
1067                 if (ses->auth_only_len) {
1068                         qm_sg_entry_set64(sg,
1069                                           dpaa_mem_vtop(sym->aead.aad.data));
1070                         sg->length = ses->auth_only_len;
1071                         length += sg->length;
1072                         cpu_to_hw_sg(sg);
1073                         sg++;
1074                 }
1075                 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
1076                 sg->length = sym->aead.data.length;
1077                 length += sg->length;
1078                 sg->final = 1;
1079                 cpu_to_hw_sg(sg);
1080         } else {
1081                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1082                 sg->length = ses->iv.length;
1083                 length += sg->length;
1084                 cpu_to_hw_sg(sg);
1085
1086                 sg++;
1087                 if (ses->auth_only_len) {
1088                         qm_sg_entry_set64(sg,
1089                                           dpaa_mem_vtop(sym->aead.aad.data));
1090                         sg->length = ses->auth_only_len;
1091                         length += sg->length;
1092                         cpu_to_hw_sg(sg);
1093                         sg++;
1094                 }
1095                 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
1096                 sg->length = sym->aead.data.length;
1097                 length += sg->length;
1098                 cpu_to_hw_sg(sg);
1099
1100                 memcpy(ctx->digest, sym->aead.digest.data,
1101                        ses->digest_length);
1102                 sg++;
1103
1104                 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, ctx->digest));
1105                 sg->length = ses->digest_length;
1106                 length += sg->length;
1107                 sg->final = 1;
1108                 cpu_to_hw_sg(sg);
1109         }
1110         /* input compound frame */
1111         cf->sg[1].length = length;
1112         cf->sg[1].extension = 1;
1113         cf->sg[1].final = 1;
1114         cpu_to_hw_sg(&cf->sg[1]);
1115
1116         /* output */
1117         sg++;
1118         qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop_ctx(ctx, sg));
1119         qm_sg_entry_set64(sg,
1120                 dst_start_addr + sym->aead.data.offset - ses->auth_only_len);
1121         sg->length = sym->aead.data.length + ses->auth_only_len;
1122         length = sg->length;
1123         if (is_encode(ses)) {
1124                 cpu_to_hw_sg(sg);
1125                 /* set auth output */
1126                 sg++;
1127                 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr);
1128                 sg->length = ses->digest_length;
1129                 length += sg->length;
1130         }
1131         sg->final = 1;
1132         cpu_to_hw_sg(sg);
1133
1134         /* output compound frame */
1135         cf->sg[0].length = length;
1136         cf->sg[0].extension = 1;
1137         cpu_to_hw_sg(&cf->sg[0]);
1138
1139         return cf;
1140 }
1141
1142 static inline struct dpaa_sec_job *
1143 build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
1144 {
1145         struct rte_crypto_sym_op *sym = op->sym;
1146         struct dpaa_sec_job *cf;
1147         struct dpaa_sec_op_ctx *ctx;
1148         struct qm_sg_entry *sg, *out_sg, *in_sg;
1149         struct rte_mbuf *mbuf;
1150         uint8_t req_segs;
1151         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1152                         ses->iv.offset);
1153
1154         if (sym->m_dst) {
1155                 mbuf = sym->m_dst;
1156                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4;
1157         } else {
1158                 mbuf = sym->m_src;
1159                 req_segs = mbuf->nb_segs * 2 + 4;
1160         }
1161
1162         if (req_segs > MAX_SG_ENTRIES) {
1163                 DPAA_SEC_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
1164                                 MAX_SG_ENTRIES);
1165                 return NULL;
1166         }
1167
1168         ctx = dpaa_sec_alloc_ctx(ses);
1169         if (!ctx)
1170                 return NULL;
1171
1172         cf = &ctx->job;
1173         ctx->op = op;
1174
1175         rte_prefetch0(cf->sg);
1176
1177         /* output */
1178         out_sg = &cf->sg[0];
1179         out_sg->extension = 1;
1180         if (is_encode(ses))
1181                 out_sg->length = sym->auth.data.length + ses->digest_length;
1182         else
1183                 out_sg->length = sym->auth.data.length;
1184
1185         /* output sg entries */
1186         sg = &cf->sg[2];
1187         qm_sg_entry_set64(out_sg, dpaa_mem_vtop_ctx(ctx, sg));
1188         cpu_to_hw_sg(out_sg);
1189
1190         /* 1st seg */
1191         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1192         sg->length = mbuf->data_len - sym->auth.data.offset;
1193         sg->offset = sym->auth.data.offset;
1194
1195         /* Successive segs */
1196         mbuf = mbuf->next;
1197         while (mbuf) {
1198                 cpu_to_hw_sg(sg);
1199                 sg++;
1200                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1201                 sg->length = mbuf->data_len;
1202                 mbuf = mbuf->next;
1203         }
1204         sg->length -= ses->digest_length;
1205
1206         if (is_encode(ses)) {
1207                 cpu_to_hw_sg(sg);
1208                 /* set auth output */
1209                 sg++;
1210                 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
1211                 sg->length = ses->digest_length;
1212         }
1213         sg->final = 1;
1214         cpu_to_hw_sg(sg);
1215
1216         /* input */
1217         mbuf = sym->m_src;
1218         in_sg = &cf->sg[1];
1219         in_sg->extension = 1;
1220         in_sg->final = 1;
1221         if (is_encode(ses))
1222                 in_sg->length = ses->iv.length + sym->auth.data.length;
1223         else
1224                 in_sg->length = ses->iv.length + sym->auth.data.length
1225                                                 + ses->digest_length;
1226
1227         /* input sg entries */
1228         sg++;
1229         qm_sg_entry_set64(in_sg, dpaa_mem_vtop_ctx(ctx, sg));
1230         cpu_to_hw_sg(in_sg);
1231
1232         /* 1st seg IV */
1233         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1234         sg->length = ses->iv.length;
1235         cpu_to_hw_sg(sg);
1236
1237         /* 2nd seg */
1238         sg++;
1239         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1240         sg->length = mbuf->data_len - sym->auth.data.offset;
1241         sg->offset = sym->auth.data.offset;
1242
1243         /* Successive segs */
1244         mbuf = mbuf->next;
1245         while (mbuf) {
1246                 cpu_to_hw_sg(sg);
1247                 sg++;
1248                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1249                 sg->length = mbuf->data_len;
1250                 mbuf = mbuf->next;
1251         }
1252
1253         sg->length -= ses->digest_length;
1254         if (is_decode(ses)) {
1255                 cpu_to_hw_sg(sg);
1256                 sg++;
1257                 memcpy(ctx->digest, sym->auth.digest.data,
1258                         ses->digest_length);
1259                 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, ctx->digest));
1260                 sg->length = ses->digest_length;
1261         }
1262         sg->final = 1;
1263         cpu_to_hw_sg(sg);
1264
1265         return cf;
1266 }
1267
1268 static inline struct dpaa_sec_job *
1269 build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses)
1270 {
1271         struct rte_crypto_sym_op *sym = op->sym;
1272         struct dpaa_sec_job *cf;
1273         struct dpaa_sec_op_ctx *ctx;
1274         struct qm_sg_entry *sg;
1275         rte_iova_t src_start_addr, dst_start_addr;
1276         uint32_t length = 0;
1277         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1278                         ses->iv.offset);
1279
1280         src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
1281         if (sym->m_dst)
1282                 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
1283         else
1284                 dst_start_addr = src_start_addr;
1285
1286         ctx = dpaa_sec_alloc_ctx(ses);
1287         if (!ctx)
1288                 return NULL;
1289
1290         cf = &ctx->job;
1291         ctx->op = op;
1292
1293         /* input */
1294         rte_prefetch0(cf->sg);
1295         sg = &cf->sg[2];
1296         qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop_ctx(ctx, sg));
1297         if (is_encode(ses)) {
1298                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1299                 sg->length = ses->iv.length;
1300                 length += sg->length;
1301                 cpu_to_hw_sg(sg);
1302
1303                 sg++;
1304                 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
1305                 sg->length = sym->auth.data.length;
1306                 length += sg->length;
1307                 sg->final = 1;
1308                 cpu_to_hw_sg(sg);
1309         } else {
1310                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1311                 sg->length = ses->iv.length;
1312                 length += sg->length;
1313                 cpu_to_hw_sg(sg);
1314
1315                 sg++;
1316
1317                 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
1318                 sg->length = sym->auth.data.length;
1319                 length += sg->length;
1320                 cpu_to_hw_sg(sg);
1321
1322                 memcpy(ctx->digest, sym->auth.digest.data,
1323                        ses->digest_length);
1324                 sg++;
1325
1326                 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, ctx->digest));
1327                 sg->length = ses->digest_length;
1328                 length += sg->length;
1329                 sg->final = 1;
1330                 cpu_to_hw_sg(sg);
1331         }
1332         /* input compound frame */
1333         cf->sg[1].length = length;
1334         cf->sg[1].extension = 1;
1335         cf->sg[1].final = 1;
1336         cpu_to_hw_sg(&cf->sg[1]);
1337
1338         /* output */
1339         sg++;
1340         qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop_ctx(ctx, sg));
1341         qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset);
1342         sg->length = sym->cipher.data.length;
1343         length = sg->length;
1344         if (is_encode(ses)) {
1345                 cpu_to_hw_sg(sg);
1346                 /* set auth output */
1347                 sg++;
1348                 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
1349                 sg->length = ses->digest_length;
1350                 length += sg->length;
1351         }
1352         sg->final = 1;
1353         cpu_to_hw_sg(sg);
1354
1355         /* output compound frame */
1356         cf->sg[0].length = length;
1357         cf->sg[0].extension = 1;
1358         cpu_to_hw_sg(&cf->sg[0]);
1359
1360         return cf;
1361 }
1362
1363 static inline struct dpaa_sec_job *
1364 build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses)
1365 {
1366         struct rte_crypto_sym_op *sym = op->sym;
1367         struct dpaa_sec_job *cf;
1368         struct dpaa_sec_op_ctx *ctx;
1369         struct qm_sg_entry *sg;
1370         phys_addr_t src_start_addr, dst_start_addr;
1371
1372         ctx = dpaa_sec_alloc_ctx(ses);
1373         if (!ctx)
1374                 return NULL;
1375         cf = &ctx->job;
1376         ctx->op = op;
1377
1378         src_start_addr = rte_pktmbuf_mtophys(sym->m_src);
1379
1380         if (sym->m_dst)
1381                 dst_start_addr = rte_pktmbuf_mtophys(sym->m_dst);
1382         else
1383                 dst_start_addr = src_start_addr;
1384
1385         /* input */
1386         sg = &cf->sg[1];
1387         qm_sg_entry_set64(sg, src_start_addr);
1388         sg->length = sym->m_src->pkt_len;
1389         sg->final = 1;
1390         cpu_to_hw_sg(sg);
1391
1392         sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
1393         /* output */
1394         sg = &cf->sg[0];
1395         qm_sg_entry_set64(sg, dst_start_addr);
1396         sg->length = sym->m_src->buf_len - sym->m_src->data_off;
1397         cpu_to_hw_sg(sg);
1398
1399         return cf;
1400 }
1401
1402 static uint16_t
1403 dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1404                        uint16_t nb_ops)
1405 {
1406         /* Function to transmit the frames to given device and queuepair */
1407         uint32_t loop;
1408         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp;
1409         uint16_t num_tx = 0;
1410         struct qm_fd fds[DPAA_SEC_BURST], *fd;
1411         uint32_t frames_to_send;
1412         struct rte_crypto_op *op;
1413         struct dpaa_sec_job *cf;
1414         dpaa_sec_session *ses;
1415         struct dpaa_sec_op_ctx *ctx;
1416         uint32_t auth_only_len;
1417         struct qman_fq *inq[DPAA_SEC_BURST];
1418
1419         while (nb_ops) {
1420                 frames_to_send = (nb_ops > DPAA_SEC_BURST) ?
1421                                 DPAA_SEC_BURST : nb_ops;
1422                 for (loop = 0; loop < frames_to_send; loop++) {
1423                         op = *(ops++);
1424                         switch (op->sess_type) {
1425                         case RTE_CRYPTO_OP_WITH_SESSION:
1426                                 ses = (dpaa_sec_session *)
1427                                         get_session_private_data(
1428                                                         op->sym->session,
1429                                                         cryptodev_driver_id);
1430                                 break;
1431                         case RTE_CRYPTO_OP_SECURITY_SESSION:
1432                                 ses = (dpaa_sec_session *)
1433                                         get_sec_session_private_data(
1434                                                         op->sym->sec_session);
1435                                 break;
1436                         default:
1437                                 DPAA_SEC_DP_ERR(
1438                                         "sessionless crypto op not supported");
1439                                 frames_to_send = loop;
1440                                 nb_ops = loop;
1441                                 goto send_pkts;
1442                         }
1443                         if (unlikely(!ses->qp || ses->qp != qp)) {
1444                                 DPAA_SEC_DP_ERR("sess->qp - %p qp %p",
1445                                              ses->qp, qp);
1446                                 if (dpaa_sec_attach_sess_q(qp, ses)) {
1447                                         frames_to_send = loop;
1448                                         nb_ops = loop;
1449                                         goto send_pkts;
1450                                 }
1451                         }
1452
1453                         auth_only_len = op->sym->auth.data.length -
1454                                                 op->sym->cipher.data.length;
1455                         if (rte_pktmbuf_is_contiguous(op->sym->m_src)) {
1456                                 if (is_auth_only(ses)) {
1457                                         cf = build_auth_only(op, ses);
1458                                 } else if (is_cipher_only(ses)) {
1459                                         cf = build_cipher_only(op, ses);
1460                                 } else if (is_aead(ses)) {
1461                                         cf = build_cipher_auth_gcm(op, ses);
1462                                         auth_only_len = ses->auth_only_len;
1463                                 } else if (is_auth_cipher(ses)) {
1464                                         cf = build_cipher_auth(op, ses);
1465                                 } else if (is_proto_ipsec(ses)) {
1466                                         cf = build_proto(op, ses);
1467                                 } else {
1468                                         DPAA_SEC_DP_ERR("not supported ops");
1469                                         frames_to_send = loop;
1470                                         nb_ops = loop;
1471                                         goto send_pkts;
1472                                 }
1473                         } else {
1474                                 if (is_auth_only(ses)) {
1475                                         cf = build_auth_only_sg(op, ses);
1476                                 } else if (is_cipher_only(ses)) {
1477                                         cf = build_cipher_only_sg(op, ses);
1478                                 } else if (is_aead(ses)) {
1479                                         cf = build_cipher_auth_gcm_sg(op, ses);
1480                                         auth_only_len = ses->auth_only_len;
1481                                 } else if (is_auth_cipher(ses)) {
1482                                         cf = build_cipher_auth_sg(op, ses);
1483                                 } else {
1484                                         DPAA_SEC_DP_ERR("not supported ops");
1485                                         frames_to_send = loop;
1486                                         nb_ops = loop;
1487                                         goto send_pkts;
1488                                 }
1489                         }
1490                         if (unlikely(!cf)) {
1491                                 frames_to_send = loop;
1492                                 nb_ops = loop;
1493                                 goto send_pkts;
1494                         }
1495
1496                         fd = &fds[loop];
1497                         inq[loop] = ses->inq;
1498                         fd->opaque_addr = 0;
1499                         fd->cmd = 0;
1500                         ctx = container_of(cf, struct dpaa_sec_op_ctx, job);
1501                         qm_fd_addr_set64(fd, dpaa_mem_vtop_ctx(ctx, cf->sg));
1502                         fd->_format1 = qm_fd_compound;
1503                         fd->length29 = 2 * sizeof(struct qm_sg_entry);
1504                         /* Auth_only_len is set as 0 in descriptor and it is
1505                          * overwritten here in the fd.cmd which will update
1506                          * the DPOVRD reg.
1507                          */
1508                         if (auth_only_len)
1509                                 fd->cmd = 0x80000000 | auth_only_len;
1510
1511                 }
1512 send_pkts:
1513                 loop = 0;
1514                 while (loop < frames_to_send) {
1515                         loop += qman_enqueue_multi_fq(&inq[loop], &fds[loop],
1516                                         frames_to_send - loop);
1517                 }
1518                 nb_ops -= frames_to_send;
1519                 num_tx += frames_to_send;
1520         }
1521
1522         dpaa_qp->tx_pkts += num_tx;
1523         dpaa_qp->tx_errs += nb_ops - num_tx;
1524
1525         return num_tx;
1526 }
1527
1528 static uint16_t
1529 dpaa_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops,
1530                        uint16_t nb_ops)
1531 {
1532         uint16_t num_rx;
1533         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp;
1534
1535         num_rx = dpaa_sec_deq(dpaa_qp, ops, nb_ops);
1536
1537         dpaa_qp->rx_pkts += num_rx;
1538         dpaa_qp->rx_errs += nb_ops - num_rx;
1539
1540         DPAA_SEC_DP_DEBUG("SEC Received %d Packets\n", num_rx);
1541
1542         return num_rx;
1543 }
1544
1545 /** Release queue pair */
1546 static int
1547 dpaa_sec_queue_pair_release(struct rte_cryptodev *dev,
1548                             uint16_t qp_id)
1549 {
1550         struct dpaa_sec_dev_private *internals;
1551         struct dpaa_sec_qp *qp = NULL;
1552
1553         PMD_INIT_FUNC_TRACE();
1554
1555         DPAA_SEC_DEBUG("dev =%p, queue =%d", dev, qp_id);
1556
1557         internals = dev->data->dev_private;
1558         if (qp_id >= internals->max_nb_queue_pairs) {
1559                 DPAA_SEC_ERR("Max supported qpid %d",
1560                              internals->max_nb_queue_pairs);
1561                 return -EINVAL;
1562         }
1563
1564         qp = &internals->qps[qp_id];
1565         qp->internals = NULL;
1566         dev->data->queue_pairs[qp_id] = NULL;
1567
1568         return 0;
1569 }
1570
1571 /** Setup a queue pair */
1572 static int
1573 dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
1574                 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
1575                 __rte_unused int socket_id,
1576                 __rte_unused struct rte_mempool *session_pool)
1577 {
1578         struct dpaa_sec_dev_private *internals;
1579         struct dpaa_sec_qp *qp = NULL;
1580
1581         DPAA_SEC_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);
1582
1583         internals = dev->data->dev_private;
1584         if (qp_id >= internals->max_nb_queue_pairs) {
1585                 DPAA_SEC_ERR("Max supported qpid %d",
1586                              internals->max_nb_queue_pairs);
1587                 return -EINVAL;
1588         }
1589
1590         qp = &internals->qps[qp_id];
1591         qp->internals = internals;
1592         dev->data->queue_pairs[qp_id] = qp;
1593
1594         return 0;
1595 }
1596
1597 /** Start queue pair */
1598 static int
1599 dpaa_sec_queue_pair_start(__rte_unused struct rte_cryptodev *dev,
1600                           __rte_unused uint16_t queue_pair_id)
1601 {
1602         PMD_INIT_FUNC_TRACE();
1603
1604         return 0;
1605 }
1606
1607 /** Stop queue pair */
1608 static int
1609 dpaa_sec_queue_pair_stop(__rte_unused struct rte_cryptodev *dev,
1610                          __rte_unused uint16_t queue_pair_id)
1611 {
1612         PMD_INIT_FUNC_TRACE();
1613
1614         return 0;
1615 }
1616
1617 /** Return the number of allocated queue pairs */
1618 static uint32_t
1619 dpaa_sec_queue_pair_count(struct rte_cryptodev *dev)
1620 {
1621         PMD_INIT_FUNC_TRACE();
1622
1623         return dev->data->nb_queue_pairs;
1624 }
1625
1626 /** Returns the size of session structure */
1627 static unsigned int
1628 dpaa_sec_session_get_size(struct rte_cryptodev *dev __rte_unused)
1629 {
1630         PMD_INIT_FUNC_TRACE();
1631
1632         return sizeof(dpaa_sec_session);
1633 }
1634
1635 static int
1636 dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused,
1637                      struct rte_crypto_sym_xform *xform,
1638                      dpaa_sec_session *session)
1639 {
1640         session->cipher_alg = xform->cipher.algo;
1641         session->iv.length = xform->cipher.iv.length;
1642         session->iv.offset = xform->cipher.iv.offset;
1643         session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
1644                                                RTE_CACHE_LINE_SIZE);
1645         if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
1646                 DPAA_SEC_ERR("No Memory for cipher key");
1647                 return -ENOMEM;
1648         }
1649         session->cipher_key.length = xform->cipher.key.length;
1650
1651         memcpy(session->cipher_key.data, xform->cipher.key.data,
1652                xform->cipher.key.length);
1653         session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
1654                         DIR_ENC : DIR_DEC;
1655
1656         return 0;
1657 }
1658
1659 static int
1660 dpaa_sec_auth_init(struct rte_cryptodev *dev __rte_unused,
1661                    struct rte_crypto_sym_xform *xform,
1662                    dpaa_sec_session *session)
1663 {
1664         session->auth_alg = xform->auth.algo;
1665         session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
1666                                              RTE_CACHE_LINE_SIZE);
1667         if (session->auth_key.data == NULL && xform->auth.key.length > 0) {
1668                 DPAA_SEC_ERR("No Memory for auth key");
1669                 return -ENOMEM;
1670         }
1671         session->auth_key.length = xform->auth.key.length;
1672         session->digest_length = xform->auth.digest_length;
1673
1674         memcpy(session->auth_key.data, xform->auth.key.data,
1675                xform->auth.key.length);
1676         session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
1677                         DIR_ENC : DIR_DEC;
1678
1679         return 0;
1680 }
1681
1682 static int
1683 dpaa_sec_aead_init(struct rte_cryptodev *dev __rte_unused,
1684                    struct rte_crypto_sym_xform *xform,
1685                    dpaa_sec_session *session)
1686 {
1687         session->aead_alg = xform->aead.algo;
1688         session->iv.length = xform->aead.iv.length;
1689         session->iv.offset = xform->aead.iv.offset;
1690         session->auth_only_len = xform->aead.aad_length;
1691         session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length,
1692                                              RTE_CACHE_LINE_SIZE);
1693         if (session->aead_key.data == NULL && xform->aead.key.length > 0) {
1694                 DPAA_SEC_ERR("No Memory for aead key\n");
1695                 return -ENOMEM;
1696         }
1697         session->aead_key.length = xform->aead.key.length;
1698         session->digest_length = xform->aead.digest_length;
1699
1700         memcpy(session->aead_key.data, xform->aead.key.data,
1701                xform->aead.key.length);
1702         session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
1703                         DIR_ENC : DIR_DEC;
1704
1705         return 0;
1706 }
1707
1708 static struct qman_fq *
1709 dpaa_sec_attach_rxq(struct dpaa_sec_dev_private *qi)
1710 {
1711         unsigned int i;
1712
1713         for (i = 0; i < qi->max_nb_sessions; i++) {
1714                 if (qi->inq_attach[i] == 0) {
1715                         qi->inq_attach[i] = 1;
1716                         return &qi->inq[i];
1717                 }
1718         }
1719         DPAA_SEC_WARN("All ses session in use %x", qi->max_nb_sessions);
1720
1721         return NULL;
1722 }
1723
1724 static int
1725 dpaa_sec_detach_rxq(struct dpaa_sec_dev_private *qi, struct qman_fq *fq)
1726 {
1727         unsigned int i;
1728
1729         for (i = 0; i < qi->max_nb_sessions; i++) {
1730                 if (&qi->inq[i] == fq) {
1731                         qman_retire_fq(fq, NULL);
1732                         qman_oos_fq(fq);
1733                         qi->inq_attach[i] = 0;
1734                         return 0;
1735                 }
1736         }
1737         return -1;
1738 }
1739
1740 static int
1741 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess)
1742 {
1743         int ret;
1744
1745         sess->qp = qp;
1746         ret = dpaa_sec_prep_cdb(sess);
1747         if (ret) {
1748                 DPAA_SEC_ERR("Unable to prepare sec cdb");
1749                 return -1;
1750         }
1751         if (unlikely(!RTE_PER_LCORE(dpaa_io))) {
1752                 ret = rte_dpaa_portal_init((void *)0);
1753                 if (ret) {
1754                         DPAA_SEC_ERR("Failure in affining portal");
1755                         return ret;
1756                 }
1757         }
1758         ret = dpaa_sec_init_rx(sess->inq, dpaa_mem_vtop(&sess->cdb),
1759                                qman_fq_fqid(&qp->outq));
1760         if (ret)
1761                 DPAA_SEC_ERR("Unable to init sec queue");
1762
1763         return ret;
1764 }
1765
1766 static int
1767 dpaa_sec_qp_attach_sess(struct rte_cryptodev *dev __rte_unused,
1768                         uint16_t qp_id __rte_unused,
1769                         void *ses __rte_unused)
1770 {
1771         PMD_INIT_FUNC_TRACE();
1772         return 0;
1773 }
1774
1775 static int
1776 dpaa_sec_qp_detach_sess(struct rte_cryptodev *dev,
1777                         uint16_t qp_id  __rte_unused,
1778                         void *ses)
1779 {
1780         dpaa_sec_session *sess = ses;
1781         struct dpaa_sec_dev_private *qi = dev->data->dev_private;
1782
1783         PMD_INIT_FUNC_TRACE();
1784
1785         if (sess->inq)
1786                 dpaa_sec_detach_rxq(qi, sess->inq);
1787         sess->inq = NULL;
1788
1789         sess->qp = NULL;
1790
1791         return 0;
1792 }
1793
1794 static int
1795 dpaa_sec_set_session_parameters(struct rte_cryptodev *dev,
1796                             struct rte_crypto_sym_xform *xform, void *sess)
1797 {
1798         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
1799         dpaa_sec_session *session = sess;
1800
1801         PMD_INIT_FUNC_TRACE();
1802
1803         if (unlikely(sess == NULL)) {
1804                 DPAA_SEC_ERR("invalid session struct");
1805                 return -EINVAL;
1806         }
1807
1808         /* Default IV length = 0 */
1809         session->iv.length = 0;
1810
1811         /* Cipher Only */
1812         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
1813                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1814                 dpaa_sec_cipher_init(dev, xform, session);
1815
1816         /* Authentication Only */
1817         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1818                    xform->next == NULL) {
1819                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
1820                 dpaa_sec_auth_init(dev, xform, session);
1821
1822         /* Cipher then Authenticate */
1823         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1824                    xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
1825                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1826                         dpaa_sec_cipher_init(dev, xform, session);
1827                         dpaa_sec_auth_init(dev, xform->next, session);
1828                 } else {
1829                         DPAA_SEC_ERR("Not supported: Auth then Cipher");
1830                         return -EINVAL;
1831                 }
1832
1833         /* Authenticate then Cipher */
1834         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1835                    xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
1836                 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
1837                         dpaa_sec_auth_init(dev, xform, session);
1838                         dpaa_sec_cipher_init(dev, xform->next, session);
1839                 } else {
1840                         DPAA_SEC_ERR("Not supported: Auth then Cipher");
1841                         return -EINVAL;
1842                 }
1843
1844         /* AEAD operation for AES-GCM kind of Algorithms */
1845         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
1846                    xform->next == NULL) {
1847                 dpaa_sec_aead_init(dev, xform, session);
1848
1849         } else {
1850                 DPAA_SEC_ERR("Invalid crypto type");
1851                 return -EINVAL;
1852         }
1853         session->ctx_pool = internals->ctx_pool;
1854         session->inq = dpaa_sec_attach_rxq(internals);
1855         if (session->inq == NULL) {
1856                 DPAA_SEC_ERR("unable to attach sec queue");
1857                 goto err1;
1858         }
1859
1860         return 0;
1861
1862 err1:
1863         rte_free(session->cipher_key.data);
1864         rte_free(session->auth_key.data);
1865         memset(session, 0, sizeof(dpaa_sec_session));
1866
1867         return -EINVAL;
1868 }
1869
1870 static int
1871 dpaa_sec_session_configure(struct rte_cryptodev *dev,
1872                 struct rte_crypto_sym_xform *xform,
1873                 struct rte_cryptodev_sym_session *sess,
1874                 struct rte_mempool *mempool)
1875 {
1876         void *sess_private_data;
1877         int ret;
1878
1879         PMD_INIT_FUNC_TRACE();
1880
1881         if (rte_mempool_get(mempool, &sess_private_data)) {
1882                 DPAA_SEC_ERR("Couldn't get object from session mempool");
1883                 return -ENOMEM;
1884         }
1885
1886         ret = dpaa_sec_set_session_parameters(dev, xform, sess_private_data);
1887         if (ret != 0) {
1888                 DPAA_SEC_ERR("failed to configure session parameters");
1889
1890                 /* Return session to mempool */
1891                 rte_mempool_put(mempool, sess_private_data);
1892                 return ret;
1893         }
1894
1895         set_session_private_data(sess, dev->driver_id,
1896                         sess_private_data);
1897
1898
1899         return 0;
1900 }
1901
1902 /** Clear the memory of session so it doesn't leave key material behind */
1903 static void
1904 dpaa_sec_session_clear(struct rte_cryptodev *dev,
1905                 struct rte_cryptodev_sym_session *sess)
1906 {
1907         struct dpaa_sec_dev_private *qi = dev->data->dev_private;
1908         uint8_t index = dev->driver_id;
1909         void *sess_priv = get_session_private_data(sess, index);
1910
1911         PMD_INIT_FUNC_TRACE();
1912
1913         dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
1914
1915         if (sess_priv) {
1916                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1917
1918                 if (s->inq)
1919                         dpaa_sec_detach_rxq(qi, s->inq);
1920                 rte_free(s->cipher_key.data);
1921                 rte_free(s->auth_key.data);
1922                 memset(s, 0, sizeof(dpaa_sec_session));
1923                 set_session_private_data(sess, index, NULL);
1924                 rte_mempool_put(sess_mp, sess_priv);
1925         }
1926 }
1927
1928 static int
1929 dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
1930                            struct rte_security_session_conf *conf,
1931                            void *sess)
1932 {
1933         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
1934         struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
1935         struct rte_crypto_auth_xform *auth_xform;
1936         struct rte_crypto_cipher_xform *cipher_xform;
1937         dpaa_sec_session *session = (dpaa_sec_session *)sess;
1938
1939         PMD_INIT_FUNC_TRACE();
1940
1941         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
1942                 cipher_xform = &conf->crypto_xform->cipher;
1943                 auth_xform = &conf->crypto_xform->next->auth;
1944         } else {
1945                 auth_xform = &conf->crypto_xform->auth;
1946                 cipher_xform = &conf->crypto_xform->next->cipher;
1947         }
1948         session->proto_alg = conf->protocol;
1949         session->cipher_key.data = rte_zmalloc(NULL,
1950                                                cipher_xform->key.length,
1951                                                RTE_CACHE_LINE_SIZE);
1952         if (session->cipher_key.data == NULL &&
1953                         cipher_xform->key.length > 0) {
1954                 DPAA_SEC_ERR("No Memory for cipher key");
1955                 return -ENOMEM;
1956         }
1957
1958         session->cipher_key.length = cipher_xform->key.length;
1959         session->auth_key.data = rte_zmalloc(NULL,
1960                                         auth_xform->key.length,
1961                                         RTE_CACHE_LINE_SIZE);
1962         if (session->auth_key.data == NULL &&
1963                         auth_xform->key.length > 0) {
1964                 DPAA_SEC_ERR("No Memory for auth key");
1965                 rte_free(session->cipher_key.data);
1966                 return -ENOMEM;
1967         }
1968         session->auth_key.length = auth_xform->key.length;
1969         memcpy(session->cipher_key.data, cipher_xform->key.data,
1970                         cipher_xform->key.length);
1971         memcpy(session->auth_key.data, auth_xform->key.data,
1972                         auth_xform->key.length);
1973
1974         switch (auth_xform->algo) {
1975         case RTE_CRYPTO_AUTH_SHA1_HMAC:
1976                 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
1977                 break;
1978         case RTE_CRYPTO_AUTH_MD5_HMAC:
1979                 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
1980                 break;
1981         case RTE_CRYPTO_AUTH_SHA256_HMAC:
1982                 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
1983                 break;
1984         case RTE_CRYPTO_AUTH_SHA384_HMAC:
1985                 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
1986                 break;
1987         case RTE_CRYPTO_AUTH_SHA512_HMAC:
1988                 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
1989                 break;
1990         case RTE_CRYPTO_AUTH_AES_CMAC:
1991                 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
1992                 break;
1993         case RTE_CRYPTO_AUTH_NULL:
1994                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1995                 break;
1996         case RTE_CRYPTO_AUTH_SHA224_HMAC:
1997         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
1998         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
1999         case RTE_CRYPTO_AUTH_SHA1:
2000         case RTE_CRYPTO_AUTH_SHA256:
2001         case RTE_CRYPTO_AUTH_SHA512:
2002         case RTE_CRYPTO_AUTH_SHA224:
2003         case RTE_CRYPTO_AUTH_SHA384:
2004         case RTE_CRYPTO_AUTH_MD5:
2005         case RTE_CRYPTO_AUTH_AES_GMAC:
2006         case RTE_CRYPTO_AUTH_KASUMI_F9:
2007         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2008         case RTE_CRYPTO_AUTH_ZUC_EIA3:
2009                 DPAA_SEC_ERR("Crypto: Unsupported auth alg %u",
2010                         auth_xform->algo);
2011                 goto out;
2012         default:
2013                 DPAA_SEC_ERR("Crypto: Undefined Auth specified %u",
2014                         auth_xform->algo);
2015                 goto out;
2016         }
2017
2018         switch (cipher_xform->algo) {
2019         case RTE_CRYPTO_CIPHER_AES_CBC:
2020                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
2021                 break;
2022         case RTE_CRYPTO_CIPHER_3DES_CBC:
2023                 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
2024                 break;
2025         case RTE_CRYPTO_CIPHER_AES_CTR:
2026                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
2027                 break;
2028         case RTE_CRYPTO_CIPHER_NULL:
2029         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2030         case RTE_CRYPTO_CIPHER_3DES_ECB:
2031         case RTE_CRYPTO_CIPHER_AES_ECB:
2032         case RTE_CRYPTO_CIPHER_KASUMI_F8:
2033                 DPAA_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2034                         cipher_xform->algo);
2035                 goto out;
2036         default:
2037                 DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u",
2038                         cipher_xform->algo);
2039                 goto out;
2040         }
2041
2042         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
2043                 memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) +
2044                                 sizeof(session->ip4_hdr));
2045                 session->ip4_hdr.ip_v = IPVERSION;
2046                 session->ip4_hdr.ip_hl = 5;
2047                 session->ip4_hdr.ip_len = rte_cpu_to_be_16(
2048                                                 sizeof(session->ip4_hdr));
2049                 session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
2050                 session->ip4_hdr.ip_id = 0;
2051                 session->ip4_hdr.ip_off = 0;
2052                 session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
2053                 session->ip4_hdr.ip_p = (ipsec_xform->proto ==
2054                                 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP
2055                                 : IPPROTO_AH;
2056                 session->ip4_hdr.ip_sum = 0;
2057                 session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
2058                 session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
2059                 session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
2060                                                 (void *)&session->ip4_hdr,
2061                                                 sizeof(struct ip));
2062
2063                 session->encap_pdb.options =
2064                         (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
2065                         PDBOPTS_ESP_OIHI_PDB_INL |
2066                         PDBOPTS_ESP_IVSRC |
2067                         PDBHMO_ESP_ENCAP_DTTL;
2068                 session->encap_pdb.spi = ipsec_xform->spi;
2069                 session->encap_pdb.ip_hdr_len = sizeof(struct ip);
2070
2071                 session->dir = DIR_ENC;
2072         } else if (ipsec_xform->direction ==
2073                         RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
2074                 memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
2075                 session->decap_pdb.options = sizeof(struct ip) << 16;
2076                 session->dir = DIR_DEC;
2077         } else
2078                 goto out;
2079         session->ctx_pool = internals->ctx_pool;
2080         session->inq = dpaa_sec_attach_rxq(internals);
2081         if (session->inq == NULL) {
2082                 DPAA_SEC_ERR("unable to attach sec queue");
2083                 goto out;
2084         }
2085
2086
2087         return 0;
2088 out:
2089         rte_free(session->auth_key.data);
2090         rte_free(session->cipher_key.data);
2091         memset(session, 0, sizeof(dpaa_sec_session));
2092         return -1;
2093 }
2094
2095 static int
2096 dpaa_sec_security_session_create(void *dev,
2097                                  struct rte_security_session_conf *conf,
2098                                  struct rte_security_session *sess,
2099                                  struct rte_mempool *mempool)
2100 {
2101         void *sess_private_data;
2102         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
2103         int ret;
2104
2105         if (rte_mempool_get(mempool, &sess_private_data)) {
2106                 DPAA_SEC_ERR("Couldn't get object from session mempool");
2107                 return -ENOMEM;
2108         }
2109
2110         switch (conf->protocol) {
2111         case RTE_SECURITY_PROTOCOL_IPSEC:
2112                 ret = dpaa_sec_set_ipsec_session(cdev, conf,
2113                                 sess_private_data);
2114                 break;
2115         case RTE_SECURITY_PROTOCOL_MACSEC:
2116                 return -ENOTSUP;
2117         default:
2118                 return -EINVAL;
2119         }
2120         if (ret != 0) {
2121                 DPAA_SEC_ERR("failed to configure session parameters");
2122                 /* Return session to mempool */
2123                 rte_mempool_put(mempool, sess_private_data);
2124                 return ret;
2125         }
2126
2127         set_sec_session_private_data(sess, sess_private_data);
2128
2129         return ret;
2130 }
2131
2132 /** Clear the memory of session so it doesn't leave key material behind */
2133 static int
2134 dpaa_sec_security_session_destroy(void *dev __rte_unused,
2135                 struct rte_security_session *sess)
2136 {
2137         PMD_INIT_FUNC_TRACE();
2138         void *sess_priv = get_sec_session_private_data(sess);
2139
2140         dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
2141
2142         if (sess_priv) {
2143                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
2144
2145                 rte_free(s->cipher_key.data);
2146                 rte_free(s->auth_key.data);
2147                 memset(sess, 0, sizeof(dpaa_sec_session));
2148                 set_sec_session_private_data(sess, NULL);
2149                 rte_mempool_put(sess_mp, sess_priv);
2150         }
2151         return 0;
2152 }
2153
2154
2155 static int
2156 dpaa_sec_dev_configure(struct rte_cryptodev *dev,
2157                        struct rte_cryptodev_config *config __rte_unused)
2158 {
2159
2160         char str[20];
2161         struct dpaa_sec_dev_private *internals;
2162
2163         PMD_INIT_FUNC_TRACE();
2164
2165         internals = dev->data->dev_private;
2166         sprintf(str, "ctx_pool_%d", dev->data->dev_id);
2167         if (!internals->ctx_pool) {
2168                 internals->ctx_pool = rte_mempool_create((const char *)str,
2169                                                         CTX_POOL_NUM_BUFS,
2170                                                         CTX_POOL_BUF_SIZE,
2171                                                         CTX_POOL_CACHE_SIZE, 0,
2172                                                         NULL, NULL, NULL, NULL,
2173                                                         SOCKET_ID_ANY, 0);
2174                 if (!internals->ctx_pool) {
2175                         DPAA_SEC_ERR("%s create failed\n", str);
2176                         return -ENOMEM;
2177                 }
2178         } else
2179                 DPAA_SEC_INFO("mempool already created for dev_id : %d",
2180                                 dev->data->dev_id);
2181
2182         return 0;
2183 }
2184
2185 static int
2186 dpaa_sec_dev_start(struct rte_cryptodev *dev __rte_unused)
2187 {
2188         PMD_INIT_FUNC_TRACE();
2189         return 0;
2190 }
2191
2192 static void
2193 dpaa_sec_dev_stop(struct rte_cryptodev *dev __rte_unused)
2194 {
2195         PMD_INIT_FUNC_TRACE();
2196 }
2197
2198 static int
2199 dpaa_sec_dev_close(struct rte_cryptodev *dev)
2200 {
2201         struct dpaa_sec_dev_private *internals;
2202
2203         PMD_INIT_FUNC_TRACE();
2204
2205         if (dev == NULL)
2206                 return -ENOMEM;
2207
2208         internals = dev->data->dev_private;
2209         rte_mempool_free(internals->ctx_pool);
2210         internals->ctx_pool = NULL;
2211
2212         return 0;
2213 }
2214
2215 static void
2216 dpaa_sec_dev_infos_get(struct rte_cryptodev *dev,
2217                        struct rte_cryptodev_info *info)
2218 {
2219         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
2220
2221         PMD_INIT_FUNC_TRACE();
2222         if (info != NULL) {
2223                 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
2224                 info->feature_flags = dev->feature_flags;
2225                 info->capabilities = dpaa_sec_capabilities;
2226                 info->sym.max_nb_sessions = internals->max_nb_sessions;
2227                 info->sym.max_nb_sessions_per_qp =
2228                         RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS /
2229                         RTE_DPAA_MAX_NB_SEC_QPS;
2230                 info->driver_id = cryptodev_driver_id;
2231         }
2232 }
2233
2234 static struct rte_cryptodev_ops crypto_ops = {
2235         .dev_configure        = dpaa_sec_dev_configure,
2236         .dev_start            = dpaa_sec_dev_start,
2237         .dev_stop             = dpaa_sec_dev_stop,
2238         .dev_close            = dpaa_sec_dev_close,
2239         .dev_infos_get        = dpaa_sec_dev_infos_get,
2240         .queue_pair_setup     = dpaa_sec_queue_pair_setup,
2241         .queue_pair_release   = dpaa_sec_queue_pair_release,
2242         .queue_pair_start     = dpaa_sec_queue_pair_start,
2243         .queue_pair_stop      = dpaa_sec_queue_pair_stop,
2244         .queue_pair_count     = dpaa_sec_queue_pair_count,
2245         .session_get_size     = dpaa_sec_session_get_size,
2246         .session_configure    = dpaa_sec_session_configure,
2247         .session_clear        = dpaa_sec_session_clear,
2248         .qp_attach_session    = dpaa_sec_qp_attach_sess,
2249         .qp_detach_session    = dpaa_sec_qp_detach_sess,
2250 };
2251
2252 static const struct rte_security_capability *
2253 dpaa_sec_capabilities_get(void *device __rte_unused)
2254 {
2255         return dpaa_sec_security_cap;
2256 }
2257
2258 struct rte_security_ops dpaa_sec_security_ops = {
2259         .session_create = dpaa_sec_security_session_create,
2260         .session_update = NULL,
2261         .session_stats_get = NULL,
2262         .session_destroy = dpaa_sec_security_session_destroy,
2263         .set_pkt_metadata = NULL,
2264         .capabilities_get = dpaa_sec_capabilities_get
2265 };
2266
2267 static int
2268 dpaa_sec_uninit(struct rte_cryptodev *dev)
2269 {
2270         struct dpaa_sec_dev_private *internals;
2271
2272         if (dev == NULL)
2273                 return -ENODEV;
2274
2275         internals = dev->data->dev_private;
2276         rte_free(dev->security_ctx);
2277
2278         /* In case close has been called, internals->ctx_pool would be NULL */
2279         rte_mempool_free(internals->ctx_pool);
2280         rte_free(internals);
2281
2282         DPAA_SEC_INFO("Closing DPAA_SEC device %s on numa socket %u",
2283                       dev->data->name, rte_socket_id());
2284
2285         return 0;
2286 }
2287
2288 static int
2289 dpaa_sec_dev_init(struct rte_cryptodev *cryptodev)
2290 {
2291         struct dpaa_sec_dev_private *internals;
2292         struct rte_security_ctx *security_instance;
2293         struct dpaa_sec_qp *qp;
2294         uint32_t i, flags;
2295         int ret;
2296
2297         PMD_INIT_FUNC_TRACE();
2298
2299         cryptodev->driver_id = cryptodev_driver_id;
2300         cryptodev->dev_ops = &crypto_ops;
2301
2302         cryptodev->enqueue_burst = dpaa_sec_enqueue_burst;
2303         cryptodev->dequeue_burst = dpaa_sec_dequeue_burst;
2304         cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2305                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
2306                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2307                         RTE_CRYPTODEV_FF_SECURITY |
2308                         RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER;
2309
2310         internals = cryptodev->data->dev_private;
2311         internals->max_nb_queue_pairs = RTE_DPAA_MAX_NB_SEC_QPS;
2312         internals->max_nb_sessions = RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS;
2313
2314         /*
2315          * For secondary processes, we don't initialise any further as primary
2316          * has already done this work. Only check we don't need a different
2317          * RX function
2318          */
2319         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2320                 DPAA_SEC_WARN("Device already init by primary process");
2321                 return 0;
2322         }
2323
2324         /* Initialize security_ctx only for primary process*/
2325         security_instance = rte_malloc("rte_security_instances_ops",
2326                                 sizeof(struct rte_security_ctx), 0);
2327         if (security_instance == NULL)
2328                 return -ENOMEM;
2329         security_instance->device = (void *)cryptodev;
2330         security_instance->ops = &dpaa_sec_security_ops;
2331         security_instance->sess_cnt = 0;
2332         cryptodev->security_ctx = security_instance;
2333
2334         for (i = 0; i < internals->max_nb_queue_pairs; i++) {
2335                 /* init qman fq for queue pair */
2336                 qp = &internals->qps[i];
2337                 ret = dpaa_sec_init_tx(&qp->outq);
2338                 if (ret) {
2339                         DPAA_SEC_ERR("config tx of queue pair  %d", i);
2340                         goto init_error;
2341                 }
2342         }
2343
2344         flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID |
2345                 QMAN_FQ_FLAG_TO_DCPORTAL;
2346         for (i = 0; i < internals->max_nb_sessions; i++) {
2347                 /* create rx qman fq for sessions*/
2348                 ret = qman_create_fq(0, flags, &internals->inq[i]);
2349                 if (unlikely(ret != 0)) {
2350                         DPAA_SEC_ERR("sec qman_create_fq failed");
2351                         goto init_error;
2352                 }
2353         }
2354
2355         RTE_LOG(INFO, PMD, "%s cryptodev init\n", cryptodev->data->name);
2356         return 0;
2357
2358 init_error:
2359         DPAA_SEC_ERR("driver %s: create failed\n", cryptodev->data->name);
2360
2361         dpaa_sec_uninit(cryptodev);
2362         return -EFAULT;
2363 }
2364
2365 static int
2366 cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv,
2367                                 struct rte_dpaa_device *dpaa_dev)
2368 {
2369         struct rte_cryptodev *cryptodev;
2370         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
2371
2372         int retval;
2373
2374         sprintf(cryptodev_name, "dpaa_sec-%d", dpaa_dev->id.dev_id);
2375
2376         cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id());
2377         if (cryptodev == NULL)
2378                 return -ENOMEM;
2379
2380         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
2381                 cryptodev->data->dev_private = rte_zmalloc_socket(
2382                                         "cryptodev private structure",
2383                                         sizeof(struct dpaa_sec_dev_private),
2384                                         RTE_CACHE_LINE_SIZE,
2385                                         rte_socket_id());
2386
2387                 if (cryptodev->data->dev_private == NULL)
2388                         rte_panic("Cannot allocate memzone for private "
2389                                         "device data");
2390         }
2391
2392         dpaa_dev->crypto_dev = cryptodev;
2393         cryptodev->device = &dpaa_dev->device;
2394         cryptodev->device->driver = &dpaa_drv->driver;
2395
2396         /* init user callbacks */
2397         TAILQ_INIT(&(cryptodev->link_intr_cbs));
2398
2399         /* if sec device version is not configured */
2400         if (!rta_get_sec_era()) {
2401                 const struct device_node *caam_node;
2402
2403                 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
2404                         const uint32_t *prop = of_get_property(caam_node,
2405                                         "fsl,sec-era",
2406                                         NULL);
2407                         if (prop) {
2408                                 rta_set_sec_era(
2409                                         INTL_SEC_ERA(rte_cpu_to_be_32(*prop)));
2410                                 break;
2411                         }
2412                 }
2413         }
2414
2415         /* Invoke PMD device initialization function */
2416         retval = dpaa_sec_dev_init(cryptodev);
2417         if (retval == 0)
2418                 return 0;
2419
2420         /* In case of error, cleanup is done */
2421         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
2422                 rte_free(cryptodev->data->dev_private);
2423
2424         rte_cryptodev_pmd_release_device(cryptodev);
2425
2426         return -ENXIO;
2427 }
2428
2429 static int
2430 cryptodev_dpaa_sec_remove(struct rte_dpaa_device *dpaa_dev)
2431 {
2432         struct rte_cryptodev *cryptodev;
2433         int ret;
2434
2435         cryptodev = dpaa_dev->crypto_dev;
2436         if (cryptodev == NULL)
2437                 return -ENODEV;
2438
2439         ret = dpaa_sec_uninit(cryptodev);
2440         if (ret)
2441                 return ret;
2442
2443         return rte_cryptodev_pmd_destroy(cryptodev);
2444 }
2445
2446 static struct rte_dpaa_driver rte_dpaa_sec_driver = {
2447         .drv_type = FSL_DPAA_CRYPTO,
2448         .driver = {
2449                 .name = "DPAA SEC PMD"
2450         },
2451         .probe = cryptodev_dpaa_sec_probe,
2452         .remove = cryptodev_dpaa_sec_remove,
2453 };
2454
2455 static struct cryptodev_driver dpaa_sec_crypto_drv;
2456
2457 RTE_PMD_REGISTER_DPAA(CRYPTODEV_NAME_DPAA_SEC_PMD, rte_dpaa_sec_driver);
2458 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa_sec_crypto_drv, rte_dpaa_sec_driver.driver,
2459                 cryptodev_driver_id);
2460
2461 RTE_INIT(dpaa_sec_init_log);
2462 static void
2463 dpaa_sec_init_log(void)
2464 {
2465         dpaa_logtype_sec = rte_log_register("pmd.crypto.dpaa");
2466         if (dpaa_logtype_sec >= 0)
2467                 rte_log_set_level(dpaa_logtype_sec, RTE_LOG_NOTICE);
2468 }