drivers/crypto/dpaa_sec/dpaa_sec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2017-2018 NXP
   5  *
   6  */
   7
   8 #include <fcntl.h>
   9 #include <unistd.h>
  10 #include <sched.h>
  11 #include <net/if.h>
  12
  13 #include <rte_byteorder.h>
  14 #include <rte_common.h>
  15 #include <rte_cryptodev_pmd.h>
  16 #include <rte_crypto.h>
  17 #include <rte_cryptodev.h>
  18 #include <rte_security_driver.h>
  19 #include <rte_cycles.h>
  20 #include <rte_dev.h>
  21 #include <rte_kvargs.h>
  22 #include <rte_malloc.h>
  23 #include <rte_mbuf.h>
  24 #include <rte_memcpy.h>
  25 #include <rte_string_fns.h>
  26
  27 #include <fsl_usd.h>
  28 #include <fsl_qman.h>
  29 #include <of.h>
  30
  31 /* RTA header files */
  32 #include <hw/desc/common.h>
  33 #include <hw/desc/algo.h>
  34 #include <hw/desc/ipsec.h>
  35
  36 #include <rte_dpaa_bus.h>
  37 #include <dpaa_sec.h>
  38 #include <dpaa_sec_log.h>
  39
  40 enum rta_sec_era rta_sec_era;
  41
  42 int dpaa_logtype_sec;
  43
  44 static uint8_t cryptodev_driver_id;
  45
  46 static __thread struct rte_crypto_op **dpaa_sec_ops;
  47 static __thread int dpaa_sec_op_nb;
  48
  49 static int
  50 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess);
  51
  52 static inline void
  53 dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx)
  54 {
  55         if (!ctx->fd_status) {
  56                 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
  57         } else {
  58                 DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status);
  59                 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
  60         }
  61
  62         /* report op status to sym->op and then free the ctx memeory  */
  63         rte_mempool_put(ctx->ctx_pool, (void *)ctx);
  64 }
  65
  66 static inline struct dpaa_sec_op_ctx *
  67 dpaa_sec_alloc_ctx(dpaa_sec_session *ses)
  68 {
  69         struct dpaa_sec_op_ctx *ctx;
  70         int retval;
  71
  72         retval = rte_mempool_get(ses->ctx_pool, (void **)(&ctx));
  73         if (!ctx || retval) {
  74                 DPAA_SEC_DP_WARN("Alloc sec descriptor failed!");
  75                 return NULL;
  76         }
  77         /*
  78          * Clear SG memory. There are 16 SG entries of 16 Bytes each.
  79          * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
  80          * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for
  81          * each packet, memset is costlier than dcbz_64().
  82          */
  83         dcbz_64(&ctx->job.sg[SG_CACHELINE_0]);
  84         dcbz_64(&ctx->job.sg[SG_CACHELINE_1]);
  85         dcbz_64(&ctx->job.sg[SG_CACHELINE_2]);
  86         dcbz_64(&ctx->job.sg[SG_CACHELINE_3]);
  87
  88         ctx->ctx_pool = ses->ctx_pool;
  89         ctx->vtop_offset = (size_t) ctx
  90                                 - rte_mempool_virt2iova(ctx);
  91
  92         return ctx;
  93 }
  94
  95 static inline rte_iova_t
  96 dpaa_mem_vtop(void *vaddr)
  97 {
  98         const struct rte_memseg *ms;
  99
 100         ms = rte_mem_virt2memseg(vaddr, NULL);
 101         if (ms)
 102                 return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr);
 103         return (size_t)NULL;
 104 }
 105
 106 static inline void *
 107 dpaa_mem_ptov(rte_iova_t paddr)
 108 {
 109         return rte_mem_iova2virt(paddr);
 110 }
 111
 112 static void
 113 ern_sec_fq_handler(struct qman_portal *qm __rte_unused,
 114                    struct qman_fq *fq,
 115                    const struct qm_mr_entry *msg)
 116 {
 117         DPAA_SEC_DP_ERR("sec fq %d error, RC = %x, seqnum = %x\n",
 118                         fq->fqid, msg->ern.rc, msg->ern.seqnum);
 119 }
 120
 121 /* initialize the queue with dest chan as caam chan so that
 122  * all the packets in this queue could be dispatched into caam
 123  */
 124 static int
 125 dpaa_sec_init_rx(struct qman_fq *fq_in, rte_iova_t hwdesc,
 126                  uint32_t fqid_out)
 127 {
 128         struct qm_mcc_initfq fq_opts;
 129         uint32_t flags;
 130         int ret = -1;
 131
 132         /* Clear FQ options */
 133         memset(&fq_opts, 0x00, sizeof(struct qm_mcc_initfq));
 134
 135         flags = QMAN_INITFQ_FLAG_SCHED;
 136         fq_opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_CONTEXTA |
 137                           QM_INITFQ_WE_CONTEXTB;
 138
 139         qm_fqd_context_a_set64(&fq_opts.fqd, hwdesc);
 140         fq_opts.fqd.context_b = fqid_out;
 141         fq_opts.fqd.dest.channel = qm_channel_caam;
 142         fq_opts.fqd.dest.wq = 0;
 143
 144         fq_in->cb.ern  = ern_sec_fq_handler;
 145
 146         DPAA_SEC_DEBUG("in-%x out-%x", fq_in->fqid, fqid_out);
 147
 148         ret = qman_init_fq(fq_in, flags, &fq_opts);
 149         if (unlikely(ret != 0))
 150                 DPAA_SEC_ERR("qman_init_fq failed %d", ret);
 151
 152         return ret;
 153 }
 154
 155 /* something is put into in_fq and caam put the crypto result into out_fq */
 156 static enum qman_cb_dqrr_result
 157 dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused,
 158                   struct qman_fq *fq __always_unused,
 159                   const struct qm_dqrr_entry *dqrr)
 160 {
 161         const struct qm_fd *fd;
 162         struct dpaa_sec_job *job;
 163         struct dpaa_sec_op_ctx *ctx;
 164
 165         if (dpaa_sec_op_nb >= DPAA_SEC_BURST)
 166                 return qman_cb_dqrr_defer;
 167
 168         if (!(dqrr->stat & QM_DQRR_STAT_FD_VALID))
 169                 return qman_cb_dqrr_consume;
 170
 171         fd = &dqrr->fd;
 172         /* sg is embedded in an op ctx,
 173          * sg[0] is for output
 174          * sg[1] for input
 175          */
 176         job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
 177
 178         ctx = container_of(job, struct dpaa_sec_op_ctx, job);
 179         ctx->fd_status = fd->status;
 180         if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 181                 struct qm_sg_entry *sg_out;
 182                 uint32_t len;
 183
 184                 sg_out = &job->sg[0];
 185                 hw_sg_to_cpu(sg_out);
 186                 len = sg_out->length;
 187                 ctx->op->sym->m_src->pkt_len = len;
 188                 ctx->op->sym->m_src->data_len = len;
 189         }
 190         dpaa_sec_ops[dpaa_sec_op_nb++] = ctx->op;
 191         dpaa_sec_op_ending(ctx);
 192
 193         return qman_cb_dqrr_consume;
 194 }
 195
 196 /* caam result is put into this queue */
 197 static int
 198 dpaa_sec_init_tx(struct qman_fq *fq)
 199 {
 200         int ret;
 201         struct qm_mcc_initfq opts;
 202         uint32_t flags;
 203
 204         flags = QMAN_FQ_FLAG_NO_ENQUEUE | QMAN_FQ_FLAG_LOCKED |
 205                 QMAN_FQ_FLAG_DYNAMIC_FQID;
 206
 207         ret = qman_create_fq(0, flags, fq);
 208         if (unlikely(ret)) {
 209                 DPAA_SEC_ERR("qman_create_fq failed");
 210                 return ret;
 211         }
 212
 213         memset(&opts, 0, sizeof(opts));
 214         opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
 215                        QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB;
 216
 217         /* opts.fqd.dest.channel = dpaa_sec_pool_chan; */
 218
 219         fq->cb.dqrr = dqrr_out_fq_cb_rx;
 220         fq->cb.ern  = ern_sec_fq_handler;
 221
 222         ret = qman_init_fq(fq, 0, &opts);
 223         if (unlikely(ret)) {
 224                 DPAA_SEC_ERR("unable to init caam source fq!");
 225                 return ret;
 226         }
 227
 228         return ret;
 229 }
 230
 231 static inline int is_cipher_only(dpaa_sec_session *ses)
 232 {
 233         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 234                 (ses->auth_alg == RTE_CRYPTO_AUTH_NULL));
 235 }
 236
 237 static inline int is_auth_only(dpaa_sec_session *ses)
 238 {
 239         return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) &&
 240                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL));
 241 }
 242
 243 static inline int is_aead(dpaa_sec_session *ses)
 244 {
 245         return ((ses->cipher_alg == 0) &&
 246                 (ses->auth_alg == 0) &&
 247                 (ses->aead_alg != 0));
 248 }
 249
 250 static inline int is_auth_cipher(dpaa_sec_session *ses)
 251 {
 252         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 253                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) &&
 254                 (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC));
 255 }
 256
 257 static inline int is_proto_ipsec(dpaa_sec_session *ses)
 258 {
 259         return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC);
 260 }
 261
 262 static inline int is_encode(dpaa_sec_session *ses)
 263 {
 264         return ses->dir == DIR_ENC;
 265 }
 266
 267 static inline int is_decode(dpaa_sec_session *ses)
 268 {
 269         return ses->dir == DIR_DEC;
 270 }
 271
 272 static inline void
 273 caam_auth_alg(dpaa_sec_session *ses, struct alginfo *alginfo_a)
 274 {
 275         switch (ses->auth_alg) {
 276         case RTE_CRYPTO_AUTH_NULL:
 277                 ses->digest_length = 0;
 278                 break;
 279         case RTE_CRYPTO_AUTH_MD5_HMAC:
 280                 alginfo_a->algtype =
 281                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 282                         OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5;
 283                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 284                 break;
 285         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 286                 alginfo_a->algtype =
 287                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 288                         OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1;
 289                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 290                 break;
 291         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 292                 alginfo_a->algtype =
 293                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 294                         OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224;
 295                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 296                 break;
 297         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 298                 alginfo_a->algtype =
 299                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 300                         OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256;
 301                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 302                 break;
 303         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 304                 alginfo_a->algtype =
 305                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 306                         OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384;
 307                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 308                 break;
 309         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 310                 alginfo_a->algtype =
 311                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 312                         OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512;
 313                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 314                 break;
 315         default:
 316                 DPAA_SEC_ERR("unsupported auth alg %u", ses->auth_alg);
 317         }
 318 }
 319
 320 static inline void
 321 caam_cipher_alg(dpaa_sec_session *ses, struct alginfo *alginfo_c)
 322 {
 323         switch (ses->cipher_alg) {
 324         case RTE_CRYPTO_CIPHER_NULL:
 325                 break;
 326         case RTE_CRYPTO_CIPHER_AES_CBC:
 327                 alginfo_c->algtype =
 328                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 329                         OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES;
 330                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 331                 break;
 332         case RTE_CRYPTO_CIPHER_3DES_CBC:
 333                 alginfo_c->algtype =
 334                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 335                         OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES;
 336                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 337                 break;
 338         case RTE_CRYPTO_CIPHER_AES_CTR:
 339                 alginfo_c->algtype =
 340                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 341                         OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES;
 342                 alginfo_c->algmode = OP_ALG_AAI_CTR;
 343                 break;
 344         default:
 345                 DPAA_SEC_ERR("unsupported cipher alg %d", ses->cipher_alg);
 346         }
 347 }
 348
 349 static inline void
 350 caam_aead_alg(dpaa_sec_session *ses, struct alginfo *alginfo)
 351 {
 352         switch (ses->aead_alg) {
 353         case RTE_CRYPTO_AEAD_AES_GCM:
 354                 alginfo->algtype = OP_ALG_ALGSEL_AES;
 355                 alginfo->algmode = OP_ALG_AAI_GCM;
 356                 break;
 357         default:
 358                 DPAA_SEC_ERR("unsupported AEAD alg %d", ses->aead_alg);
 359         }
 360 }
 361
 362
 363 /* prepare command block of the session */
 364 static int
 365 dpaa_sec_prep_cdb(dpaa_sec_session *ses)
 366 {
 367         struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0};
 368         int32_t shared_desc_len = 0;
 369         struct sec_cdb *cdb = &ses->cdb;
 370         int err;
 371 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 372         int swap = false;
 373 #else
 374         int swap = true;
 375 #endif
 376
 377         memset(cdb, 0, sizeof(struct sec_cdb));
 378
 379         if (is_cipher_only(ses)) {
 380                 caam_cipher_alg(ses, &alginfo_c);
 381                 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 382                         DPAA_SEC_ERR("not supported cipher alg");
 383                         return -ENOTSUP;
 384                 }
 385
 386                 alginfo_c.key = (size_t)ses->cipher_key.data;
 387                 alginfo_c.keylen = ses->cipher_key.length;
 388                 alginfo_c.key_enc_flags = 0;
 389                 alginfo_c.key_type = RTA_DATA_IMM;
 390
 391                 shared_desc_len = cnstr_shdsc_blkcipher(
 392                                                 cdb->sh_desc, true,
 393                                                 swap, &alginfo_c,
 394                                                 NULL,
 395                                                 ses->iv.length,
 396                                                 ses->dir);
 397         } else if (is_auth_only(ses)) {
 398                 caam_auth_alg(ses, &alginfo_a);
 399                 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 400                         DPAA_SEC_ERR("not supported auth alg");
 401                         return -ENOTSUP;
 402                 }
 403
 404                 alginfo_a.key = (size_t)ses->auth_key.data;
 405                 alginfo_a.keylen = ses->auth_key.length;
 406                 alginfo_a.key_enc_flags = 0;
 407                 alginfo_a.key_type = RTA_DATA_IMM;
 408
 409                 shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true,
 410                                                    swap, &alginfo_a,
 411                                                    !ses->dir,
 412                                                    ses->digest_length);
 413         } else if (is_aead(ses)) {
 414                 caam_aead_alg(ses, &alginfo);
 415                 if (alginfo.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 416                         DPAA_SEC_ERR("not supported aead alg");
 417                         return -ENOTSUP;
 418                 }
 419                 alginfo.key = (size_t)ses->aead_key.data;
 420                 alginfo.keylen = ses->aead_key.length;
 421                 alginfo.key_enc_flags = 0;
 422                 alginfo.key_type = RTA_DATA_IMM;
 423
 424                 if (ses->dir == DIR_ENC)
 425                         shared_desc_len = cnstr_shdsc_gcm_encap(
 426                                         cdb->sh_desc, true, swap,
 427                                         &alginfo,
 428                                         ses->iv.length,
 429                                         ses->digest_length);
 430                 else
 431                         shared_desc_len = cnstr_shdsc_gcm_decap(
 432                                         cdb->sh_desc, true, swap,
 433                                         &alginfo,
 434                                         ses->iv.length,
 435                                         ses->digest_length);
 436         } else {
 437                 caam_cipher_alg(ses, &alginfo_c);
 438                 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 439                         DPAA_SEC_ERR("not supported cipher alg");
 440                         return -ENOTSUP;
 441                 }
 442
 443                 alginfo_c.key = (size_t)ses->cipher_key.data;
 444                 alginfo_c.keylen = ses->cipher_key.length;
 445                 alginfo_c.key_enc_flags = 0;
 446                 alginfo_c.key_type = RTA_DATA_IMM;
 447
 448                 caam_auth_alg(ses, &alginfo_a);
 449                 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 450                         DPAA_SEC_ERR("not supported auth alg");
 451                         return -ENOTSUP;
 452                 }
 453
 454                 alginfo_a.key = (size_t)ses->auth_key.data;
 455                 alginfo_a.keylen = ses->auth_key.length;
 456                 alginfo_a.key_enc_flags = 0;
 457                 alginfo_a.key_type = RTA_DATA_IMM;
 458
 459                 cdb->sh_desc[0] = alginfo_c.keylen;
 460                 cdb->sh_desc[1] = alginfo_a.keylen;
 461                 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
 462                                        MIN_JOB_DESC_SIZE,
 463                                        (unsigned int *)cdb->sh_desc,
 464                                        &cdb->sh_desc[2], 2);
 465
 466                 if (err < 0) {
 467                         DPAA_SEC_ERR("Crypto: Incorrect key lengths");
 468                         return err;
 469                 }
 470                 if (cdb->sh_desc[2] & 1)
 471                         alginfo_c.key_type = RTA_DATA_IMM;
 472                 else {
 473                         alginfo_c.key = (size_t)dpaa_mem_vtop(
 474                                                 (void *)(size_t)alginfo_c.key);
 475                         alginfo_c.key_type = RTA_DATA_PTR;
 476                 }
 477                 if (cdb->sh_desc[2] & (1<<1))
 478                         alginfo_a.key_type = RTA_DATA_IMM;
 479                 else {
 480                         alginfo_a.key = (size_t)dpaa_mem_vtop(
 481                                                 (void *)(size_t)alginfo_a.key);
 482                         alginfo_a.key_type = RTA_DATA_PTR;
 483                 }
 484                 cdb->sh_desc[0] = 0;
 485                 cdb->sh_desc[1] = 0;
 486                 cdb->sh_desc[2] = 0;
 487                 if (is_proto_ipsec(ses)) {
 488                         if (ses->dir == DIR_ENC) {
 489                                 shared_desc_len = cnstr_shdsc_ipsec_new_encap(
 490                                                 cdb->sh_desc,
 491                                                 true, swap, &ses->encap_pdb,
 492                                                 (uint8_t *)&ses->ip4_hdr,
 493                                                 &alginfo_c, &alginfo_a);
 494                         } else if (ses->dir == DIR_DEC) {
 495                                 shared_desc_len = cnstr_shdsc_ipsec_new_decap(
 496                                                 cdb->sh_desc,
 497                                                 true, swap, &ses->decap_pdb,
 498                                                 &alginfo_c, &alginfo_a);
 499                         }
 500                 } else {
 501                         /* Auth_only_len is set as 0 here and it will be
 502                          * overwritten in fd for each packet.
 503                          */
 504                         shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
 505                                         true, swap, &alginfo_c, &alginfo_a,
 506                                         ses->iv.length, 0,
 507                                         ses->digest_length, ses->dir);
 508                 }
 509         }
 510
 511         if (shared_desc_len < 0) {
 512                 DPAA_SEC_ERR("error in preparing command block");
 513                 return shared_desc_len;
 514         }
 515
 516         cdb->sh_hdr.hi.field.idlen = shared_desc_len;
 517         cdb->sh_hdr.hi.word = rte_cpu_to_be_32(cdb->sh_hdr.hi.word);
 518         cdb->sh_hdr.lo.word = rte_cpu_to_be_32(cdb->sh_hdr.lo.word);
 519
 520         return 0;
 521 }
 522
 523 /* qp is lockless, should be accessed by only one thread */
 524 static int
 525 dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops)
 526 {
 527         struct qman_fq *fq;
 528         unsigned int pkts = 0;
 529         int num_rx_bufs, ret;
 530         struct qm_dqrr_entry *dq;
 531         uint32_t vdqcr_flags = 0;
 532
 533         fq = &qp->outq;
 534         /*
 535          * Until request for four buffers, we provide exact number of buffers.
 536          * Otherwise we do not set the QM_VDQCR_EXACT flag.
 537          * Not setting QM_VDQCR_EXACT flag can provide two more buffers than
 538          * requested, so we request two less in this case.
 539          */
 540         if (nb_ops < 4) {
 541                 vdqcr_flags = QM_VDQCR_EXACT;
 542                 num_rx_bufs = nb_ops;
 543         } else {
 544                 num_rx_bufs = nb_ops > DPAA_MAX_DEQUEUE_NUM_FRAMES ?
 545                         (DPAA_MAX_DEQUEUE_NUM_FRAMES - 2) : (nb_ops - 2);
 546         }
 547         ret = qman_set_vdq(fq, num_rx_bufs, vdqcr_flags);
 548         if (ret)
 549                 return 0;
 550
 551         do {
 552                 const struct qm_fd *fd;
 553                 struct dpaa_sec_job *job;
 554                 struct dpaa_sec_op_ctx *ctx;
 555                 struct rte_crypto_op *op;
 556
 557                 dq = qman_dequeue(fq);
 558                 if (!dq)
 559                         continue;
 560
 561                 fd = &dq->fd;
 562                 /* sg is embedded in an op ctx,
 563                  * sg[0] is for output
 564                  * sg[1] for input
 565                  */
 566                 job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
 567
 568                 ctx = container_of(job, struct dpaa_sec_op_ctx, job);
 569                 ctx->fd_status = fd->status;
 570                 op = ctx->op;
 571                 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 572                         struct qm_sg_entry *sg_out;
 573                         uint32_t len;
 574
 575                         sg_out = &job->sg[0];
 576                         hw_sg_to_cpu(sg_out);
 577                         len = sg_out->length;
 578                         op->sym->m_src->pkt_len = len;
 579                         op->sym->m_src->data_len = len;
 580                 }
 581                 if (!ctx->fd_status) {
 582                         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 583                 } else {
 584                         DPAA_SEC_DP_WARN("SEC return err:0x%x", ctx->fd_status);
 585                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 586                 }
 587                 ops[pkts++] = op;
 588
 589                 /* report op status to sym->op and then free the ctx memeory */
 590                 rte_mempool_put(ctx->ctx_pool, (void *)ctx);
 591
 592                 qman_dqrr_consume(fq, dq);
 593         } while (fq->flags & QMAN_FQ_STATE_VDQCR);
 594
 595         return pkts;
 596 }
 597
 598 static inline struct dpaa_sec_job *
 599 build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 600 {
 601         struct rte_crypto_sym_op *sym = op->sym;
 602         struct rte_mbuf *mbuf = sym->m_src;
 603         struct dpaa_sec_job *cf;
 604         struct dpaa_sec_op_ctx *ctx;
 605         struct qm_sg_entry *sg, *out_sg, *in_sg;
 606         phys_addr_t start_addr;
 607         uint8_t *old_digest, extra_segs;
 608
 609         if (is_decode(ses))
 610                 extra_segs = 3;
 611         else
 612                 extra_segs = 2;
 613
 614         if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) {
 615                 DPAA_SEC_DP_ERR("Auth: Max sec segs supported is %d",
 616                                 MAX_SG_ENTRIES);
 617                 return NULL;
 618         }
 619         ctx = dpaa_sec_alloc_ctx(ses);
 620         if (!ctx)
 621                 return NULL;
 622
 623         cf = &ctx->job;
 624         ctx->op = op;
 625         old_digest = ctx->digest;
 626
 627         /* output */
 628         out_sg = &cf->sg[0];
 629         qm_sg_entry_set64(out_sg, sym->auth.digest.phys_addr);
 630         out_sg->length = ses->digest_length;
 631         cpu_to_hw_sg(out_sg);
 632
 633         /* input */
 634         in_sg = &cf->sg[1];
 635         /* need to extend the input to a compound frame */
 636         in_sg->extension = 1;
 637         in_sg->final = 1;
 638         in_sg->length = sym->auth.data.length;
 639         qm_sg_entry_set64(in_sg, dpaa_mem_vtop(&cf->sg[2]));
 640
 641         /* 1st seg */
 642         sg = in_sg + 1;
 643         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 644         sg->length = mbuf->data_len - sym->auth.data.offset;
 645         sg->offset = sym->auth.data.offset;
 646
 647         /* Successive segs */
 648         mbuf = mbuf->next;
 649         while (mbuf) {
 650                 cpu_to_hw_sg(sg);
 651                 sg++;
 652                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 653                 sg->length = mbuf->data_len;
 654                 mbuf = mbuf->next;
 655         }
 656
 657         if (is_decode(ses)) {
 658                 /* Digest verification case */
 659                 cpu_to_hw_sg(sg);
 660                 sg++;
 661                 rte_memcpy(old_digest, sym->auth.digest.data,
 662                                 ses->digest_length);
 663                 start_addr = dpaa_mem_vtop(old_digest);
 664                 qm_sg_entry_set64(sg, start_addr);
 665                 sg->length = ses->digest_length;
 666                 in_sg->length += ses->digest_length;
 667         } else {
 668                 /* Digest calculation case */
 669                 sg->length -= ses->digest_length;
 670         }
 671         sg->final = 1;
 672         cpu_to_hw_sg(sg);
 673         cpu_to_hw_sg(in_sg);
 674
 675         return cf;
 676 }
 677
 678 /**
 679  * packet looks like:
 680  *              |<----data_len------->|
 681  *    |ip_header|ah_header|icv|payload|
 682  *              ^
 683  *              |
 684  *         mbuf->pkt.data
 685  */
 686 static inline struct dpaa_sec_job *
 687 build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
 688 {
 689         struct rte_crypto_sym_op *sym = op->sym;
 690         struct rte_mbuf *mbuf = sym->m_src;
 691         struct dpaa_sec_job *cf;
 692         struct dpaa_sec_op_ctx *ctx;
 693         struct qm_sg_entry *sg;
 694         rte_iova_t start_addr;
 695         uint8_t *old_digest;
 696
 697         ctx = dpaa_sec_alloc_ctx(ses);
 698         if (!ctx)
 699                 return NULL;
 700
 701         cf = &ctx->job;
 702         ctx->op = op;
 703         old_digest = ctx->digest;
 704
 705         start_addr = rte_pktmbuf_iova(mbuf);
 706         /* output */
 707         sg = &cf->sg[0];
 708         qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
 709         sg->length = ses->digest_length;
 710         cpu_to_hw_sg(sg);
 711
 712         /* input */
 713         sg = &cf->sg[1];
 714         if (is_decode(ses)) {
 715                 /* need to extend the input to a compound frame */
 716                 sg->extension = 1;
 717                 qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2]));
 718                 sg->length = sym->auth.data.length + ses->digest_length;
 719                 sg->final = 1;
 720                 cpu_to_hw_sg(sg);
 721
 722                 sg = &cf->sg[2];
 723                 /* hash result or digest, save digest first */
 724                 rte_memcpy(old_digest, sym->auth.digest.data,
 725                            ses->digest_length);
 726                 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset);
 727                 sg->length = sym->auth.data.length;
 728                 cpu_to_hw_sg(sg);
 729
 730                 /* let's check digest by hw */
 731                 start_addr = dpaa_mem_vtop(old_digest);
 732                 sg++;
 733                 qm_sg_entry_set64(sg, start_addr);
 734                 sg->length = ses->digest_length;
 735                 sg->final = 1;
 736                 cpu_to_hw_sg(sg);
 737         } else {
 738                 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset);
 739                 sg->length = sym->auth.data.length;
 740                 sg->final = 1;
 741                 cpu_to_hw_sg(sg);
 742         }
 743
 744         return cf;
 745 }
 746
 747 static inline struct dpaa_sec_job *
 748 build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 749 {
 750         struct rte_crypto_sym_op *sym = op->sym;
 751         struct dpaa_sec_job *cf;
 752         struct dpaa_sec_op_ctx *ctx;
 753         struct qm_sg_entry *sg, *out_sg, *in_sg;
 754         struct rte_mbuf *mbuf;
 755         uint8_t req_segs;
 756         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 757                         ses->iv.offset);
 758
 759         if (sym->m_dst) {
 760                 mbuf = sym->m_dst;
 761                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3;
 762         } else {
 763                 mbuf = sym->m_src;
 764                 req_segs = mbuf->nb_segs * 2 + 3;
 765         }
 766
 767         if (req_segs > MAX_SG_ENTRIES) {
 768                 DPAA_SEC_DP_ERR("Cipher: Max sec segs supported is %d",
 769                                 MAX_SG_ENTRIES);
 770                 return NULL;
 771         }
 772
 773         ctx = dpaa_sec_alloc_ctx(ses);
 774         if (!ctx)
 775                 return NULL;
 776
 777         cf = &ctx->job;
 778         ctx->op = op;
 779
 780         /* output */
 781         out_sg = &cf->sg[0];
 782         out_sg->extension = 1;
 783         out_sg->length = sym->cipher.data.length;
 784         qm_sg_entry_set64(out_sg, dpaa_mem_vtop(&cf->sg[2]));
 785         cpu_to_hw_sg(out_sg);
 786
 787         /* 1st seg */
 788         sg = &cf->sg[2];
 789         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 790         sg->length = mbuf->data_len - sym->cipher.data.offset;
 791         sg->offset = sym->cipher.data.offset;
 792
 793         /* Successive segs */
 794         mbuf = mbuf->next;
 795         while (mbuf) {
 796                 cpu_to_hw_sg(sg);
 797                 sg++;
 798                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 799                 sg->length = mbuf->data_len;
 800                 mbuf = mbuf->next;
 801         }
 802         sg->final = 1;
 803         cpu_to_hw_sg(sg);
 804
 805         /* input */
 806         mbuf = sym->m_src;
 807         in_sg = &cf->sg[1];
 808         in_sg->extension = 1;
 809         in_sg->final = 1;
 810         in_sg->length = sym->cipher.data.length + ses->iv.length;
 811
 812         sg++;
 813         qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
 814         cpu_to_hw_sg(in_sg);
 815
 816         /* IV */
 817         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
 818         sg->length = ses->iv.length;
 819         cpu_to_hw_sg(sg);
 820
 821         /* 1st seg */
 822         sg++;
 823         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 824         sg->length = mbuf->data_len - sym->cipher.data.offset;
 825         sg->offset = sym->cipher.data.offset;
 826
 827         /* Successive segs */
 828         mbuf = mbuf->next;
 829         while (mbuf) {
 830                 cpu_to_hw_sg(sg);
 831                 sg++;
 832                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 833                 sg->length = mbuf->data_len;
 834                 mbuf = mbuf->next;
 835         }
 836         sg->final = 1;
 837         cpu_to_hw_sg(sg);
 838
 839         return cf;
 840 }
 841
 842 static inline struct dpaa_sec_job *
 843 build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
 844 {
 845         struct rte_crypto_sym_op *sym = op->sym;
 846         struct dpaa_sec_job *cf;
 847         struct dpaa_sec_op_ctx *ctx;
 848         struct qm_sg_entry *sg;
 849         rte_iova_t src_start_addr, dst_start_addr;
 850         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 851                         ses->iv.offset);
 852
 853         ctx = dpaa_sec_alloc_ctx(ses);
 854         if (!ctx)
 855                 return NULL;
 856
 857         cf = &ctx->job;
 858         ctx->op = op;
 859
 860         src_start_addr = rte_pktmbuf_iova(sym->m_src);
 861
 862         if (sym->m_dst)
 863                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
 864         else
 865                 dst_start_addr = src_start_addr;
 866
 867         /* output */
 868         sg = &cf->sg[0];
 869         qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset);
 870         sg->length = sym->cipher.data.length + ses->iv.length;
 871         cpu_to_hw_sg(sg);
 872
 873         /* input */
 874         sg = &cf->sg[1];
 875
 876         /* need to extend the input to a compound frame */
 877         sg->extension = 1;
 878         sg->final = 1;
 879         sg->length = sym->cipher.data.length + ses->iv.length;
 880         qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2]));
 881         cpu_to_hw_sg(sg);
 882
 883         sg = &cf->sg[2];
 884         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
 885         sg->length = ses->iv.length;
 886         cpu_to_hw_sg(sg);
 887
 888         sg++;
 889         qm_sg_entry_set64(sg, src_start_addr + sym->cipher.data.offset);
 890         sg->length = sym->cipher.data.length;
 891         sg->final = 1;
 892         cpu_to_hw_sg(sg);
 893
 894         return cf;
 895 }
 896
 897 static inline struct dpaa_sec_job *
 898 build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 899 {
 900         struct rte_crypto_sym_op *sym = op->sym;
 901         struct dpaa_sec_job *cf;
 902         struct dpaa_sec_op_ctx *ctx;
 903         struct qm_sg_entry *sg, *out_sg, *in_sg;
 904         struct rte_mbuf *mbuf;
 905         uint8_t req_segs;
 906         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 907                         ses->iv.offset);
 908
 909         if (sym->m_dst) {
 910                 mbuf = sym->m_dst;
 911                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4;
 912         } else {
 913                 mbuf = sym->m_src;
 914                 req_segs = mbuf->nb_segs * 2 + 4;
 915         }
 916
 917         if (ses->auth_only_len)
 918                 req_segs++;
 919
 920         if (req_segs > MAX_SG_ENTRIES) {
 921                 DPAA_SEC_DP_ERR("AEAD: Max sec segs supported is %d",
 922                                 MAX_SG_ENTRIES);
 923                 return NULL;
 924         }
 925
 926         ctx = dpaa_sec_alloc_ctx(ses);
 927         if (!ctx)
 928                 return NULL;
 929
 930         cf = &ctx->job;
 931         ctx->op = op;
 932
 933         rte_prefetch0(cf->sg);
 934
 935         /* output */
 936         out_sg = &cf->sg[0];
 937         out_sg->extension = 1;
 938         if (is_encode(ses))
 939                 out_sg->length = sym->aead.data.length + ses->auth_only_len
 940                                                 + ses->digest_length;
 941         else
 942                 out_sg->length = sym->aead.data.length + ses->auth_only_len;
 943
 944         /* output sg entries */
 945         sg = &cf->sg[2];
 946         qm_sg_entry_set64(out_sg, dpaa_mem_vtop(sg));
 947         cpu_to_hw_sg(out_sg);
 948
 949         /* 1st seg */
 950         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 951         sg->length = mbuf->data_len - sym->aead.data.offset +
 952                                         ses->auth_only_len;
 953         sg->offset = sym->aead.data.offset - ses->auth_only_len;
 954
 955         /* Successive segs */
 956         mbuf = mbuf->next;
 957         while (mbuf) {
 958                 cpu_to_hw_sg(sg);
 959                 sg++;
 960                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 961                 sg->length = mbuf->data_len;
 962                 mbuf = mbuf->next;
 963         }
 964         sg->length -= ses->digest_length;
 965
 966         if (is_encode(ses)) {
 967                 cpu_to_hw_sg(sg);
 968                 /* set auth output */
 969                 sg++;
 970                 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr);
 971                 sg->length = ses->digest_length;
 972         }
 973         sg->final = 1;
 974         cpu_to_hw_sg(sg);
 975
 976         /* input */
 977         mbuf = sym->m_src;
 978         in_sg = &cf->sg[1];
 979         in_sg->extension = 1;
 980         in_sg->final = 1;
 981         if (is_encode(ses))
 982                 in_sg->length = ses->iv.length + sym->aead.data.length
 983                                                         + ses->auth_only_len;
 984         else
 985                 in_sg->length = ses->iv.length + sym->aead.data.length
 986                                 + ses->auth_only_len + ses->digest_length;
 987
 988         /* input sg entries */
 989         sg++;
 990         qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
 991         cpu_to_hw_sg(in_sg);
 992
 993         /* 1st seg IV */
 994         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
 995         sg->length = ses->iv.length;
 996         cpu_to_hw_sg(sg);
 997
 998         /* 2nd seg auth only */
 999         if (ses->auth_only_len) {
1000                 sg++;
1001                 qm_sg_entry_set64(sg, dpaa_mem_vtop(sym->aead.aad.data));
1002                 sg->length = ses->auth_only_len;
1003                 cpu_to_hw_sg(sg);
1004         }
1005
1006         /* 3rd seg */
1007         sg++;
1008         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1009         sg->length = mbuf->data_len - sym->aead.data.offset;
1010         sg->offset = sym->aead.data.offset;
1011
1012         /* Successive segs */
1013         mbuf = mbuf->next;
1014         while (mbuf) {
1015                 cpu_to_hw_sg(sg);
1016                 sg++;
1017                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1018                 sg->length = mbuf->data_len;
1019                 mbuf = mbuf->next;
1020         }
1021
1022         if (is_decode(ses)) {
1023                 cpu_to_hw_sg(sg);
1024                 sg++;
1025                 memcpy(ctx->digest, sym->aead.digest.data,
1026                         ses->digest_length);
1027                 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
1028                 sg->length = ses->digest_length;
1029         }
1030         sg->final = 1;
1031         cpu_to_hw_sg(sg);
1032
1033         return cf;
1034 }
1035
1036 static inline struct dpaa_sec_job *
1037 build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses)
1038 {
1039         struct rte_crypto_sym_op *sym = op->sym;
1040         struct dpaa_sec_job *cf;
1041         struct dpaa_sec_op_ctx *ctx;
1042         struct qm_sg_entry *sg;
1043         uint32_t length = 0;
1044         rte_iova_t src_start_addr, dst_start_addr;
1045         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1046                         ses->iv.offset);
1047
1048         src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
1049
1050         if (sym->m_dst)
1051                 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
1052         else
1053                 dst_start_addr = src_start_addr;
1054
1055         ctx = dpaa_sec_alloc_ctx(ses);
1056         if (!ctx)
1057                 return NULL;
1058
1059         cf = &ctx->job;
1060         ctx->op = op;
1061
1062         /* input */
1063         rte_prefetch0(cf->sg);
1064         sg = &cf->sg[2];
1065         qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg));
1066         if (is_encode(ses)) {
1067                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1068                 sg->length = ses->iv.length;
1069                 length += sg->length;
1070                 cpu_to_hw_sg(sg);
1071
1072                 sg++;
1073                 if (ses->auth_only_len) {
1074                         qm_sg_entry_set64(sg,
1075                                           dpaa_mem_vtop(sym->aead.aad.data));
1076                         sg->length = ses->auth_only_len;
1077                         length += sg->length;
1078                         cpu_to_hw_sg(sg);
1079                         sg++;
1080                 }
1081                 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
1082                 sg->length = sym->aead.data.length;
1083                 length += sg->length;
1084                 sg->final = 1;
1085                 cpu_to_hw_sg(sg);
1086         } else {
1087                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1088                 sg->length = ses->iv.length;
1089                 length += sg->length;
1090                 cpu_to_hw_sg(sg);
1091
1092                 sg++;
1093                 if (ses->auth_only_len) {
1094                         qm_sg_entry_set64(sg,
1095                                           dpaa_mem_vtop(sym->aead.aad.data));
1096                         sg->length = ses->auth_only_len;
1097                         length += sg->length;
1098                         cpu_to_hw_sg(sg);
1099                         sg++;
1100                 }
1101                 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
1102                 sg->length = sym->aead.data.length;
1103                 length += sg->length;
1104                 cpu_to_hw_sg(sg);
1105
1106                 memcpy(ctx->digest, sym->aead.digest.data,
1107                        ses->digest_length);
1108                 sg++;
1109
1110                 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
1111                 sg->length = ses->digest_length;
1112                 length += sg->length;
1113                 sg->final = 1;
1114                 cpu_to_hw_sg(sg);
1115         }
1116         /* input compound frame */
1117         cf->sg[1].length = length;
1118         cf->sg[1].extension = 1;
1119         cf->sg[1].final = 1;
1120         cpu_to_hw_sg(&cf->sg[1]);
1121
1122         /* output */
1123         sg++;
1124         qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg));
1125         qm_sg_entry_set64(sg,
1126                 dst_start_addr + sym->aead.data.offset - ses->auth_only_len);
1127         sg->length = sym->aead.data.length + ses->auth_only_len;
1128         length = sg->length;
1129         if (is_encode(ses)) {
1130                 cpu_to_hw_sg(sg);
1131                 /* set auth output */
1132                 sg++;
1133                 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr);
1134                 sg->length = ses->digest_length;
1135                 length += sg->length;
1136         }
1137         sg->final = 1;
1138         cpu_to_hw_sg(sg);
1139
1140         /* output compound frame */
1141         cf->sg[0].length = length;
1142         cf->sg[0].extension = 1;
1143         cpu_to_hw_sg(&cf->sg[0]);
1144
1145         return cf;
1146 }
1147
1148 static inline struct dpaa_sec_job *
1149 build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
1150 {
1151         struct rte_crypto_sym_op *sym = op->sym;
1152         struct dpaa_sec_job *cf;
1153         struct dpaa_sec_op_ctx *ctx;
1154         struct qm_sg_entry *sg, *out_sg, *in_sg;
1155         struct rte_mbuf *mbuf;
1156         uint8_t req_segs;
1157         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1158                         ses->iv.offset);
1159
1160         if (sym->m_dst) {
1161                 mbuf = sym->m_dst;
1162                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4;
1163         } else {
1164                 mbuf = sym->m_src;
1165                 req_segs = mbuf->nb_segs * 2 + 4;
1166         }
1167
1168         if (req_segs > MAX_SG_ENTRIES) {
1169                 DPAA_SEC_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
1170                                 MAX_SG_ENTRIES);
1171                 return NULL;
1172         }
1173
1174         ctx = dpaa_sec_alloc_ctx(ses);
1175         if (!ctx)
1176                 return NULL;
1177
1178         cf = &ctx->job;
1179         ctx->op = op;
1180
1181         rte_prefetch0(cf->sg);
1182
1183         /* output */
1184         out_sg = &cf->sg[0];
1185         out_sg->extension = 1;
1186         if (is_encode(ses))
1187                 out_sg->length = sym->auth.data.length + ses->digest_length;
1188         else
1189                 out_sg->length = sym->auth.data.length;
1190
1191         /* output sg entries */
1192         sg = &cf->sg[2];
1193         qm_sg_entry_set64(out_sg, dpaa_mem_vtop(sg));
1194         cpu_to_hw_sg(out_sg);
1195
1196         /* 1st seg */
1197         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1198         sg->length = mbuf->data_len - sym->auth.data.offset;
1199         sg->offset = sym->auth.data.offset;
1200
1201         /* Successive segs */
1202         mbuf = mbuf->next;
1203         while (mbuf) {
1204                 cpu_to_hw_sg(sg);
1205                 sg++;
1206                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1207                 sg->length = mbuf->data_len;
1208                 mbuf = mbuf->next;
1209         }
1210         sg->length -= ses->digest_length;
1211
1212         if (is_encode(ses)) {
1213                 cpu_to_hw_sg(sg);
1214                 /* set auth output */
1215                 sg++;
1216                 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
1217                 sg->length = ses->digest_length;
1218         }
1219         sg->final = 1;
1220         cpu_to_hw_sg(sg);
1221
1222         /* input */
1223         mbuf = sym->m_src;
1224         in_sg = &cf->sg[1];
1225         in_sg->extension = 1;
1226         in_sg->final = 1;
1227         if (is_encode(ses))
1228                 in_sg->length = ses->iv.length + sym->auth.data.length;
1229         else
1230                 in_sg->length = ses->iv.length + sym->auth.data.length
1231                                                 + ses->digest_length;
1232
1233         /* input sg entries */
1234         sg++;
1235         qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
1236         cpu_to_hw_sg(in_sg);
1237
1238         /* 1st seg IV */
1239         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1240         sg->length = ses->iv.length;
1241         cpu_to_hw_sg(sg);
1242
1243         /* 2nd seg */
1244         sg++;
1245         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1246         sg->length = mbuf->data_len - sym->auth.data.offset;
1247         sg->offset = sym->auth.data.offset;
1248
1249         /* Successive segs */
1250         mbuf = mbuf->next;
1251         while (mbuf) {
1252                 cpu_to_hw_sg(sg);
1253                 sg++;
1254                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1255                 sg->length = mbuf->data_len;
1256                 mbuf = mbuf->next;
1257         }
1258
1259         sg->length -= ses->digest_length;
1260         if (is_decode(ses)) {
1261                 cpu_to_hw_sg(sg);
1262                 sg++;
1263                 memcpy(ctx->digest, sym->auth.digest.data,
1264                         ses->digest_length);
1265                 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
1266                 sg->length = ses->digest_length;
1267         }
1268         sg->final = 1;
1269         cpu_to_hw_sg(sg);
1270
1271         return cf;
1272 }
1273
1274 static inline struct dpaa_sec_job *
1275 build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses)
1276 {
1277         struct rte_crypto_sym_op *sym = op->sym;
1278         struct dpaa_sec_job *cf;
1279         struct dpaa_sec_op_ctx *ctx;
1280         struct qm_sg_entry *sg;
1281         rte_iova_t src_start_addr, dst_start_addr;
1282         uint32_t length = 0;
1283         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1284                         ses->iv.offset);
1285
1286         src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
1287         if (sym->m_dst)
1288                 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
1289         else
1290                 dst_start_addr = src_start_addr;
1291
1292         ctx = dpaa_sec_alloc_ctx(ses);
1293         if (!ctx)
1294                 return NULL;
1295
1296         cf = &ctx->job;
1297         ctx->op = op;
1298
1299         /* input */
1300         rte_prefetch0(cf->sg);
1301         sg = &cf->sg[2];
1302         qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg));
1303         if (is_encode(ses)) {
1304                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1305                 sg->length = ses->iv.length;
1306                 length += sg->length;
1307                 cpu_to_hw_sg(sg);
1308
1309                 sg++;
1310                 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
1311                 sg->length = sym->auth.data.length;
1312                 length += sg->length;
1313                 sg->final = 1;
1314                 cpu_to_hw_sg(sg);
1315         } else {
1316                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1317                 sg->length = ses->iv.length;
1318                 length += sg->length;
1319                 cpu_to_hw_sg(sg);
1320
1321                 sg++;
1322
1323                 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
1324                 sg->length = sym->auth.data.length;
1325                 length += sg->length;
1326                 cpu_to_hw_sg(sg);
1327
1328                 memcpy(ctx->digest, sym->auth.digest.data,
1329                        ses->digest_length);
1330                 sg++;
1331
1332                 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
1333                 sg->length = ses->digest_length;
1334                 length += sg->length;
1335                 sg->final = 1;
1336                 cpu_to_hw_sg(sg);
1337         }
1338         /* input compound frame */
1339         cf->sg[1].length = length;
1340         cf->sg[1].extension = 1;
1341         cf->sg[1].final = 1;
1342         cpu_to_hw_sg(&cf->sg[1]);
1343
1344         /* output */
1345         sg++;
1346         qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg));
1347         qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset);
1348         sg->length = sym->cipher.data.length;
1349         length = sg->length;
1350         if (is_encode(ses)) {
1351                 cpu_to_hw_sg(sg);
1352                 /* set auth output */
1353                 sg++;
1354                 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
1355                 sg->length = ses->digest_length;
1356                 length += sg->length;
1357         }
1358         sg->final = 1;
1359         cpu_to_hw_sg(sg);
1360
1361         /* output compound frame */
1362         cf->sg[0].length = length;
1363         cf->sg[0].extension = 1;
1364         cpu_to_hw_sg(&cf->sg[0]);
1365
1366         return cf;
1367 }
1368
1369 static inline struct dpaa_sec_job *
1370 build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses)
1371 {
1372         struct rte_crypto_sym_op *sym = op->sym;
1373         struct dpaa_sec_job *cf;
1374         struct dpaa_sec_op_ctx *ctx;
1375         struct qm_sg_entry *sg;
1376         phys_addr_t src_start_addr, dst_start_addr;
1377
1378         ctx = dpaa_sec_alloc_ctx(ses);
1379         if (!ctx)
1380                 return NULL;
1381         cf = &ctx->job;
1382         ctx->op = op;
1383
1384         src_start_addr = rte_pktmbuf_mtophys(sym->m_src);
1385
1386         if (sym->m_dst)
1387                 dst_start_addr = rte_pktmbuf_mtophys(sym->m_dst);
1388         else
1389                 dst_start_addr = src_start_addr;
1390
1391         /* input */
1392         sg = &cf->sg[1];
1393         qm_sg_entry_set64(sg, src_start_addr);
1394         sg->length = sym->m_src->pkt_len;
1395         sg->final = 1;
1396         cpu_to_hw_sg(sg);
1397
1398         sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
1399         /* output */
1400         sg = &cf->sg[0];
1401         qm_sg_entry_set64(sg, dst_start_addr);
1402         sg->length = sym->m_src->buf_len - sym->m_src->data_off;
1403         cpu_to_hw_sg(sg);
1404
1405         return cf;
1406 }
1407
1408 static uint16_t
1409 dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1410                        uint16_t nb_ops)
1411 {
1412         /* Function to transmit the frames to given device and queuepair */
1413         uint32_t loop;
1414         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp;
1415         uint16_t num_tx = 0;
1416         struct qm_fd fds[DPAA_SEC_BURST], *fd;
1417         uint32_t frames_to_send;
1418         struct rte_crypto_op *op;
1419         struct dpaa_sec_job *cf;
1420         dpaa_sec_session *ses;
1421         uint32_t auth_only_len;
1422         struct qman_fq *inq[DPAA_SEC_BURST];
1423
1424         while (nb_ops) {
1425                 frames_to_send = (nb_ops > DPAA_SEC_BURST) ?
1426                                 DPAA_SEC_BURST : nb_ops;
1427                 for (loop = 0; loop < frames_to_send; loop++) {
1428                         op = *(ops++);
1429                         switch (op->sess_type) {
1430                         case RTE_CRYPTO_OP_WITH_SESSION:
1431                                 ses = (dpaa_sec_session *)
1432                                         get_sym_session_private_data(
1433                                                         op->sym->session,
1434                                                         cryptodev_driver_id);
1435                                 break;
1436                         case RTE_CRYPTO_OP_SECURITY_SESSION:
1437                                 ses = (dpaa_sec_session *)
1438                                         get_sec_session_private_data(
1439                                                         op->sym->sec_session);
1440                                 break;
1441                         default:
1442                                 DPAA_SEC_DP_ERR(
1443                                         "sessionless crypto op not supported");
1444                                 frames_to_send = loop;
1445                                 nb_ops = loop;
1446                                 goto send_pkts;
1447                         }
1448                         if (unlikely(!ses->qp || ses->qp != qp)) {
1449                                 DPAA_SEC_DP_ERR("sess->qp - %p qp %p",
1450                                              ses->qp, qp);
1451                                 if (dpaa_sec_attach_sess_q(qp, ses)) {
1452                                         frames_to_send = loop;
1453                                         nb_ops = loop;
1454                                         goto send_pkts;
1455                                 }
1456                         }
1457
1458                         auth_only_len = op->sym->auth.data.length -
1459                                                 op->sym->cipher.data.length;
1460                         if (rte_pktmbuf_is_contiguous(op->sym->m_src)) {
1461                                 if (is_auth_only(ses)) {
1462                                         cf = build_auth_only(op, ses);
1463                                 } else if (is_cipher_only(ses)) {
1464                                         cf = build_cipher_only(op, ses);
1465                                 } else if (is_aead(ses)) {
1466                                         cf = build_cipher_auth_gcm(op, ses);
1467                                         auth_only_len = ses->auth_only_len;
1468                                 } else if (is_auth_cipher(ses)) {
1469                                         cf = build_cipher_auth(op, ses);
1470                                 } else if (is_proto_ipsec(ses)) {
1471                                         cf = build_proto(op, ses);
1472                                 } else {
1473                                         DPAA_SEC_DP_ERR("not supported ops");
1474                                         frames_to_send = loop;
1475                                         nb_ops = loop;
1476                                         goto send_pkts;
1477                                 }
1478                         } else {
1479                                 if (is_auth_only(ses)) {
1480                                         cf = build_auth_only_sg(op, ses);
1481                                 } else if (is_cipher_only(ses)) {
1482                                         cf = build_cipher_only_sg(op, ses);
1483                                 } else if (is_aead(ses)) {
1484                                         cf = build_cipher_auth_gcm_sg(op, ses);
1485                                         auth_only_len = ses->auth_only_len;
1486                                 } else if (is_auth_cipher(ses)) {
1487                                         cf = build_cipher_auth_sg(op, ses);
1488                                 } else {
1489                                         DPAA_SEC_DP_ERR("not supported ops");
1490                                         frames_to_send = loop;
1491                                         nb_ops = loop;
1492                                         goto send_pkts;
1493                                 }
1494                         }
1495                         if (unlikely(!cf)) {
1496                                 frames_to_send = loop;
1497                                 nb_ops = loop;
1498                                 goto send_pkts;
1499                         }
1500
1501                         fd = &fds[loop];
1502                         inq[loop] = ses->inq;
1503                         fd->opaque_addr = 0;
1504                         fd->cmd = 0;
1505                         qm_fd_addr_set64(fd, dpaa_mem_vtop(cf->sg));
1506                         fd->_format1 = qm_fd_compound;
1507                         fd->length29 = 2 * sizeof(struct qm_sg_entry);
1508                         /* Auth_only_len is set as 0 in descriptor and it is
1509                          * overwritten here in the fd.cmd which will update
1510                          * the DPOVRD reg.
1511                          */
1512                         if (auth_only_len)
1513                                 fd->cmd = 0x80000000 | auth_only_len;
1514
1515                 }
1516 send_pkts:
1517                 loop = 0;
1518                 while (loop < frames_to_send) {
1519                         loop += qman_enqueue_multi_fq(&inq[loop], &fds[loop],
1520                                         frames_to_send - loop);
1521                 }
1522                 nb_ops -= frames_to_send;
1523                 num_tx += frames_to_send;
1524         }
1525
1526         dpaa_qp->tx_pkts += num_tx;
1527         dpaa_qp->tx_errs += nb_ops - num_tx;
1528
1529         return num_tx;
1530 }
1531
1532 static uint16_t
1533 dpaa_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops,
1534                        uint16_t nb_ops)
1535 {
1536         uint16_t num_rx;
1537         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp;
1538
1539         num_rx = dpaa_sec_deq(dpaa_qp, ops, nb_ops);
1540
1541         dpaa_qp->rx_pkts += num_rx;
1542         dpaa_qp->rx_errs += nb_ops - num_rx;
1543
1544         DPAA_SEC_DP_DEBUG("SEC Received %d Packets\n", num_rx);
1545
1546         return num_rx;
1547 }
1548
1549 /** Release queue pair */
1550 static int
1551 dpaa_sec_queue_pair_release(struct rte_cryptodev *dev,
1552                             uint16_t qp_id)
1553 {
1554         struct dpaa_sec_dev_private *internals;
1555         struct dpaa_sec_qp *qp = NULL;
1556
1557         PMD_INIT_FUNC_TRACE();
1558
1559         DPAA_SEC_DEBUG("dev =%p, queue =%d", dev, qp_id);
1560
1561         internals = dev->data->dev_private;
1562         if (qp_id >= internals->max_nb_queue_pairs) {
1563                 DPAA_SEC_ERR("Max supported qpid %d",
1564                              internals->max_nb_queue_pairs);
1565                 return -EINVAL;
1566         }
1567
1568         qp = &internals->qps[qp_id];
1569         qp->internals = NULL;
1570         dev->data->queue_pairs[qp_id] = NULL;
1571
1572         return 0;
1573 }
1574
1575 /** Setup a queue pair */
1576 static int
1577 dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
1578                 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
1579                 __rte_unused int socket_id,
1580                 __rte_unused struct rte_mempool *session_pool)
1581 {
1582         struct dpaa_sec_dev_private *internals;
1583         struct dpaa_sec_qp *qp = NULL;
1584
1585         DPAA_SEC_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);
1586
1587         internals = dev->data->dev_private;
1588         if (qp_id >= internals->max_nb_queue_pairs) {
1589                 DPAA_SEC_ERR("Max supported qpid %d",
1590                              internals->max_nb_queue_pairs);
1591                 return -EINVAL;
1592         }
1593
1594         qp = &internals->qps[qp_id];
1595         qp->internals = internals;
1596         dev->data->queue_pairs[qp_id] = qp;
1597
1598         return 0;
1599 }
1600
1601 /** Return the number of allocated queue pairs */
1602 static uint32_t
1603 dpaa_sec_queue_pair_count(struct rte_cryptodev *dev)
1604 {
1605         PMD_INIT_FUNC_TRACE();
1606
1607         return dev->data->nb_queue_pairs;
1608 }
1609
1610 /** Returns the size of session structure */
1611 static unsigned int
1612 dpaa_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
1613 {
1614         PMD_INIT_FUNC_TRACE();
1615
1616         return sizeof(dpaa_sec_session);
1617 }
1618
1619 static int
1620 dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused,
1621                      struct rte_crypto_sym_xform *xform,
1622                      dpaa_sec_session *session)
1623 {
1624         session->cipher_alg = xform->cipher.algo;
1625         session->iv.length = xform->cipher.iv.length;
1626         session->iv.offset = xform->cipher.iv.offset;
1627         session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
1628                                                RTE_CACHE_LINE_SIZE);
1629         if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
1630                 DPAA_SEC_ERR("No Memory for cipher key");
1631                 return -ENOMEM;
1632         }
1633         session->cipher_key.length = xform->cipher.key.length;
1634
1635         memcpy(session->cipher_key.data, xform->cipher.key.data,
1636                xform->cipher.key.length);
1637         session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
1638                         DIR_ENC : DIR_DEC;
1639
1640         return 0;
1641 }
1642
1643 static int
1644 dpaa_sec_auth_init(struct rte_cryptodev *dev __rte_unused,
1645                    struct rte_crypto_sym_xform *xform,
1646                    dpaa_sec_session *session)
1647 {
1648         session->auth_alg = xform->auth.algo;
1649         session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
1650                                              RTE_CACHE_LINE_SIZE);
1651         if (session->auth_key.data == NULL && xform->auth.key.length > 0) {
1652                 DPAA_SEC_ERR("No Memory for auth key");
1653                 return -ENOMEM;
1654         }
1655         session->auth_key.length = xform->auth.key.length;
1656         session->digest_length = xform->auth.digest_length;
1657
1658         memcpy(session->auth_key.data, xform->auth.key.data,
1659                xform->auth.key.length);
1660         session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
1661                         DIR_ENC : DIR_DEC;
1662
1663         return 0;
1664 }
1665
1666 static int
1667 dpaa_sec_aead_init(struct rte_cryptodev *dev __rte_unused,
1668                    struct rte_crypto_sym_xform *xform,
1669                    dpaa_sec_session *session)
1670 {
1671         session->aead_alg = xform->aead.algo;
1672         session->iv.length = xform->aead.iv.length;
1673         session->iv.offset = xform->aead.iv.offset;
1674         session->auth_only_len = xform->aead.aad_length;
1675         session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length,
1676                                              RTE_CACHE_LINE_SIZE);
1677         if (session->aead_key.data == NULL && xform->aead.key.length > 0) {
1678                 DPAA_SEC_ERR("No Memory for aead key\n");
1679                 return -ENOMEM;
1680         }
1681         session->aead_key.length = xform->aead.key.length;
1682         session->digest_length = xform->aead.digest_length;
1683
1684         memcpy(session->aead_key.data, xform->aead.key.data,
1685                xform->aead.key.length);
1686         session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
1687                         DIR_ENC : DIR_DEC;
1688
1689         return 0;
1690 }
1691
1692 static struct qman_fq *
1693 dpaa_sec_attach_rxq(struct dpaa_sec_dev_private *qi)
1694 {
1695         unsigned int i;
1696
1697         for (i = 0; i < qi->max_nb_sessions; i++) {
1698                 if (qi->inq_attach[i] == 0) {
1699                         qi->inq_attach[i] = 1;
1700                         return &qi->inq[i];
1701                 }
1702         }
1703         DPAA_SEC_WARN("All ses session in use %x", qi->max_nb_sessions);
1704
1705         return NULL;
1706 }
1707
1708 static int
1709 dpaa_sec_detach_rxq(struct dpaa_sec_dev_private *qi, struct qman_fq *fq)
1710 {
1711         unsigned int i;
1712
1713         for (i = 0; i < qi->max_nb_sessions; i++) {
1714                 if (&qi->inq[i] == fq) {
1715                         qman_retire_fq(fq, NULL);
1716                         qman_oos_fq(fq);
1717                         qi->inq_attach[i] = 0;
1718                         return 0;
1719                 }
1720         }
1721         return -1;
1722 }
1723
1724 static int
1725 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess)
1726 {
1727         int ret;
1728
1729         sess->qp = qp;
1730         ret = dpaa_sec_prep_cdb(sess);
1731         if (ret) {
1732                 DPAA_SEC_ERR("Unable to prepare sec cdb");
1733                 return -1;
1734         }
1735         if (unlikely(!RTE_PER_LCORE(dpaa_io))) {
1736                 ret = rte_dpaa_portal_init((void *)0);
1737                 if (ret) {
1738                         DPAA_SEC_ERR("Failure in affining portal");
1739                         return ret;
1740                 }
1741         }
1742         ret = dpaa_sec_init_rx(sess->inq, dpaa_mem_vtop(&sess->cdb),
1743                                qman_fq_fqid(&qp->outq));
1744         if (ret)
1745                 DPAA_SEC_ERR("Unable to init sec queue");
1746
1747         return ret;
1748 }
1749
1750 static int
1751 dpaa_sec_set_session_parameters(struct rte_cryptodev *dev,
1752                             struct rte_crypto_sym_xform *xform, void *sess)
1753 {
1754         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
1755         dpaa_sec_session *session = sess;
1756
1757         PMD_INIT_FUNC_TRACE();
1758
1759         if (unlikely(sess == NULL)) {
1760                 DPAA_SEC_ERR("invalid session struct");
1761                 return -EINVAL;
1762         }
1763
1764         /* Default IV length = 0 */
1765         session->iv.length = 0;
1766
1767         /* Cipher Only */
1768         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
1769                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1770                 dpaa_sec_cipher_init(dev, xform, session);
1771
1772         /* Authentication Only */
1773         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1774                    xform->next == NULL) {
1775                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
1776                 dpaa_sec_auth_init(dev, xform, session);
1777
1778         /* Cipher then Authenticate */
1779         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1780                    xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
1781                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1782                         dpaa_sec_cipher_init(dev, xform, session);
1783                         dpaa_sec_auth_init(dev, xform->next, session);
1784                 } else {
1785                         DPAA_SEC_ERR("Not supported: Auth then Cipher");
1786                         return -EINVAL;
1787                 }
1788
1789         /* Authenticate then Cipher */
1790         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1791                    xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
1792                 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
1793                         dpaa_sec_auth_init(dev, xform, session);
1794                         dpaa_sec_cipher_init(dev, xform->next, session);
1795                 } else {
1796                         DPAA_SEC_ERR("Not supported: Auth then Cipher");
1797                         return -EINVAL;
1798                 }
1799
1800         /* AEAD operation for AES-GCM kind of Algorithms */
1801         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
1802                    xform->next == NULL) {
1803                 dpaa_sec_aead_init(dev, xform, session);
1804
1805         } else {
1806                 DPAA_SEC_ERR("Invalid crypto type");
1807                 return -EINVAL;
1808         }
1809         session->ctx_pool = internals->ctx_pool;
1810         session->inq = dpaa_sec_attach_rxq(internals);
1811         if (session->inq == NULL) {
1812                 DPAA_SEC_ERR("unable to attach sec queue");
1813                 goto err1;
1814         }
1815
1816         return 0;
1817
1818 err1:
1819         rte_free(session->cipher_key.data);
1820         rte_free(session->auth_key.data);
1821         memset(session, 0, sizeof(dpaa_sec_session));
1822
1823         return -EINVAL;
1824 }
1825
1826 static int
1827 dpaa_sec_sym_session_configure(struct rte_cryptodev *dev,
1828                 struct rte_crypto_sym_xform *xform,
1829                 struct rte_cryptodev_sym_session *sess,
1830                 struct rte_mempool *mempool)
1831 {
1832         void *sess_private_data;
1833         int ret;
1834
1835         PMD_INIT_FUNC_TRACE();
1836
1837         if (rte_mempool_get(mempool, &sess_private_data)) {
1838                 DPAA_SEC_ERR("Couldn't get object from session mempool");
1839                 return -ENOMEM;
1840         }
1841
1842         ret = dpaa_sec_set_session_parameters(dev, xform, sess_private_data);
1843         if (ret != 0) {
1844                 DPAA_SEC_ERR("failed to configure session parameters");
1845
1846                 /* Return session to mempool */
1847                 rte_mempool_put(mempool, sess_private_data);
1848                 return ret;
1849         }
1850
1851         set_sym_session_private_data(sess, dev->driver_id,
1852                         sess_private_data);
1853
1854
1855         return 0;
1856 }
1857
1858 /** Clear the memory of session so it doesn't leave key material behind */
1859 static void
1860 dpaa_sec_sym_session_clear(struct rte_cryptodev *dev,
1861                 struct rte_cryptodev_sym_session *sess)
1862 {
1863         struct dpaa_sec_dev_private *qi = dev->data->dev_private;
1864         uint8_t index = dev->driver_id;
1865         void *sess_priv = get_sym_session_private_data(sess, index);
1866
1867         PMD_INIT_FUNC_TRACE();
1868
1869         dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
1870
1871         if (sess_priv) {
1872                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1873
1874                 if (s->inq)
1875                         dpaa_sec_detach_rxq(qi, s->inq);
1876                 rte_free(s->cipher_key.data);
1877                 rte_free(s->auth_key.data);
1878                 memset(s, 0, sizeof(dpaa_sec_session));
1879                 set_sym_session_private_data(sess, index, NULL);
1880                 rte_mempool_put(sess_mp, sess_priv);
1881         }
1882 }
1883
1884 static int
1885 dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
1886                            struct rte_security_session_conf *conf,
1887                            void *sess)
1888 {
1889         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
1890         struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
1891         struct rte_crypto_auth_xform *auth_xform;
1892         struct rte_crypto_cipher_xform *cipher_xform;
1893         dpaa_sec_session *session = (dpaa_sec_session *)sess;
1894
1895         PMD_INIT_FUNC_TRACE();
1896
1897         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
1898                 cipher_xform = &conf->crypto_xform->cipher;
1899                 auth_xform = &conf->crypto_xform->next->auth;
1900         } else {
1901                 auth_xform = &conf->crypto_xform->auth;
1902                 cipher_xform = &conf->crypto_xform->next->cipher;
1903         }
1904         session->proto_alg = conf->protocol;
1905         session->cipher_key.data = rte_zmalloc(NULL,
1906                                                cipher_xform->key.length,
1907                                                RTE_CACHE_LINE_SIZE);
1908         if (session->cipher_key.data == NULL &&
1909                         cipher_xform->key.length > 0) {
1910                 DPAA_SEC_ERR("No Memory for cipher key");
1911                 return -ENOMEM;
1912         }
1913
1914         session->cipher_key.length = cipher_xform->key.length;
1915         session->auth_key.data = rte_zmalloc(NULL,
1916                                         auth_xform->key.length,
1917                                         RTE_CACHE_LINE_SIZE);
1918         if (session->auth_key.data == NULL &&
1919                         auth_xform->key.length > 0) {
1920                 DPAA_SEC_ERR("No Memory for auth key");
1921                 rte_free(session->cipher_key.data);
1922                 return -ENOMEM;
1923         }
1924         session->auth_key.length = auth_xform->key.length;
1925         memcpy(session->cipher_key.data, cipher_xform->key.data,
1926                         cipher_xform->key.length);
1927         memcpy(session->auth_key.data, auth_xform->key.data,
1928                         auth_xform->key.length);
1929
1930         switch (auth_xform->algo) {
1931         case RTE_CRYPTO_AUTH_SHA1_HMAC:
1932                 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
1933                 break;
1934         case RTE_CRYPTO_AUTH_MD5_HMAC:
1935                 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
1936                 break;
1937         case RTE_CRYPTO_AUTH_SHA256_HMAC:
1938                 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
1939                 break;
1940         case RTE_CRYPTO_AUTH_SHA384_HMAC:
1941                 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
1942                 break;
1943         case RTE_CRYPTO_AUTH_SHA512_HMAC:
1944                 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
1945                 break;
1946         case RTE_CRYPTO_AUTH_AES_CMAC:
1947                 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
1948                 break;
1949         case RTE_CRYPTO_AUTH_NULL:
1950                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1951                 break;
1952         case RTE_CRYPTO_AUTH_SHA224_HMAC:
1953         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
1954         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
1955         case RTE_CRYPTO_AUTH_SHA1:
1956         case RTE_CRYPTO_AUTH_SHA256:
1957         case RTE_CRYPTO_AUTH_SHA512:
1958         case RTE_CRYPTO_AUTH_SHA224:
1959         case RTE_CRYPTO_AUTH_SHA384:
1960         case RTE_CRYPTO_AUTH_MD5:
1961         case RTE_CRYPTO_AUTH_AES_GMAC:
1962         case RTE_CRYPTO_AUTH_KASUMI_F9:
1963         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
1964         case RTE_CRYPTO_AUTH_ZUC_EIA3:
1965                 DPAA_SEC_ERR("Crypto: Unsupported auth alg %u",
1966                         auth_xform->algo);
1967                 goto out;
1968         default:
1969                 DPAA_SEC_ERR("Crypto: Undefined Auth specified %u",
1970                         auth_xform->algo);
1971                 goto out;
1972         }
1973
1974         switch (cipher_xform->algo) {
1975         case RTE_CRYPTO_CIPHER_AES_CBC:
1976                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
1977                 break;
1978         case RTE_CRYPTO_CIPHER_3DES_CBC:
1979                 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
1980                 break;
1981         case RTE_CRYPTO_CIPHER_AES_CTR:
1982                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
1983                 break;
1984         case RTE_CRYPTO_CIPHER_NULL:
1985         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
1986         case RTE_CRYPTO_CIPHER_3DES_ECB:
1987         case RTE_CRYPTO_CIPHER_AES_ECB:
1988         case RTE_CRYPTO_CIPHER_KASUMI_F8:
1989                 DPAA_SEC_ERR("Crypto: Unsupported Cipher alg %u",
1990                         cipher_xform->algo);
1991                 goto out;
1992         default:
1993                 DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u",
1994                         cipher_xform->algo);
1995                 goto out;
1996         }
1997
1998         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
1999                 memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) +
2000                                 sizeof(session->ip4_hdr));
2001                 session->ip4_hdr.ip_v = IPVERSION;
2002                 session->ip4_hdr.ip_hl = 5;
2003                 session->ip4_hdr.ip_len = rte_cpu_to_be_16(
2004                                                 sizeof(session->ip4_hdr));
2005                 session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
2006                 session->ip4_hdr.ip_id = 0;
2007                 session->ip4_hdr.ip_off = 0;
2008                 session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
2009                 session->ip4_hdr.ip_p = (ipsec_xform->proto ==
2010                                 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP
2011                                 : IPPROTO_AH;
2012                 session->ip4_hdr.ip_sum = 0;
2013                 session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
2014                 session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
2015                 session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
2016                                                 (void *)&session->ip4_hdr,
2017                                                 sizeof(struct ip));
2018
2019                 session->encap_pdb.options =
2020                         (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
2021                         PDBOPTS_ESP_OIHI_PDB_INL |
2022                         PDBOPTS_ESP_IVSRC |
2023                         PDBHMO_ESP_ENCAP_DTTL;
2024                 session->encap_pdb.spi = ipsec_xform->spi;
2025                 session->encap_pdb.ip_hdr_len = sizeof(struct ip);
2026
2027                 session->dir = DIR_ENC;
2028         } else if (ipsec_xform->direction ==
2029                         RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
2030                 memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
2031                 session->decap_pdb.options = sizeof(struct ip) << 16;
2032                 session->dir = DIR_DEC;
2033         } else
2034                 goto out;
2035         session->ctx_pool = internals->ctx_pool;
2036         session->inq = dpaa_sec_attach_rxq(internals);
2037         if (session->inq == NULL) {
2038                 DPAA_SEC_ERR("unable to attach sec queue");
2039                 goto out;
2040         }
2041
2042
2043         return 0;
2044 out:
2045         rte_free(session->auth_key.data);
2046         rte_free(session->cipher_key.data);
2047         memset(session, 0, sizeof(dpaa_sec_session));
2048         return -1;
2049 }
2050
2051 static int
2052 dpaa_sec_security_session_create(void *dev,
2053                                  struct rte_security_session_conf *conf,
2054                                  struct rte_security_session *sess,
2055                                  struct rte_mempool *mempool)
2056 {
2057         void *sess_private_data;
2058         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
2059         int ret;
2060
2061         if (rte_mempool_get(mempool, &sess_private_data)) {
2062                 DPAA_SEC_ERR("Couldn't get object from session mempool");
2063                 return -ENOMEM;
2064         }
2065
2066         switch (conf->protocol) {
2067         case RTE_SECURITY_PROTOCOL_IPSEC:
2068                 ret = dpaa_sec_set_ipsec_session(cdev, conf,
2069                                 sess_private_data);
2070                 break;
2071         case RTE_SECURITY_PROTOCOL_MACSEC:
2072                 return -ENOTSUP;
2073         default:
2074                 return -EINVAL;
2075         }
2076         if (ret != 0) {
2077                 DPAA_SEC_ERR("failed to configure session parameters");
2078                 /* Return session to mempool */
2079                 rte_mempool_put(mempool, sess_private_data);
2080                 return ret;
2081         }
2082
2083         set_sec_session_private_data(sess, sess_private_data);
2084
2085         return ret;
2086 }
2087
2088 /** Clear the memory of session so it doesn't leave key material behind */
2089 static int
2090 dpaa_sec_security_session_destroy(void *dev __rte_unused,
2091                 struct rte_security_session *sess)
2092 {
2093         PMD_INIT_FUNC_TRACE();
2094         void *sess_priv = get_sec_session_private_data(sess);
2095
2096         dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
2097
2098         if (sess_priv) {
2099                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
2100
2101                 rte_free(s->cipher_key.data);
2102                 rte_free(s->auth_key.data);
2103                 memset(sess, 0, sizeof(dpaa_sec_session));
2104                 set_sec_session_private_data(sess, NULL);
2105                 rte_mempool_put(sess_mp, sess_priv);
2106         }
2107         return 0;
2108 }
2109
2110
2111 static int
2112 dpaa_sec_dev_configure(struct rte_cryptodev *dev,
2113                        struct rte_cryptodev_config *config __rte_unused)
2114 {
2115
2116         char str[20];
2117         struct dpaa_sec_dev_private *internals;
2118
2119         PMD_INIT_FUNC_TRACE();
2120
2121         internals = dev->data->dev_private;
2122         sprintf(str, "ctx_pool_%d", dev->data->dev_id);
2123         if (!internals->ctx_pool) {
2124                 internals->ctx_pool = rte_mempool_create((const char *)str,
2125                                                         CTX_POOL_NUM_BUFS,
2126                                                         CTX_POOL_BUF_SIZE,
2127                                                         CTX_POOL_CACHE_SIZE, 0,
2128                                                         NULL, NULL, NULL, NULL,
2129                                                         SOCKET_ID_ANY, 0);
2130                 if (!internals->ctx_pool) {
2131                         DPAA_SEC_ERR("%s create failed\n", str);
2132                         return -ENOMEM;
2133                 }
2134         } else
2135                 DPAA_SEC_INFO("mempool already created for dev_id : %d",
2136                                 dev->data->dev_id);
2137
2138         return 0;
2139 }
2140
2141 static int
2142 dpaa_sec_dev_start(struct rte_cryptodev *dev __rte_unused)
2143 {
2144         PMD_INIT_FUNC_TRACE();
2145         return 0;
2146 }
2147
2148 static void
2149 dpaa_sec_dev_stop(struct rte_cryptodev *dev __rte_unused)
2150 {
2151         PMD_INIT_FUNC_TRACE();
2152 }
2153
2154 static int
2155 dpaa_sec_dev_close(struct rte_cryptodev *dev)
2156 {
2157         struct dpaa_sec_dev_private *internals;
2158
2159         PMD_INIT_FUNC_TRACE();
2160
2161         if (dev == NULL)
2162                 return -ENOMEM;
2163
2164         internals = dev->data->dev_private;
2165         rte_mempool_free(internals->ctx_pool);
2166         internals->ctx_pool = NULL;
2167
2168         return 0;
2169 }
2170
2171 static void
2172 dpaa_sec_dev_infos_get(struct rte_cryptodev *dev,
2173                        struct rte_cryptodev_info *info)
2174 {
2175         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
2176
2177         PMD_INIT_FUNC_TRACE();
2178         if (info != NULL) {
2179                 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
2180                 info->feature_flags = dev->feature_flags;
2181                 info->capabilities = dpaa_sec_capabilities;
2182                 info->sym.max_nb_sessions = internals->max_nb_sessions;
2183                 info->driver_id = cryptodev_driver_id;
2184         }
2185 }
2186
2187 static struct rte_cryptodev_ops crypto_ops = {
2188         .dev_configure        = dpaa_sec_dev_configure,
2189         .dev_start            = dpaa_sec_dev_start,
2190         .dev_stop             = dpaa_sec_dev_stop,
2191         .dev_close            = dpaa_sec_dev_close,
2192         .dev_infos_get        = dpaa_sec_dev_infos_get,
2193         .queue_pair_setup     = dpaa_sec_queue_pair_setup,
2194         .queue_pair_release   = dpaa_sec_queue_pair_release,
2195         .queue_pair_count     = dpaa_sec_queue_pair_count,
2196         .sym_session_get_size     = dpaa_sec_sym_session_get_size,
2197         .sym_session_configure    = dpaa_sec_sym_session_configure,
2198         .sym_session_clear        = dpaa_sec_sym_session_clear
2199 };
2200
2201 static const struct rte_security_capability *
2202 dpaa_sec_capabilities_get(void *device __rte_unused)
2203 {
2204         return dpaa_sec_security_cap;
2205 }
2206
2207 struct rte_security_ops dpaa_sec_security_ops = {
2208         .session_create = dpaa_sec_security_session_create,
2209         .session_update = NULL,
2210         .session_stats_get = NULL,
2211         .session_destroy = dpaa_sec_security_session_destroy,
2212         .set_pkt_metadata = NULL,
2213         .capabilities_get = dpaa_sec_capabilities_get
2214 };
2215
2216 static int
2217 dpaa_sec_uninit(struct rte_cryptodev *dev)
2218 {
2219         struct dpaa_sec_dev_private *internals;
2220
2221         if (dev == NULL)
2222                 return -ENODEV;
2223
2224         internals = dev->data->dev_private;
2225         rte_free(dev->security_ctx);
2226
2227         /* In case close has been called, internals->ctx_pool would be NULL */
2228         rte_mempool_free(internals->ctx_pool);
2229         rte_free(internals);
2230
2231         DPAA_SEC_INFO("Closing DPAA_SEC device %s on numa socket %u",
2232                       dev->data->name, rte_socket_id());
2233
2234         return 0;
2235 }
2236
2237 static int
2238 dpaa_sec_dev_init(struct rte_cryptodev *cryptodev)
2239 {
2240         struct dpaa_sec_dev_private *internals;
2241         struct rte_security_ctx *security_instance;
2242         struct dpaa_sec_qp *qp;
2243         uint32_t i, flags;
2244         int ret;
2245
2246         PMD_INIT_FUNC_TRACE();
2247
2248         cryptodev->driver_id = cryptodev_driver_id;
2249         cryptodev->dev_ops = &crypto_ops;
2250
2251         cryptodev->enqueue_burst = dpaa_sec_enqueue_burst;
2252         cryptodev->dequeue_burst = dpaa_sec_dequeue_burst;
2253         cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2254                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
2255                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2256                         RTE_CRYPTODEV_FF_SECURITY |
2257                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
2258                         RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
2259                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
2260                         RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
2261                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
2262
2263         internals = cryptodev->data->dev_private;
2264         internals->max_nb_queue_pairs = RTE_DPAA_MAX_NB_SEC_QPS;
2265         internals->max_nb_sessions = RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS;
2266
2267         /*
2268          * For secondary processes, we don't initialise any further as primary
2269          * has already done this work. Only check we don't need a different
2270          * RX function
2271          */
2272         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2273                 DPAA_SEC_WARN("Device already init by primary process");
2274                 return 0;
2275         }
2276
2277         /* Initialize security_ctx only for primary process*/
2278         security_instance = rte_malloc("rte_security_instances_ops",
2279                                 sizeof(struct rte_security_ctx), 0);
2280         if (security_instance == NULL)
2281                 return -ENOMEM;
2282         security_instance->device = (void *)cryptodev;
2283         security_instance->ops = &dpaa_sec_security_ops;
2284         security_instance->sess_cnt = 0;
2285         cryptodev->security_ctx = security_instance;
2286
2287         for (i = 0; i < internals->max_nb_queue_pairs; i++) {
2288                 /* init qman fq for queue pair */
2289                 qp = &internals->qps[i];
2290                 ret = dpaa_sec_init_tx(&qp->outq);
2291                 if (ret) {
2292                         DPAA_SEC_ERR("config tx of queue pair  %d", i);
2293                         goto init_error;
2294                 }
2295         }
2296
2297         flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID |
2298                 QMAN_FQ_FLAG_TO_DCPORTAL;
2299         for (i = 0; i < internals->max_nb_sessions; i++) {
2300                 /* create rx qman fq for sessions*/
2301                 ret = qman_create_fq(0, flags, &internals->inq[i]);
2302                 if (unlikely(ret != 0)) {
2303                         DPAA_SEC_ERR("sec qman_create_fq failed");
2304                         goto init_error;
2305                 }
2306         }
2307
2308         RTE_LOG(INFO, PMD, "%s cryptodev init\n", cryptodev->data->name);
2309         return 0;
2310
2311 init_error:
2312         DPAA_SEC_ERR("driver %s: create failed\n", cryptodev->data->name);
2313
2314         dpaa_sec_uninit(cryptodev);
2315         return -EFAULT;
2316 }
2317
2318 static int
2319 cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv,
2320                                 struct rte_dpaa_device *dpaa_dev)
2321 {
2322         struct rte_cryptodev *cryptodev;
2323         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
2324
2325         int retval;
2326
2327         sprintf(cryptodev_name, "dpaa_sec-%d", dpaa_dev->id.dev_id);
2328
2329         cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id());
2330         if (cryptodev == NULL)
2331                 return -ENOMEM;
2332
2333         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
2334                 cryptodev->data->dev_private = rte_zmalloc_socket(
2335                                         "cryptodev private structure",
2336                                         sizeof(struct dpaa_sec_dev_private),
2337                                         RTE_CACHE_LINE_SIZE,
2338                                         rte_socket_id());
2339
2340                 if (cryptodev->data->dev_private == NULL)
2341                         rte_panic("Cannot allocate memzone for private "
2342                                         "device data");
2343         }
2344
2345         dpaa_dev->crypto_dev = cryptodev;
2346         cryptodev->device = &dpaa_dev->device;
2347         cryptodev->device->driver = &dpaa_drv->driver;
2348
2349         /* init user callbacks */
2350         TAILQ_INIT(&(cryptodev->link_intr_cbs));
2351
2352         /* if sec device version is not configured */
2353         if (!rta_get_sec_era()) {
2354                 const struct device_node *caam_node;
2355
2356                 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
2357                         const uint32_t *prop = of_get_property(caam_node,
2358                                         "fsl,sec-era",
2359                                         NULL);
2360                         if (prop) {
2361                                 rta_set_sec_era(
2362                                         INTL_SEC_ERA(rte_cpu_to_be_32(*prop)));
2363                                 break;
2364                         }
2365                 }
2366         }
2367
2368         /* Invoke PMD device initialization function */
2369         retval = dpaa_sec_dev_init(cryptodev);
2370         if (retval == 0)
2371                 return 0;
2372
2373         /* In case of error, cleanup is done */
2374         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
2375                 rte_free(cryptodev->data->dev_private);
2376
2377         rte_cryptodev_pmd_release_device(cryptodev);
2378
2379         return -ENXIO;
2380 }
2381
2382 static int
2383 cryptodev_dpaa_sec_remove(struct rte_dpaa_device *dpaa_dev)
2384 {
2385         struct rte_cryptodev *cryptodev;
2386         int ret;
2387
2388         cryptodev = dpaa_dev->crypto_dev;
2389         if (cryptodev == NULL)
2390                 return -ENODEV;
2391
2392         ret = dpaa_sec_uninit(cryptodev);
2393         if (ret)
2394                 return ret;
2395
2396         return rte_cryptodev_pmd_destroy(cryptodev);
2397 }
2398
2399 static struct rte_dpaa_driver rte_dpaa_sec_driver = {
2400         .drv_type = FSL_DPAA_CRYPTO,
2401         .driver = {
2402                 .name = "DPAA SEC PMD"
2403         },
2404         .probe = cryptodev_dpaa_sec_probe,
2405         .remove = cryptodev_dpaa_sec_remove,
2406 };
2407
2408 static struct cryptodev_driver dpaa_sec_crypto_drv;
2409
2410 RTE_PMD_REGISTER_DPAA(CRYPTODEV_NAME_DPAA_SEC_PMD, rte_dpaa_sec_driver);
2411 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa_sec_crypto_drv, rte_dpaa_sec_driver.driver,
2412                 cryptodev_driver_id);
2413
2414 RTE_INIT(dpaa_sec_init_log)
2415 {
2416         dpaa_logtype_sec = rte_log_register("pmd.crypto.dpaa");
2417         if (dpaa_logtype_sec >= 0)
2418                 rte_log_set_level(dpaa_logtype_sec, RTE_LOG_NOTICE);
2419 }