drivers/crypto/dpaa_sec/dpaa_sec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2017-2018 NXP
   5  *
   6  */
   7
   8 #include <fcntl.h>
   9 #include <unistd.h>
  10 #include <sched.h>
  11 #include <net/if.h>
  12
  13 #include <rte_byteorder.h>
  14 #include <rte_common.h>
  15 #include <rte_cryptodev_pmd.h>
  16 #include <rte_crypto.h>
  17 #include <rte_cryptodev.h>
  18 #include <rte_security_driver.h>
  19 #include <rte_cycles.h>
  20 #include <rte_dev.h>
  21 #include <rte_kvargs.h>
  22 #include <rte_malloc.h>
  23 #include <rte_mbuf.h>
  24 #include <rte_memcpy.h>
  25 #include <rte_string_fns.h>
  26 #include <rte_spinlock.h>
  27
  28 #include <fsl_usd.h>
  29 #include <fsl_qman.h>
  30 #include <of.h>
  31
  32 /* RTA header files */
  33 #include <hw/desc/common.h>
  34 #include <hw/desc/algo.h>
  35 #include <hw/desc/ipsec.h>
  36
  37 #include <rte_dpaa_bus.h>
  38 #include <dpaa_sec.h>
  39 #include <dpaa_sec_log.h>
  40
  41 enum rta_sec_era rta_sec_era;
  42
  43 int dpaa_logtype_sec;
  44
  45 static uint8_t cryptodev_driver_id;
  46
  47 static __thread struct rte_crypto_op **dpaa_sec_ops;
  48 static __thread int dpaa_sec_op_nb;
  49
  50 static int
  51 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess);
  52
  53 static inline void
  54 dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx)
  55 {
  56         if (!ctx->fd_status) {
  57                 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
  58         } else {
  59                 DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status);
  60                 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
  61         }
  62
  63         /* report op status to sym->op and then free the ctx memeory  */
  64         rte_mempool_put(ctx->ctx_pool, (void *)ctx);
  65 }
  66
  67 static inline struct dpaa_sec_op_ctx *
  68 dpaa_sec_alloc_ctx(dpaa_sec_session *ses)
  69 {
  70         struct dpaa_sec_op_ctx *ctx;
  71         int retval;
  72
  73         retval = rte_mempool_get(ses->ctx_pool, (void **)(&ctx));
  74         if (!ctx || retval) {
  75                 DPAA_SEC_DP_WARN("Alloc sec descriptor failed!");
  76                 return NULL;
  77         }
  78         /*
  79          * Clear SG memory. There are 16 SG entries of 16 Bytes each.
  80          * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
  81          * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for
  82          * each packet, memset is costlier than dcbz_64().
  83          */
  84         dcbz_64(&ctx->job.sg[SG_CACHELINE_0]);
  85         dcbz_64(&ctx->job.sg[SG_CACHELINE_1]);
  86         dcbz_64(&ctx->job.sg[SG_CACHELINE_2]);
  87         dcbz_64(&ctx->job.sg[SG_CACHELINE_3]);
  88
  89         ctx->ctx_pool = ses->ctx_pool;
  90         ctx->vtop_offset = (size_t) ctx
  91                                 - rte_mempool_virt2iova(ctx);
  92
  93         return ctx;
  94 }
  95
  96 static inline rte_iova_t
  97 dpaa_mem_vtop(void *vaddr)
  98 {
  99         const struct rte_memseg *ms;
 100
 101         ms = rte_mem_virt2memseg(vaddr, NULL);
 102         if (ms)
 103                 return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr);
 104         return (size_t)NULL;
 105 }
 106
 107 static inline void *
 108 dpaa_mem_ptov(rte_iova_t paddr)
 109 {
 110         return rte_mem_iova2virt(paddr);
 111 }
 112
 113 static void
 114 ern_sec_fq_handler(struct qman_portal *qm __rte_unused,
 115                    struct qman_fq *fq,
 116                    const struct qm_mr_entry *msg)
 117 {
 118         DPAA_SEC_DP_ERR("sec fq %d error, RC = %x, seqnum = %x\n",
 119                         fq->fqid, msg->ern.rc, msg->ern.seqnum);
 120 }
 121
 122 /* initialize the queue with dest chan as caam chan so that
 123  * all the packets in this queue could be dispatched into caam
 124  */
 125 static int
 126 dpaa_sec_init_rx(struct qman_fq *fq_in, rte_iova_t hwdesc,
 127                  uint32_t fqid_out)
 128 {
 129         struct qm_mcc_initfq fq_opts;
 130         uint32_t flags;
 131         int ret = -1;
 132
 133         /* Clear FQ options */
 134         memset(&fq_opts, 0x00, sizeof(struct qm_mcc_initfq));
 135
 136         flags = QMAN_INITFQ_FLAG_SCHED;
 137         fq_opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_CONTEXTA |
 138                           QM_INITFQ_WE_CONTEXTB;
 139
 140         qm_fqd_context_a_set64(&fq_opts.fqd, hwdesc);
 141         fq_opts.fqd.context_b = fqid_out;
 142         fq_opts.fqd.dest.channel = qm_channel_caam;
 143         fq_opts.fqd.dest.wq = 0;
 144
 145         fq_in->cb.ern  = ern_sec_fq_handler;
 146
 147         DPAA_SEC_DEBUG("in-%x out-%x", fq_in->fqid, fqid_out);
 148
 149         ret = qman_init_fq(fq_in, flags, &fq_opts);
 150         if (unlikely(ret != 0))
 151                 DPAA_SEC_ERR("qman_init_fq failed %d", ret);
 152
 153         return ret;
 154 }
 155
 156 /* something is put into in_fq and caam put the crypto result into out_fq */
 157 static enum qman_cb_dqrr_result
 158 dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused,
 159                   struct qman_fq *fq __always_unused,
 160                   const struct qm_dqrr_entry *dqrr)
 161 {
 162         const struct qm_fd *fd;
 163         struct dpaa_sec_job *job;
 164         struct dpaa_sec_op_ctx *ctx;
 165
 166         if (dpaa_sec_op_nb >= DPAA_SEC_BURST)
 167                 return qman_cb_dqrr_defer;
 168
 169         if (!(dqrr->stat & QM_DQRR_STAT_FD_VALID))
 170                 return qman_cb_dqrr_consume;
 171
 172         fd = &dqrr->fd;
 173         /* sg is embedded in an op ctx,
 174          * sg[0] is for output
 175          * sg[1] for input
 176          */
 177         job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
 178
 179         ctx = container_of(job, struct dpaa_sec_op_ctx, job);
 180         ctx->fd_status = fd->status;
 181         if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 182                 struct qm_sg_entry *sg_out;
 183                 uint32_t len;
 184
 185                 sg_out = &job->sg[0];
 186                 hw_sg_to_cpu(sg_out);
 187                 len = sg_out->length;
 188                 ctx->op->sym->m_src->pkt_len = len;
 189                 ctx->op->sym->m_src->data_len = len;
 190         }
 191         dpaa_sec_ops[dpaa_sec_op_nb++] = ctx->op;
 192         dpaa_sec_op_ending(ctx);
 193
 194         return qman_cb_dqrr_consume;
 195 }
 196
 197 /* caam result is put into this queue */
 198 static int
 199 dpaa_sec_init_tx(struct qman_fq *fq)
 200 {
 201         int ret;
 202         struct qm_mcc_initfq opts;
 203         uint32_t flags;
 204
 205         flags = QMAN_FQ_FLAG_NO_ENQUEUE | QMAN_FQ_FLAG_LOCKED |
 206                 QMAN_FQ_FLAG_DYNAMIC_FQID;
 207
 208         ret = qman_create_fq(0, flags, fq);
 209         if (unlikely(ret)) {
 210                 DPAA_SEC_ERR("qman_create_fq failed");
 211                 return ret;
 212         }
 213
 214         memset(&opts, 0, sizeof(opts));
 215         opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
 216                        QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB;
 217
 218         /* opts.fqd.dest.channel = dpaa_sec_pool_chan; */
 219
 220         fq->cb.dqrr = dqrr_out_fq_cb_rx;
 221         fq->cb.ern  = ern_sec_fq_handler;
 222
 223         ret = qman_init_fq(fq, 0, &opts);
 224         if (unlikely(ret)) {
 225                 DPAA_SEC_ERR("unable to init caam source fq!");
 226                 return ret;
 227         }
 228
 229         return ret;
 230 }
 231
 232 static inline int is_cipher_only(dpaa_sec_session *ses)
 233 {
 234         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 235                 (ses->auth_alg == RTE_CRYPTO_AUTH_NULL));
 236 }
 237
 238 static inline int is_auth_only(dpaa_sec_session *ses)
 239 {
 240         return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) &&
 241                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL));
 242 }
 243
 244 static inline int is_aead(dpaa_sec_session *ses)
 245 {
 246         return ((ses->cipher_alg == 0) &&
 247                 (ses->auth_alg == 0) &&
 248                 (ses->aead_alg != 0));
 249 }
 250
 251 static inline int is_auth_cipher(dpaa_sec_session *ses)
 252 {
 253         return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) &&
 254                 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) &&
 255                 (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC));
 256 }
 257
 258 static inline int is_proto_ipsec(dpaa_sec_session *ses)
 259 {
 260         return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC);
 261 }
 262
 263 static inline int is_encode(dpaa_sec_session *ses)
 264 {
 265         return ses->dir == DIR_ENC;
 266 }
 267
 268 static inline int is_decode(dpaa_sec_session *ses)
 269 {
 270         return ses->dir == DIR_DEC;
 271 }
 272
 273 static inline void
 274 caam_auth_alg(dpaa_sec_session *ses, struct alginfo *alginfo_a)
 275 {
 276         switch (ses->auth_alg) {
 277         case RTE_CRYPTO_AUTH_NULL:
 278                 ses->digest_length = 0;
 279                 break;
 280         case RTE_CRYPTO_AUTH_MD5_HMAC:
 281                 alginfo_a->algtype =
 282                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 283                         OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5;
 284                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 285                 break;
 286         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 287                 alginfo_a->algtype =
 288                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 289                         OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1;
 290                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 291                 break;
 292         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 293                 alginfo_a->algtype =
 294                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 295                         OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224;
 296                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 297                 break;
 298         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 299                 alginfo_a->algtype =
 300                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 301                         OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256;
 302                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 303                 break;
 304         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 305                 alginfo_a->algtype =
 306                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 307                         OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384;
 308                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 309                 break;
 310         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 311                 alginfo_a->algtype =
 312                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 313                         OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512;
 314                 alginfo_a->algmode = OP_ALG_AAI_HMAC;
 315                 break;
 316         default:
 317                 DPAA_SEC_ERR("unsupported auth alg %u", ses->auth_alg);
 318         }
 319 }
 320
 321 static inline void
 322 caam_cipher_alg(dpaa_sec_session *ses, struct alginfo *alginfo_c)
 323 {
 324         switch (ses->cipher_alg) {
 325         case RTE_CRYPTO_CIPHER_NULL:
 326                 break;
 327         case RTE_CRYPTO_CIPHER_AES_CBC:
 328                 alginfo_c->algtype =
 329                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 330                         OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES;
 331                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 332                 break;
 333         case RTE_CRYPTO_CIPHER_3DES_CBC:
 334                 alginfo_c->algtype =
 335                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 336                         OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES;
 337                 alginfo_c->algmode = OP_ALG_AAI_CBC;
 338                 break;
 339         case RTE_CRYPTO_CIPHER_AES_CTR:
 340                 alginfo_c->algtype =
 341                         (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ?
 342                         OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES;
 343                 alginfo_c->algmode = OP_ALG_AAI_CTR;
 344                 break;
 345         default:
 346                 DPAA_SEC_ERR("unsupported cipher alg %d", ses->cipher_alg);
 347         }
 348 }
 349
 350 static inline void
 351 caam_aead_alg(dpaa_sec_session *ses, struct alginfo *alginfo)
 352 {
 353         switch (ses->aead_alg) {
 354         case RTE_CRYPTO_AEAD_AES_GCM:
 355                 alginfo->algtype = OP_ALG_ALGSEL_AES;
 356                 alginfo->algmode = OP_ALG_AAI_GCM;
 357                 break;
 358         default:
 359                 DPAA_SEC_ERR("unsupported AEAD alg %d", ses->aead_alg);
 360         }
 361 }
 362
 363
 364 /* prepare command block of the session */
 365 static int
 366 dpaa_sec_prep_cdb(dpaa_sec_session *ses)
 367 {
 368         struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0};
 369         int32_t shared_desc_len = 0;
 370         struct sec_cdb *cdb = &ses->cdb;
 371         int err;
 372 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 373         int swap = false;
 374 #else
 375         int swap = true;
 376 #endif
 377
 378         memset(cdb, 0, sizeof(struct sec_cdb));
 379
 380         if (is_cipher_only(ses)) {
 381                 caam_cipher_alg(ses, &alginfo_c);
 382                 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 383                         DPAA_SEC_ERR("not supported cipher alg");
 384                         return -ENOTSUP;
 385                 }
 386
 387                 alginfo_c.key = (size_t)ses->cipher_key.data;
 388                 alginfo_c.keylen = ses->cipher_key.length;
 389                 alginfo_c.key_enc_flags = 0;
 390                 alginfo_c.key_type = RTA_DATA_IMM;
 391
 392                 shared_desc_len = cnstr_shdsc_blkcipher(
 393                                                 cdb->sh_desc, true,
 394                                                 swap, &alginfo_c,
 395                                                 NULL,
 396                                                 ses->iv.length,
 397                                                 ses->dir);
 398         } else if (is_auth_only(ses)) {
 399                 caam_auth_alg(ses, &alginfo_a);
 400                 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 401                         DPAA_SEC_ERR("not supported auth alg");
 402                         return -ENOTSUP;
 403                 }
 404
 405                 alginfo_a.key = (size_t)ses->auth_key.data;
 406                 alginfo_a.keylen = ses->auth_key.length;
 407                 alginfo_a.key_enc_flags = 0;
 408                 alginfo_a.key_type = RTA_DATA_IMM;
 409
 410                 shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true,
 411                                                    swap, &alginfo_a,
 412                                                    !ses->dir,
 413                                                    ses->digest_length);
 414         } else if (is_aead(ses)) {
 415                 caam_aead_alg(ses, &alginfo);
 416                 if (alginfo.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 417                         DPAA_SEC_ERR("not supported aead alg");
 418                         return -ENOTSUP;
 419                 }
 420                 alginfo.key = (size_t)ses->aead_key.data;
 421                 alginfo.keylen = ses->aead_key.length;
 422                 alginfo.key_enc_flags = 0;
 423                 alginfo.key_type = RTA_DATA_IMM;
 424
 425                 if (ses->dir == DIR_ENC)
 426                         shared_desc_len = cnstr_shdsc_gcm_encap(
 427                                         cdb->sh_desc, true, swap,
 428                                         &alginfo,
 429                                         ses->iv.length,
 430                                         ses->digest_length);
 431                 else
 432                         shared_desc_len = cnstr_shdsc_gcm_decap(
 433                                         cdb->sh_desc, true, swap,
 434                                         &alginfo,
 435                                         ses->iv.length,
 436                                         ses->digest_length);
 437         } else {
 438                 caam_cipher_alg(ses, &alginfo_c);
 439                 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 440                         DPAA_SEC_ERR("not supported cipher alg");
 441                         return -ENOTSUP;
 442                 }
 443
 444                 alginfo_c.key = (size_t)ses->cipher_key.data;
 445                 alginfo_c.keylen = ses->cipher_key.length;
 446                 alginfo_c.key_enc_flags = 0;
 447                 alginfo_c.key_type = RTA_DATA_IMM;
 448
 449                 caam_auth_alg(ses, &alginfo_a);
 450                 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 451                         DPAA_SEC_ERR("not supported auth alg");
 452                         return -ENOTSUP;
 453                 }
 454
 455                 alginfo_a.key = (size_t)ses->auth_key.data;
 456                 alginfo_a.keylen = ses->auth_key.length;
 457                 alginfo_a.key_enc_flags = 0;
 458                 alginfo_a.key_type = RTA_DATA_IMM;
 459
 460                 cdb->sh_desc[0] = alginfo_c.keylen;
 461                 cdb->sh_desc[1] = alginfo_a.keylen;
 462                 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
 463                                        MIN_JOB_DESC_SIZE,
 464                                        (unsigned int *)cdb->sh_desc,
 465                                        &cdb->sh_desc[2], 2);
 466
 467                 if (err < 0) {
 468                         DPAA_SEC_ERR("Crypto: Incorrect key lengths");
 469                         return err;
 470                 }
 471                 if (cdb->sh_desc[2] & 1)
 472                         alginfo_c.key_type = RTA_DATA_IMM;
 473                 else {
 474                         alginfo_c.key = (size_t)dpaa_mem_vtop(
 475                                                 (void *)(size_t)alginfo_c.key);
 476                         alginfo_c.key_type = RTA_DATA_PTR;
 477                 }
 478                 if (cdb->sh_desc[2] & (1<<1))
 479                         alginfo_a.key_type = RTA_DATA_IMM;
 480                 else {
 481                         alginfo_a.key = (size_t)dpaa_mem_vtop(
 482                                                 (void *)(size_t)alginfo_a.key);
 483                         alginfo_a.key_type = RTA_DATA_PTR;
 484                 }
 485                 cdb->sh_desc[0] = 0;
 486                 cdb->sh_desc[1] = 0;
 487                 cdb->sh_desc[2] = 0;
 488                 if (is_proto_ipsec(ses)) {
 489                         if (ses->dir == DIR_ENC) {
 490                                 shared_desc_len = cnstr_shdsc_ipsec_new_encap(
 491                                                 cdb->sh_desc,
 492                                                 true, swap, SHR_SERIAL,
 493                                                 &ses->encap_pdb,
 494                                                 (uint8_t *)&ses->ip4_hdr,
 495                                                 &alginfo_c, &alginfo_a);
 496                         } else if (ses->dir == DIR_DEC) {
 497                                 shared_desc_len = cnstr_shdsc_ipsec_new_decap(
 498                                                 cdb->sh_desc,
 499                                                 true, swap, SHR_SERIAL,
 500                                                 &ses->decap_pdb,
 501                                                 &alginfo_c, &alginfo_a);
 502                         }
 503                 } else {
 504                         /* Auth_only_len is set as 0 here and it will be
 505                          * overwritten in fd for each packet.
 506                          */
 507                         shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
 508                                         true, swap, &alginfo_c, &alginfo_a,
 509                                         ses->iv.length, 0,
 510                                         ses->digest_length, ses->dir);
 511                 }
 512         }
 513
 514         if (shared_desc_len < 0) {
 515                 DPAA_SEC_ERR("error in preparing command block");
 516                 return shared_desc_len;
 517         }
 518
 519         cdb->sh_hdr.hi.field.idlen = shared_desc_len;
 520         cdb->sh_hdr.hi.word = rte_cpu_to_be_32(cdb->sh_hdr.hi.word);
 521         cdb->sh_hdr.lo.word = rte_cpu_to_be_32(cdb->sh_hdr.lo.word);
 522
 523         return 0;
 524 }
 525
 526 /* qp is lockless, should be accessed by only one thread */
 527 static int
 528 dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops)
 529 {
 530         struct qman_fq *fq;
 531         unsigned int pkts = 0;
 532         int num_rx_bufs, ret;
 533         struct qm_dqrr_entry *dq;
 534         uint32_t vdqcr_flags = 0;
 535
 536         fq = &qp->outq;
 537         /*
 538          * Until request for four buffers, we provide exact number of buffers.
 539          * Otherwise we do not set the QM_VDQCR_EXACT flag.
 540          * Not setting QM_VDQCR_EXACT flag can provide two more buffers than
 541          * requested, so we request two less in this case.
 542          */
 543         if (nb_ops < 4) {
 544                 vdqcr_flags = QM_VDQCR_EXACT;
 545                 num_rx_bufs = nb_ops;
 546         } else {
 547                 num_rx_bufs = nb_ops > DPAA_MAX_DEQUEUE_NUM_FRAMES ?
 548                         (DPAA_MAX_DEQUEUE_NUM_FRAMES - 2) : (nb_ops - 2);
 549         }
 550         ret = qman_set_vdq(fq, num_rx_bufs, vdqcr_flags);
 551         if (ret)
 552                 return 0;
 553
 554         do {
 555                 const struct qm_fd *fd;
 556                 struct dpaa_sec_job *job;
 557                 struct dpaa_sec_op_ctx *ctx;
 558                 struct rte_crypto_op *op;
 559
 560                 dq = qman_dequeue(fq);
 561                 if (!dq)
 562                         continue;
 563
 564                 fd = &dq->fd;
 565                 /* sg is embedded in an op ctx,
 566                  * sg[0] is for output
 567                  * sg[1] for input
 568                  */
 569                 job = dpaa_mem_ptov(qm_fd_addr_get64(fd));
 570
 571                 ctx = container_of(job, struct dpaa_sec_op_ctx, job);
 572                 ctx->fd_status = fd->status;
 573                 op = ctx->op;
 574                 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 575                         struct qm_sg_entry *sg_out;
 576                         uint32_t len;
 577
 578                         sg_out = &job->sg[0];
 579                         hw_sg_to_cpu(sg_out);
 580                         len = sg_out->length;
 581                         op->sym->m_src->pkt_len = len;
 582                         op->sym->m_src->data_len = len;
 583                 }
 584                 if (!ctx->fd_status) {
 585                         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 586                 } else {
 587                         DPAA_SEC_DP_WARN("SEC return err:0x%x", ctx->fd_status);
 588                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 589                 }
 590                 ops[pkts++] = op;
 591
 592                 /* report op status to sym->op and then free the ctx memeory */
 593                 rte_mempool_put(ctx->ctx_pool, (void *)ctx);
 594
 595                 qman_dqrr_consume(fq, dq);
 596         } while (fq->flags & QMAN_FQ_STATE_VDQCR);
 597
 598         return pkts;
 599 }
 600
 601 static inline struct dpaa_sec_job *
 602 build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 603 {
 604         struct rte_crypto_sym_op *sym = op->sym;
 605         struct rte_mbuf *mbuf = sym->m_src;
 606         struct dpaa_sec_job *cf;
 607         struct dpaa_sec_op_ctx *ctx;
 608         struct qm_sg_entry *sg, *out_sg, *in_sg;
 609         phys_addr_t start_addr;
 610         uint8_t *old_digest, extra_segs;
 611
 612         if (is_decode(ses))
 613                 extra_segs = 3;
 614         else
 615                 extra_segs = 2;
 616
 617         if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) {
 618                 DPAA_SEC_DP_ERR("Auth: Max sec segs supported is %d",
 619                                 MAX_SG_ENTRIES);
 620                 return NULL;
 621         }
 622         ctx = dpaa_sec_alloc_ctx(ses);
 623         if (!ctx)
 624                 return NULL;
 625
 626         cf = &ctx->job;
 627         ctx->op = op;
 628         old_digest = ctx->digest;
 629
 630         /* output */
 631         out_sg = &cf->sg[0];
 632         qm_sg_entry_set64(out_sg, sym->auth.digest.phys_addr);
 633         out_sg->length = ses->digest_length;
 634         cpu_to_hw_sg(out_sg);
 635
 636         /* input */
 637         in_sg = &cf->sg[1];
 638         /* need to extend the input to a compound frame */
 639         in_sg->extension = 1;
 640         in_sg->final = 1;
 641         in_sg->length = sym->auth.data.length;
 642         qm_sg_entry_set64(in_sg, dpaa_mem_vtop(&cf->sg[2]));
 643
 644         /* 1st seg */
 645         sg = in_sg + 1;
 646         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 647         sg->length = mbuf->data_len - sym->auth.data.offset;
 648         sg->offset = sym->auth.data.offset;
 649
 650         /* Successive segs */
 651         mbuf = mbuf->next;
 652         while (mbuf) {
 653                 cpu_to_hw_sg(sg);
 654                 sg++;
 655                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 656                 sg->length = mbuf->data_len;
 657                 mbuf = mbuf->next;
 658         }
 659
 660         if (is_decode(ses)) {
 661                 /* Digest verification case */
 662                 cpu_to_hw_sg(sg);
 663                 sg++;
 664                 rte_memcpy(old_digest, sym->auth.digest.data,
 665                                 ses->digest_length);
 666                 start_addr = dpaa_mem_vtop(old_digest);
 667                 qm_sg_entry_set64(sg, start_addr);
 668                 sg->length = ses->digest_length;
 669                 in_sg->length += ses->digest_length;
 670         } else {
 671                 /* Digest calculation case */
 672                 sg->length -= ses->digest_length;
 673         }
 674         sg->final = 1;
 675         cpu_to_hw_sg(sg);
 676         cpu_to_hw_sg(in_sg);
 677
 678         return cf;
 679 }
 680
 681 /**
 682  * packet looks like:
 683  *              |<----data_len------->|
 684  *    |ip_header|ah_header|icv|payload|
 685  *              ^
 686  *              |
 687  *         mbuf->pkt.data
 688  */
 689 static inline struct dpaa_sec_job *
 690 build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
 691 {
 692         struct rte_crypto_sym_op *sym = op->sym;
 693         struct rte_mbuf *mbuf = sym->m_src;
 694         struct dpaa_sec_job *cf;
 695         struct dpaa_sec_op_ctx *ctx;
 696         struct qm_sg_entry *sg;
 697         rte_iova_t start_addr;
 698         uint8_t *old_digest;
 699
 700         ctx = dpaa_sec_alloc_ctx(ses);
 701         if (!ctx)
 702                 return NULL;
 703
 704         cf = &ctx->job;
 705         ctx->op = op;
 706         old_digest = ctx->digest;
 707
 708         start_addr = rte_pktmbuf_iova(mbuf);
 709         /* output */
 710         sg = &cf->sg[0];
 711         qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
 712         sg->length = ses->digest_length;
 713         cpu_to_hw_sg(sg);
 714
 715         /* input */
 716         sg = &cf->sg[1];
 717         if (is_decode(ses)) {
 718                 /* need to extend the input to a compound frame */
 719                 sg->extension = 1;
 720                 qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2]));
 721                 sg->length = sym->auth.data.length + ses->digest_length;
 722                 sg->final = 1;
 723                 cpu_to_hw_sg(sg);
 724
 725                 sg = &cf->sg[2];
 726                 /* hash result or digest, save digest first */
 727                 rte_memcpy(old_digest, sym->auth.digest.data,
 728                            ses->digest_length);
 729                 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset);
 730                 sg->length = sym->auth.data.length;
 731                 cpu_to_hw_sg(sg);
 732
 733                 /* let's check digest by hw */
 734                 start_addr = dpaa_mem_vtop(old_digest);
 735                 sg++;
 736                 qm_sg_entry_set64(sg, start_addr);
 737                 sg->length = ses->digest_length;
 738                 sg->final = 1;
 739                 cpu_to_hw_sg(sg);
 740         } else {
 741                 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset);
 742                 sg->length = sym->auth.data.length;
 743                 sg->final = 1;
 744                 cpu_to_hw_sg(sg);
 745         }
 746
 747         return cf;
 748 }
 749
 750 static inline struct dpaa_sec_job *
 751 build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 752 {
 753         struct rte_crypto_sym_op *sym = op->sym;
 754         struct dpaa_sec_job *cf;
 755         struct dpaa_sec_op_ctx *ctx;
 756         struct qm_sg_entry *sg, *out_sg, *in_sg;
 757         struct rte_mbuf *mbuf;
 758         uint8_t req_segs;
 759         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 760                         ses->iv.offset);
 761
 762         if (sym->m_dst) {
 763                 mbuf = sym->m_dst;
 764                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3;
 765         } else {
 766                 mbuf = sym->m_src;
 767                 req_segs = mbuf->nb_segs * 2 + 3;
 768         }
 769
 770         if (req_segs > MAX_SG_ENTRIES) {
 771                 DPAA_SEC_DP_ERR("Cipher: Max sec segs supported is %d",
 772                                 MAX_SG_ENTRIES);
 773                 return NULL;
 774         }
 775
 776         ctx = dpaa_sec_alloc_ctx(ses);
 777         if (!ctx)
 778                 return NULL;
 779
 780         cf = &ctx->job;
 781         ctx->op = op;
 782
 783         /* output */
 784         out_sg = &cf->sg[0];
 785         out_sg->extension = 1;
 786         out_sg->length = sym->cipher.data.length;
 787         qm_sg_entry_set64(out_sg, dpaa_mem_vtop(&cf->sg[2]));
 788         cpu_to_hw_sg(out_sg);
 789
 790         /* 1st seg */
 791         sg = &cf->sg[2];
 792         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 793         sg->length = mbuf->data_len - sym->cipher.data.offset;
 794         sg->offset = sym->cipher.data.offset;
 795
 796         /* Successive segs */
 797         mbuf = mbuf->next;
 798         while (mbuf) {
 799                 cpu_to_hw_sg(sg);
 800                 sg++;
 801                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 802                 sg->length = mbuf->data_len;
 803                 mbuf = mbuf->next;
 804         }
 805         sg->final = 1;
 806         cpu_to_hw_sg(sg);
 807
 808         /* input */
 809         mbuf = sym->m_src;
 810         in_sg = &cf->sg[1];
 811         in_sg->extension = 1;
 812         in_sg->final = 1;
 813         in_sg->length = sym->cipher.data.length + ses->iv.length;
 814
 815         sg++;
 816         qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
 817         cpu_to_hw_sg(in_sg);
 818
 819         /* IV */
 820         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
 821         sg->length = ses->iv.length;
 822         cpu_to_hw_sg(sg);
 823
 824         /* 1st seg */
 825         sg++;
 826         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 827         sg->length = mbuf->data_len - sym->cipher.data.offset;
 828         sg->offset = sym->cipher.data.offset;
 829
 830         /* Successive segs */
 831         mbuf = mbuf->next;
 832         while (mbuf) {
 833                 cpu_to_hw_sg(sg);
 834                 sg++;
 835                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 836                 sg->length = mbuf->data_len;
 837                 mbuf = mbuf->next;
 838         }
 839         sg->final = 1;
 840         cpu_to_hw_sg(sg);
 841
 842         return cf;
 843 }
 844
 845 static inline struct dpaa_sec_job *
 846 build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
 847 {
 848         struct rte_crypto_sym_op *sym = op->sym;
 849         struct dpaa_sec_job *cf;
 850         struct dpaa_sec_op_ctx *ctx;
 851         struct qm_sg_entry *sg;
 852         rte_iova_t src_start_addr, dst_start_addr;
 853         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 854                         ses->iv.offset);
 855
 856         ctx = dpaa_sec_alloc_ctx(ses);
 857         if (!ctx)
 858                 return NULL;
 859
 860         cf = &ctx->job;
 861         ctx->op = op;
 862
 863         src_start_addr = rte_pktmbuf_iova(sym->m_src);
 864
 865         if (sym->m_dst)
 866                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
 867         else
 868                 dst_start_addr = src_start_addr;
 869
 870         /* output */
 871         sg = &cf->sg[0];
 872         qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset);
 873         sg->length = sym->cipher.data.length + ses->iv.length;
 874         cpu_to_hw_sg(sg);
 875
 876         /* input */
 877         sg = &cf->sg[1];
 878
 879         /* need to extend the input to a compound frame */
 880         sg->extension = 1;
 881         sg->final = 1;
 882         sg->length = sym->cipher.data.length + ses->iv.length;
 883         qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2]));
 884         cpu_to_hw_sg(sg);
 885
 886         sg = &cf->sg[2];
 887         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
 888         sg->length = ses->iv.length;
 889         cpu_to_hw_sg(sg);
 890
 891         sg++;
 892         qm_sg_entry_set64(sg, src_start_addr + sym->cipher.data.offset);
 893         sg->length = sym->cipher.data.length;
 894         sg->final = 1;
 895         cpu_to_hw_sg(sg);
 896
 897         return cf;
 898 }
 899
 900 static inline struct dpaa_sec_job *
 901 build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 902 {
 903         struct rte_crypto_sym_op *sym = op->sym;
 904         struct dpaa_sec_job *cf;
 905         struct dpaa_sec_op_ctx *ctx;
 906         struct qm_sg_entry *sg, *out_sg, *in_sg;
 907         struct rte_mbuf *mbuf;
 908         uint8_t req_segs;
 909         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 910                         ses->iv.offset);
 911
 912         if (sym->m_dst) {
 913                 mbuf = sym->m_dst;
 914                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4;
 915         } else {
 916                 mbuf = sym->m_src;
 917                 req_segs = mbuf->nb_segs * 2 + 4;
 918         }
 919
 920         if (ses->auth_only_len)
 921                 req_segs++;
 922
 923         if (req_segs > MAX_SG_ENTRIES) {
 924                 DPAA_SEC_DP_ERR("AEAD: Max sec segs supported is %d",
 925                                 MAX_SG_ENTRIES);
 926                 return NULL;
 927         }
 928
 929         ctx = dpaa_sec_alloc_ctx(ses);
 930         if (!ctx)
 931                 return NULL;
 932
 933         cf = &ctx->job;
 934         ctx->op = op;
 935
 936         rte_prefetch0(cf->sg);
 937
 938         /* output */
 939         out_sg = &cf->sg[0];
 940         out_sg->extension = 1;
 941         if (is_encode(ses))
 942                 out_sg->length = sym->aead.data.length + ses->auth_only_len
 943                                                 + ses->digest_length;
 944         else
 945                 out_sg->length = sym->aead.data.length + ses->auth_only_len;
 946
 947         /* output sg entries */
 948         sg = &cf->sg[2];
 949         qm_sg_entry_set64(out_sg, dpaa_mem_vtop(sg));
 950         cpu_to_hw_sg(out_sg);
 951
 952         /* 1st seg */
 953         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 954         sg->length = mbuf->data_len - sym->aead.data.offset +
 955                                         ses->auth_only_len;
 956         sg->offset = sym->aead.data.offset - ses->auth_only_len;
 957
 958         /* Successive segs */
 959         mbuf = mbuf->next;
 960         while (mbuf) {
 961                 cpu_to_hw_sg(sg);
 962                 sg++;
 963                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
 964                 sg->length = mbuf->data_len;
 965                 mbuf = mbuf->next;
 966         }
 967         sg->length -= ses->digest_length;
 968
 969         if (is_encode(ses)) {
 970                 cpu_to_hw_sg(sg);
 971                 /* set auth output */
 972                 sg++;
 973                 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr);
 974                 sg->length = ses->digest_length;
 975         }
 976         sg->final = 1;
 977         cpu_to_hw_sg(sg);
 978
 979         /* input */
 980         mbuf = sym->m_src;
 981         in_sg = &cf->sg[1];
 982         in_sg->extension = 1;
 983         in_sg->final = 1;
 984         if (is_encode(ses))
 985                 in_sg->length = ses->iv.length + sym->aead.data.length
 986                                                         + ses->auth_only_len;
 987         else
 988                 in_sg->length = ses->iv.length + sym->aead.data.length
 989                                 + ses->auth_only_len + ses->digest_length;
 990
 991         /* input sg entries */
 992         sg++;
 993         qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
 994         cpu_to_hw_sg(in_sg);
 995
 996         /* 1st seg IV */
 997         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
 998         sg->length = ses->iv.length;
 999         cpu_to_hw_sg(sg);
1000
1001         /* 2nd seg auth only */
1002         if (ses->auth_only_len) {
1003                 sg++;
1004                 qm_sg_entry_set64(sg, dpaa_mem_vtop(sym->aead.aad.data));
1005                 sg->length = ses->auth_only_len;
1006                 cpu_to_hw_sg(sg);
1007         }
1008
1009         /* 3rd seg */
1010         sg++;
1011         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1012         sg->length = mbuf->data_len - sym->aead.data.offset;
1013         sg->offset = sym->aead.data.offset;
1014
1015         /* Successive segs */
1016         mbuf = mbuf->next;
1017         while (mbuf) {
1018                 cpu_to_hw_sg(sg);
1019                 sg++;
1020                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1021                 sg->length = mbuf->data_len;
1022                 mbuf = mbuf->next;
1023         }
1024
1025         if (is_decode(ses)) {
1026                 cpu_to_hw_sg(sg);
1027                 sg++;
1028                 memcpy(ctx->digest, sym->aead.digest.data,
1029                         ses->digest_length);
1030                 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
1031                 sg->length = ses->digest_length;
1032         }
1033         sg->final = 1;
1034         cpu_to_hw_sg(sg);
1035
1036         return cf;
1037 }
1038
1039 static inline struct dpaa_sec_job *
1040 build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses)
1041 {
1042         struct rte_crypto_sym_op *sym = op->sym;
1043         struct dpaa_sec_job *cf;
1044         struct dpaa_sec_op_ctx *ctx;
1045         struct qm_sg_entry *sg;
1046         uint32_t length = 0;
1047         rte_iova_t src_start_addr, dst_start_addr;
1048         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1049                         ses->iv.offset);
1050
1051         src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
1052
1053         if (sym->m_dst)
1054                 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
1055         else
1056                 dst_start_addr = src_start_addr;
1057
1058         ctx = dpaa_sec_alloc_ctx(ses);
1059         if (!ctx)
1060                 return NULL;
1061
1062         cf = &ctx->job;
1063         ctx->op = op;
1064
1065         /* input */
1066         rte_prefetch0(cf->sg);
1067         sg = &cf->sg[2];
1068         qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg));
1069         if (is_encode(ses)) {
1070                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1071                 sg->length = ses->iv.length;
1072                 length += sg->length;
1073                 cpu_to_hw_sg(sg);
1074
1075                 sg++;
1076                 if (ses->auth_only_len) {
1077                         qm_sg_entry_set64(sg,
1078                                           dpaa_mem_vtop(sym->aead.aad.data));
1079                         sg->length = ses->auth_only_len;
1080                         length += sg->length;
1081                         cpu_to_hw_sg(sg);
1082                         sg++;
1083                 }
1084                 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
1085                 sg->length = sym->aead.data.length;
1086                 length += sg->length;
1087                 sg->final = 1;
1088                 cpu_to_hw_sg(sg);
1089         } else {
1090                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1091                 sg->length = ses->iv.length;
1092                 length += sg->length;
1093                 cpu_to_hw_sg(sg);
1094
1095                 sg++;
1096                 if (ses->auth_only_len) {
1097                         qm_sg_entry_set64(sg,
1098                                           dpaa_mem_vtop(sym->aead.aad.data));
1099                         sg->length = ses->auth_only_len;
1100                         length += sg->length;
1101                         cpu_to_hw_sg(sg);
1102                         sg++;
1103                 }
1104                 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
1105                 sg->length = sym->aead.data.length;
1106                 length += sg->length;
1107                 cpu_to_hw_sg(sg);
1108
1109                 memcpy(ctx->digest, sym->aead.digest.data,
1110                        ses->digest_length);
1111                 sg++;
1112
1113                 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
1114                 sg->length = ses->digest_length;
1115                 length += sg->length;
1116                 sg->final = 1;
1117                 cpu_to_hw_sg(sg);
1118         }
1119         /* input compound frame */
1120         cf->sg[1].length = length;
1121         cf->sg[1].extension = 1;
1122         cf->sg[1].final = 1;
1123         cpu_to_hw_sg(&cf->sg[1]);
1124
1125         /* output */
1126         sg++;
1127         qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg));
1128         qm_sg_entry_set64(sg,
1129                 dst_start_addr + sym->aead.data.offset - ses->auth_only_len);
1130         sg->length = sym->aead.data.length + ses->auth_only_len;
1131         length = sg->length;
1132         if (is_encode(ses)) {
1133                 cpu_to_hw_sg(sg);
1134                 /* set auth output */
1135                 sg++;
1136                 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr);
1137                 sg->length = ses->digest_length;
1138                 length += sg->length;
1139         }
1140         sg->final = 1;
1141         cpu_to_hw_sg(sg);
1142
1143         /* output compound frame */
1144         cf->sg[0].length = length;
1145         cf->sg[0].extension = 1;
1146         cpu_to_hw_sg(&cf->sg[0]);
1147
1148         return cf;
1149 }
1150
1151 static inline struct dpaa_sec_job *
1152 build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
1153 {
1154         struct rte_crypto_sym_op *sym = op->sym;
1155         struct dpaa_sec_job *cf;
1156         struct dpaa_sec_op_ctx *ctx;
1157         struct qm_sg_entry *sg, *out_sg, *in_sg;
1158         struct rte_mbuf *mbuf;
1159         uint8_t req_segs;
1160         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1161                         ses->iv.offset);
1162
1163         if (sym->m_dst) {
1164                 mbuf = sym->m_dst;
1165                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4;
1166         } else {
1167                 mbuf = sym->m_src;
1168                 req_segs = mbuf->nb_segs * 2 + 4;
1169         }
1170
1171         if (req_segs > MAX_SG_ENTRIES) {
1172                 DPAA_SEC_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
1173                                 MAX_SG_ENTRIES);
1174                 return NULL;
1175         }
1176
1177         ctx = dpaa_sec_alloc_ctx(ses);
1178         if (!ctx)
1179                 return NULL;
1180
1181         cf = &ctx->job;
1182         ctx->op = op;
1183
1184         rte_prefetch0(cf->sg);
1185
1186         /* output */
1187         out_sg = &cf->sg[0];
1188         out_sg->extension = 1;
1189         if (is_encode(ses))
1190                 out_sg->length = sym->auth.data.length + ses->digest_length;
1191         else
1192                 out_sg->length = sym->auth.data.length;
1193
1194         /* output sg entries */
1195         sg = &cf->sg[2];
1196         qm_sg_entry_set64(out_sg, dpaa_mem_vtop(sg));
1197         cpu_to_hw_sg(out_sg);
1198
1199         /* 1st seg */
1200         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1201         sg->length = mbuf->data_len - sym->auth.data.offset;
1202         sg->offset = sym->auth.data.offset;
1203
1204         /* Successive segs */
1205         mbuf = mbuf->next;
1206         while (mbuf) {
1207                 cpu_to_hw_sg(sg);
1208                 sg++;
1209                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1210                 sg->length = mbuf->data_len;
1211                 mbuf = mbuf->next;
1212         }
1213         sg->length -= ses->digest_length;
1214
1215         if (is_encode(ses)) {
1216                 cpu_to_hw_sg(sg);
1217                 /* set auth output */
1218                 sg++;
1219                 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
1220                 sg->length = ses->digest_length;
1221         }
1222         sg->final = 1;
1223         cpu_to_hw_sg(sg);
1224
1225         /* input */
1226         mbuf = sym->m_src;
1227         in_sg = &cf->sg[1];
1228         in_sg->extension = 1;
1229         in_sg->final = 1;
1230         if (is_encode(ses))
1231                 in_sg->length = ses->iv.length + sym->auth.data.length;
1232         else
1233                 in_sg->length = ses->iv.length + sym->auth.data.length
1234                                                 + ses->digest_length;
1235
1236         /* input sg entries */
1237         sg++;
1238         qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg));
1239         cpu_to_hw_sg(in_sg);
1240
1241         /* 1st seg IV */
1242         qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1243         sg->length = ses->iv.length;
1244         cpu_to_hw_sg(sg);
1245
1246         /* 2nd seg */
1247         sg++;
1248         qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1249         sg->length = mbuf->data_len - sym->auth.data.offset;
1250         sg->offset = sym->auth.data.offset;
1251
1252         /* Successive segs */
1253         mbuf = mbuf->next;
1254         while (mbuf) {
1255                 cpu_to_hw_sg(sg);
1256                 sg++;
1257                 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf));
1258                 sg->length = mbuf->data_len;
1259                 mbuf = mbuf->next;
1260         }
1261
1262         sg->length -= ses->digest_length;
1263         if (is_decode(ses)) {
1264                 cpu_to_hw_sg(sg);
1265                 sg++;
1266                 memcpy(ctx->digest, sym->auth.digest.data,
1267                         ses->digest_length);
1268                 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
1269                 sg->length = ses->digest_length;
1270         }
1271         sg->final = 1;
1272         cpu_to_hw_sg(sg);
1273
1274         return cf;
1275 }
1276
1277 static inline struct dpaa_sec_job *
1278 build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses)
1279 {
1280         struct rte_crypto_sym_op *sym = op->sym;
1281         struct dpaa_sec_job *cf;
1282         struct dpaa_sec_op_ctx *ctx;
1283         struct qm_sg_entry *sg;
1284         rte_iova_t src_start_addr, dst_start_addr;
1285         uint32_t length = 0;
1286         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1287                         ses->iv.offset);
1288
1289         src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
1290         if (sym->m_dst)
1291                 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
1292         else
1293                 dst_start_addr = src_start_addr;
1294
1295         ctx = dpaa_sec_alloc_ctx(ses);
1296         if (!ctx)
1297                 return NULL;
1298
1299         cf = &ctx->job;
1300         ctx->op = op;
1301
1302         /* input */
1303         rte_prefetch0(cf->sg);
1304         sg = &cf->sg[2];
1305         qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg));
1306         if (is_encode(ses)) {
1307                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1308                 sg->length = ses->iv.length;
1309                 length += sg->length;
1310                 cpu_to_hw_sg(sg);
1311
1312                 sg++;
1313                 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
1314                 sg->length = sym->auth.data.length;
1315                 length += sg->length;
1316                 sg->final = 1;
1317                 cpu_to_hw_sg(sg);
1318         } else {
1319                 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr));
1320                 sg->length = ses->iv.length;
1321                 length += sg->length;
1322                 cpu_to_hw_sg(sg);
1323
1324                 sg++;
1325
1326                 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
1327                 sg->length = sym->auth.data.length;
1328                 length += sg->length;
1329                 cpu_to_hw_sg(sg);
1330
1331                 memcpy(ctx->digest, sym->auth.digest.data,
1332                        ses->digest_length);
1333                 sg++;
1334
1335                 qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest));
1336                 sg->length = ses->digest_length;
1337                 length += sg->length;
1338                 sg->final = 1;
1339                 cpu_to_hw_sg(sg);
1340         }
1341         /* input compound frame */
1342         cf->sg[1].length = length;
1343         cf->sg[1].extension = 1;
1344         cf->sg[1].final = 1;
1345         cpu_to_hw_sg(&cf->sg[1]);
1346
1347         /* output */
1348         sg++;
1349         qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg));
1350         qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset);
1351         sg->length = sym->cipher.data.length;
1352         length = sg->length;
1353         if (is_encode(ses)) {
1354                 cpu_to_hw_sg(sg);
1355                 /* set auth output */
1356                 sg++;
1357                 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
1358                 sg->length = ses->digest_length;
1359                 length += sg->length;
1360         }
1361         sg->final = 1;
1362         cpu_to_hw_sg(sg);
1363
1364         /* output compound frame */
1365         cf->sg[0].length = length;
1366         cf->sg[0].extension = 1;
1367         cpu_to_hw_sg(&cf->sg[0]);
1368
1369         return cf;
1370 }
1371
1372 static inline struct dpaa_sec_job *
1373 build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses)
1374 {
1375         struct rte_crypto_sym_op *sym = op->sym;
1376         struct dpaa_sec_job *cf;
1377         struct dpaa_sec_op_ctx *ctx;
1378         struct qm_sg_entry *sg;
1379         phys_addr_t src_start_addr, dst_start_addr;
1380
1381         ctx = dpaa_sec_alloc_ctx(ses);
1382         if (!ctx)
1383                 return NULL;
1384         cf = &ctx->job;
1385         ctx->op = op;
1386
1387         src_start_addr = rte_pktmbuf_mtophys(sym->m_src);
1388
1389         if (sym->m_dst)
1390                 dst_start_addr = rte_pktmbuf_mtophys(sym->m_dst);
1391         else
1392                 dst_start_addr = src_start_addr;
1393
1394         /* input */
1395         sg = &cf->sg[1];
1396         qm_sg_entry_set64(sg, src_start_addr);
1397         sg->length = sym->m_src->pkt_len;
1398         sg->final = 1;
1399         cpu_to_hw_sg(sg);
1400
1401         sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
1402         /* output */
1403         sg = &cf->sg[0];
1404         qm_sg_entry_set64(sg, dst_start_addr);
1405         sg->length = sym->m_src->buf_len - sym->m_src->data_off;
1406         cpu_to_hw_sg(sg);
1407
1408         return cf;
1409 }
1410
1411 static uint16_t
1412 dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1413                        uint16_t nb_ops)
1414 {
1415         /* Function to transmit the frames to given device and queuepair */
1416         uint32_t loop;
1417         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp;
1418         uint16_t num_tx = 0;
1419         struct qm_fd fds[DPAA_SEC_BURST], *fd;
1420         uint32_t frames_to_send;
1421         struct rte_crypto_op *op;
1422         struct dpaa_sec_job *cf;
1423         dpaa_sec_session *ses;
1424         uint32_t auth_only_len;
1425         struct qman_fq *inq[DPAA_SEC_BURST];
1426
1427         while (nb_ops) {
1428                 frames_to_send = (nb_ops > DPAA_SEC_BURST) ?
1429                                 DPAA_SEC_BURST : nb_ops;
1430                 for (loop = 0; loop < frames_to_send; loop++) {
1431                         op = *(ops++);
1432                         switch (op->sess_type) {
1433                         case RTE_CRYPTO_OP_WITH_SESSION:
1434                                 ses = (dpaa_sec_session *)
1435                                         get_sym_session_private_data(
1436                                                         op->sym->session,
1437                                                         cryptodev_driver_id);
1438                                 break;
1439                         case RTE_CRYPTO_OP_SECURITY_SESSION:
1440                                 ses = (dpaa_sec_session *)
1441                                         get_sec_session_private_data(
1442                                                         op->sym->sec_session);
1443                                 break;
1444                         default:
1445                                 DPAA_SEC_DP_ERR(
1446                                         "sessionless crypto op not supported");
1447                                 frames_to_send = loop;
1448                                 nb_ops = loop;
1449                                 goto send_pkts;
1450                         }
1451                         if (unlikely(!ses->qp)) {
1452                                 if (dpaa_sec_attach_sess_q(qp, ses)) {
1453                                         frames_to_send = loop;
1454                                         nb_ops = loop;
1455                                         goto send_pkts;
1456                                 }
1457                         } else if (unlikely(ses->qp != qp)) {
1458                                 DPAA_SEC_DP_ERR("Old:sess->qp = %p"
1459                                         " New qp = %p\n", ses->qp, qp);
1460                                 frames_to_send = loop;
1461                                 nb_ops = loop;
1462                                 goto send_pkts;
1463                         }
1464
1465                         auth_only_len = op->sym->auth.data.length -
1466                                                 op->sym->cipher.data.length;
1467                         if (rte_pktmbuf_is_contiguous(op->sym->m_src)) {
1468                                 if (is_auth_only(ses)) {
1469                                         cf = build_auth_only(op, ses);
1470                                 } else if (is_cipher_only(ses)) {
1471                                         cf = build_cipher_only(op, ses);
1472                                 } else if (is_aead(ses)) {
1473                                         cf = build_cipher_auth_gcm(op, ses);
1474                                         auth_only_len = ses->auth_only_len;
1475                                 } else if (is_auth_cipher(ses)) {
1476                                         cf = build_cipher_auth(op, ses);
1477                                 } else if (is_proto_ipsec(ses)) {
1478                                         cf = build_proto(op, ses);
1479                                 } else {
1480                                         DPAA_SEC_DP_ERR("not supported ops");
1481                                         frames_to_send = loop;
1482                                         nb_ops = loop;
1483                                         goto send_pkts;
1484                                 }
1485                         } else {
1486                                 if (is_auth_only(ses)) {
1487                                         cf = build_auth_only_sg(op, ses);
1488                                 } else if (is_cipher_only(ses)) {
1489                                         cf = build_cipher_only_sg(op, ses);
1490                                 } else if (is_aead(ses)) {
1491                                         cf = build_cipher_auth_gcm_sg(op, ses);
1492                                         auth_only_len = ses->auth_only_len;
1493                                 } else if (is_auth_cipher(ses)) {
1494                                         cf = build_cipher_auth_sg(op, ses);
1495                                 } else {
1496                                         DPAA_SEC_DP_ERR("not supported ops");
1497                                         frames_to_send = loop;
1498                                         nb_ops = loop;
1499                                         goto send_pkts;
1500                                 }
1501                         }
1502                         if (unlikely(!cf)) {
1503                                 frames_to_send = loop;
1504                                 nb_ops = loop;
1505                                 goto send_pkts;
1506                         }
1507
1508                         fd = &fds[loop];
1509                         inq[loop] = ses->inq;
1510                         fd->opaque_addr = 0;
1511                         fd->cmd = 0;
1512                         qm_fd_addr_set64(fd, dpaa_mem_vtop(cf->sg));
1513                         fd->_format1 = qm_fd_compound;
1514                         fd->length29 = 2 * sizeof(struct qm_sg_entry);
1515                         /* Auth_only_len is set as 0 in descriptor and it is
1516                          * overwritten here in the fd.cmd which will update
1517                          * the DPOVRD reg.
1518                          */
1519                         if (auth_only_len)
1520                                 fd->cmd = 0x80000000 | auth_only_len;
1521
1522                 }
1523 send_pkts:
1524                 loop = 0;
1525                 while (loop < frames_to_send) {
1526                         loop += qman_enqueue_multi_fq(&inq[loop], &fds[loop],
1527                                         frames_to_send - loop);
1528                 }
1529                 nb_ops -= frames_to_send;
1530                 num_tx += frames_to_send;
1531         }
1532
1533         dpaa_qp->tx_pkts += num_tx;
1534         dpaa_qp->tx_errs += nb_ops - num_tx;
1535
1536         return num_tx;
1537 }
1538
1539 static uint16_t
1540 dpaa_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops,
1541                        uint16_t nb_ops)
1542 {
1543         uint16_t num_rx;
1544         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp;
1545
1546         num_rx = dpaa_sec_deq(dpaa_qp, ops, nb_ops);
1547
1548         dpaa_qp->rx_pkts += num_rx;
1549         dpaa_qp->rx_errs += nb_ops - num_rx;
1550
1551         DPAA_SEC_DP_DEBUG("SEC Received %d Packets\n", num_rx);
1552
1553         return num_rx;
1554 }
1555
1556 /** Release queue pair */
1557 static int
1558 dpaa_sec_queue_pair_release(struct rte_cryptodev *dev,
1559                             uint16_t qp_id)
1560 {
1561         struct dpaa_sec_dev_private *internals;
1562         struct dpaa_sec_qp *qp = NULL;
1563
1564         PMD_INIT_FUNC_TRACE();
1565
1566         DPAA_SEC_DEBUG("dev =%p, queue =%d", dev, qp_id);
1567
1568         internals = dev->data->dev_private;
1569         if (qp_id >= internals->max_nb_queue_pairs) {
1570                 DPAA_SEC_ERR("Max supported qpid %d",
1571                              internals->max_nb_queue_pairs);
1572                 return -EINVAL;
1573         }
1574
1575         qp = &internals->qps[qp_id];
1576         qp->internals = NULL;
1577         dev->data->queue_pairs[qp_id] = NULL;
1578
1579         return 0;
1580 }
1581
1582 /** Setup a queue pair */
1583 static int
1584 dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
1585                 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
1586                 __rte_unused int socket_id,
1587                 __rte_unused struct rte_mempool *session_pool)
1588 {
1589         struct dpaa_sec_dev_private *internals;
1590         struct dpaa_sec_qp *qp = NULL;
1591
1592         DPAA_SEC_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);
1593
1594         internals = dev->data->dev_private;
1595         if (qp_id >= internals->max_nb_queue_pairs) {
1596                 DPAA_SEC_ERR("Max supported qpid %d",
1597                              internals->max_nb_queue_pairs);
1598                 return -EINVAL;
1599         }
1600
1601         qp = &internals->qps[qp_id];
1602         qp->internals = internals;
1603         dev->data->queue_pairs[qp_id] = qp;
1604
1605         return 0;
1606 }
1607
1608 /** Return the number of allocated queue pairs */
1609 static uint32_t
1610 dpaa_sec_queue_pair_count(struct rte_cryptodev *dev)
1611 {
1612         PMD_INIT_FUNC_TRACE();
1613
1614         return dev->data->nb_queue_pairs;
1615 }
1616
1617 /** Returns the size of session structure */
1618 static unsigned int
1619 dpaa_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
1620 {
1621         PMD_INIT_FUNC_TRACE();
1622
1623         return sizeof(dpaa_sec_session);
1624 }
1625
1626 static int
1627 dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused,
1628                      struct rte_crypto_sym_xform *xform,
1629                      dpaa_sec_session *session)
1630 {
1631         session->cipher_alg = xform->cipher.algo;
1632         session->iv.length = xform->cipher.iv.length;
1633         session->iv.offset = xform->cipher.iv.offset;
1634         session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
1635                                                RTE_CACHE_LINE_SIZE);
1636         if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
1637                 DPAA_SEC_ERR("No Memory for cipher key");
1638                 return -ENOMEM;
1639         }
1640         session->cipher_key.length = xform->cipher.key.length;
1641
1642         memcpy(session->cipher_key.data, xform->cipher.key.data,
1643                xform->cipher.key.length);
1644         session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
1645                         DIR_ENC : DIR_DEC;
1646
1647         return 0;
1648 }
1649
1650 static int
1651 dpaa_sec_auth_init(struct rte_cryptodev *dev __rte_unused,
1652                    struct rte_crypto_sym_xform *xform,
1653                    dpaa_sec_session *session)
1654 {
1655         session->auth_alg = xform->auth.algo;
1656         session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
1657                                              RTE_CACHE_LINE_SIZE);
1658         if (session->auth_key.data == NULL && xform->auth.key.length > 0) {
1659                 DPAA_SEC_ERR("No Memory for auth key");
1660                 return -ENOMEM;
1661         }
1662         session->auth_key.length = xform->auth.key.length;
1663         session->digest_length = xform->auth.digest_length;
1664
1665         memcpy(session->auth_key.data, xform->auth.key.data,
1666                xform->auth.key.length);
1667         session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
1668                         DIR_ENC : DIR_DEC;
1669
1670         return 0;
1671 }
1672
1673 static int
1674 dpaa_sec_aead_init(struct rte_cryptodev *dev __rte_unused,
1675                    struct rte_crypto_sym_xform *xform,
1676                    dpaa_sec_session *session)
1677 {
1678         session->aead_alg = xform->aead.algo;
1679         session->iv.length = xform->aead.iv.length;
1680         session->iv.offset = xform->aead.iv.offset;
1681         session->auth_only_len = xform->aead.aad_length;
1682         session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length,
1683                                              RTE_CACHE_LINE_SIZE);
1684         if (session->aead_key.data == NULL && xform->aead.key.length > 0) {
1685                 DPAA_SEC_ERR("No Memory for aead key\n");
1686                 return -ENOMEM;
1687         }
1688         session->aead_key.length = xform->aead.key.length;
1689         session->digest_length = xform->aead.digest_length;
1690
1691         memcpy(session->aead_key.data, xform->aead.key.data,
1692                xform->aead.key.length);
1693         session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
1694                         DIR_ENC : DIR_DEC;
1695
1696         return 0;
1697 }
1698
1699 static struct qman_fq *
1700 dpaa_sec_attach_rxq(struct dpaa_sec_dev_private *qi)
1701 {
1702         unsigned int i;
1703
1704         for (i = 0; i < qi->max_nb_sessions; i++) {
1705                 if (qi->inq_attach[i] == 0) {
1706                         qi->inq_attach[i] = 1;
1707                         return &qi->inq[i];
1708                 }
1709         }
1710         DPAA_SEC_WARN("All ses session in use %x", qi->max_nb_sessions);
1711
1712         return NULL;
1713 }
1714
1715 static int
1716 dpaa_sec_detach_rxq(struct dpaa_sec_dev_private *qi, struct qman_fq *fq)
1717 {
1718         unsigned int i;
1719
1720         for (i = 0; i < qi->max_nb_sessions; i++) {
1721                 if (&qi->inq[i] == fq) {
1722                         qman_retire_fq(fq, NULL);
1723                         qman_oos_fq(fq);
1724                         qi->inq_attach[i] = 0;
1725                         return 0;
1726                 }
1727         }
1728         return -1;
1729 }
1730
1731 static int
1732 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess)
1733 {
1734         int ret;
1735
1736         sess->qp = qp;
1737         ret = dpaa_sec_prep_cdb(sess);
1738         if (ret) {
1739                 DPAA_SEC_ERR("Unable to prepare sec cdb");
1740                 return -1;
1741         }
1742         if (unlikely(!RTE_PER_LCORE(dpaa_io))) {
1743                 ret = rte_dpaa_portal_init((void *)0);
1744                 if (ret) {
1745                         DPAA_SEC_ERR("Failure in affining portal");
1746                         return ret;
1747                 }
1748         }
1749         ret = dpaa_sec_init_rx(sess->inq, dpaa_mem_vtop(&sess->cdb),
1750                                qman_fq_fqid(&qp->outq));
1751         if (ret)
1752                 DPAA_SEC_ERR("Unable to init sec queue");
1753
1754         return ret;
1755 }
1756
1757 static int
1758 dpaa_sec_set_session_parameters(struct rte_cryptodev *dev,
1759                             struct rte_crypto_sym_xform *xform, void *sess)
1760 {
1761         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
1762         dpaa_sec_session *session = sess;
1763
1764         PMD_INIT_FUNC_TRACE();
1765
1766         if (unlikely(sess == NULL)) {
1767                 DPAA_SEC_ERR("invalid session struct");
1768                 return -EINVAL;
1769         }
1770         memset(session, 0, sizeof(dpaa_sec_session));
1771
1772         /* Default IV length = 0 */
1773         session->iv.length = 0;
1774
1775         /* Cipher Only */
1776         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
1777                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1778                 dpaa_sec_cipher_init(dev, xform, session);
1779
1780         /* Authentication Only */
1781         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1782                    xform->next == NULL) {
1783                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
1784                 dpaa_sec_auth_init(dev, xform, session);
1785
1786         /* Cipher then Authenticate */
1787         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
1788                    xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
1789                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1790                         dpaa_sec_cipher_init(dev, xform, session);
1791                         dpaa_sec_auth_init(dev, xform->next, session);
1792                 } else {
1793                         DPAA_SEC_ERR("Not supported: Auth then Cipher");
1794                         return -EINVAL;
1795                 }
1796
1797         /* Authenticate then Cipher */
1798         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
1799                    xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
1800                 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
1801                         dpaa_sec_auth_init(dev, xform, session);
1802                         dpaa_sec_cipher_init(dev, xform->next, session);
1803                 } else {
1804                         DPAA_SEC_ERR("Not supported: Auth then Cipher");
1805                         return -EINVAL;
1806                 }
1807
1808         /* AEAD operation for AES-GCM kind of Algorithms */
1809         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
1810                    xform->next == NULL) {
1811                 dpaa_sec_aead_init(dev, xform, session);
1812
1813         } else {
1814                 DPAA_SEC_ERR("Invalid crypto type");
1815                 return -EINVAL;
1816         }
1817         session->ctx_pool = internals->ctx_pool;
1818         rte_spinlock_lock(&internals->lock);
1819         session->inq = dpaa_sec_attach_rxq(internals);
1820         rte_spinlock_unlock(&internals->lock);
1821         if (session->inq == NULL) {
1822                 DPAA_SEC_ERR("unable to attach sec queue");
1823                 goto err1;
1824         }
1825
1826         return 0;
1827
1828 err1:
1829         rte_free(session->cipher_key.data);
1830         rte_free(session->auth_key.data);
1831         memset(session, 0, sizeof(dpaa_sec_session));
1832
1833         return -EINVAL;
1834 }
1835
1836 static int
1837 dpaa_sec_sym_session_configure(struct rte_cryptodev *dev,
1838                 struct rte_crypto_sym_xform *xform,
1839                 struct rte_cryptodev_sym_session *sess,
1840                 struct rte_mempool *mempool)
1841 {
1842         void *sess_private_data;
1843         int ret;
1844
1845         PMD_INIT_FUNC_TRACE();
1846
1847         if (rte_mempool_get(mempool, &sess_private_data)) {
1848                 DPAA_SEC_ERR("Couldn't get object from session mempool");
1849                 return -ENOMEM;
1850         }
1851
1852         ret = dpaa_sec_set_session_parameters(dev, xform, sess_private_data);
1853         if (ret != 0) {
1854                 DPAA_SEC_ERR("failed to configure session parameters");
1855
1856                 /* Return session to mempool */
1857                 rte_mempool_put(mempool, sess_private_data);
1858                 return ret;
1859         }
1860
1861         set_sym_session_private_data(sess, dev->driver_id,
1862                         sess_private_data);
1863
1864
1865         return 0;
1866 }
1867
1868 /** Clear the memory of session so it doesn't leave key material behind */
1869 static void
1870 dpaa_sec_sym_session_clear(struct rte_cryptodev *dev,
1871                 struct rte_cryptodev_sym_session *sess)
1872 {
1873         struct dpaa_sec_dev_private *qi = dev->data->dev_private;
1874         uint8_t index = dev->driver_id;
1875         void *sess_priv = get_sym_session_private_data(sess, index);
1876
1877         PMD_INIT_FUNC_TRACE();
1878
1879         dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
1880
1881         if (sess_priv) {
1882                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
1883
1884                 if (s->inq)
1885                         dpaa_sec_detach_rxq(qi, s->inq);
1886                 rte_free(s->cipher_key.data);
1887                 rte_free(s->auth_key.data);
1888                 memset(s, 0, sizeof(dpaa_sec_session));
1889                 set_sym_session_private_data(sess, index, NULL);
1890                 rte_mempool_put(sess_mp, sess_priv);
1891         }
1892 }
1893
1894 static int
1895 dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
1896                            struct rte_security_session_conf *conf,
1897                            void *sess)
1898 {
1899         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
1900         struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
1901         struct rte_crypto_auth_xform *auth_xform;
1902         struct rte_crypto_cipher_xform *cipher_xform;
1903         dpaa_sec_session *session = (dpaa_sec_session *)sess;
1904
1905         PMD_INIT_FUNC_TRACE();
1906
1907         memset(session, 0, sizeof(dpaa_sec_session));
1908         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
1909                 cipher_xform = &conf->crypto_xform->cipher;
1910                 auth_xform = &conf->crypto_xform->next->auth;
1911         } else {
1912                 auth_xform = &conf->crypto_xform->auth;
1913                 cipher_xform = &conf->crypto_xform->next->cipher;
1914         }
1915         session->proto_alg = conf->protocol;
1916         session->cipher_key.data = rte_zmalloc(NULL,
1917                                                cipher_xform->key.length,
1918                                                RTE_CACHE_LINE_SIZE);
1919         if (session->cipher_key.data == NULL &&
1920                         cipher_xform->key.length > 0) {
1921                 DPAA_SEC_ERR("No Memory for cipher key");
1922                 return -ENOMEM;
1923         }
1924
1925         session->cipher_key.length = cipher_xform->key.length;
1926         session->auth_key.data = rte_zmalloc(NULL,
1927                                         auth_xform->key.length,
1928                                         RTE_CACHE_LINE_SIZE);
1929         if (session->auth_key.data == NULL &&
1930                         auth_xform->key.length > 0) {
1931                 DPAA_SEC_ERR("No Memory for auth key");
1932                 rte_free(session->cipher_key.data);
1933                 return -ENOMEM;
1934         }
1935         session->auth_key.length = auth_xform->key.length;
1936         memcpy(session->cipher_key.data, cipher_xform->key.data,
1937                         cipher_xform->key.length);
1938         memcpy(session->auth_key.data, auth_xform->key.data,
1939                         auth_xform->key.length);
1940
1941         switch (auth_xform->algo) {
1942         case RTE_CRYPTO_AUTH_SHA1_HMAC:
1943                 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
1944                 break;
1945         case RTE_CRYPTO_AUTH_MD5_HMAC:
1946                 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
1947                 break;
1948         case RTE_CRYPTO_AUTH_SHA256_HMAC:
1949                 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
1950                 break;
1951         case RTE_CRYPTO_AUTH_SHA384_HMAC:
1952                 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
1953                 break;
1954         case RTE_CRYPTO_AUTH_SHA512_HMAC:
1955                 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
1956                 break;
1957         case RTE_CRYPTO_AUTH_AES_CMAC:
1958                 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
1959                 break;
1960         case RTE_CRYPTO_AUTH_NULL:
1961                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
1962                 break;
1963         case RTE_CRYPTO_AUTH_SHA224_HMAC:
1964         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
1965         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
1966         case RTE_CRYPTO_AUTH_SHA1:
1967         case RTE_CRYPTO_AUTH_SHA256:
1968         case RTE_CRYPTO_AUTH_SHA512:
1969         case RTE_CRYPTO_AUTH_SHA224:
1970         case RTE_CRYPTO_AUTH_SHA384:
1971         case RTE_CRYPTO_AUTH_MD5:
1972         case RTE_CRYPTO_AUTH_AES_GMAC:
1973         case RTE_CRYPTO_AUTH_KASUMI_F9:
1974         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
1975         case RTE_CRYPTO_AUTH_ZUC_EIA3:
1976                 DPAA_SEC_ERR("Crypto: Unsupported auth alg %u",
1977                         auth_xform->algo);
1978                 goto out;
1979         default:
1980                 DPAA_SEC_ERR("Crypto: Undefined Auth specified %u",
1981                         auth_xform->algo);
1982                 goto out;
1983         }
1984
1985         switch (cipher_xform->algo) {
1986         case RTE_CRYPTO_CIPHER_AES_CBC:
1987                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
1988                 break;
1989         case RTE_CRYPTO_CIPHER_3DES_CBC:
1990                 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
1991                 break;
1992         case RTE_CRYPTO_CIPHER_AES_CTR:
1993                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
1994                 break;
1995         case RTE_CRYPTO_CIPHER_NULL:
1996         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
1997         case RTE_CRYPTO_CIPHER_3DES_ECB:
1998         case RTE_CRYPTO_CIPHER_AES_ECB:
1999         case RTE_CRYPTO_CIPHER_KASUMI_F8:
2000                 DPAA_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2001                         cipher_xform->algo);
2002                 goto out;
2003         default:
2004                 DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u",
2005                         cipher_xform->algo);
2006                 goto out;
2007         }
2008
2009         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
2010                 memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) +
2011                                 sizeof(session->ip4_hdr));
2012                 session->ip4_hdr.ip_v = IPVERSION;
2013                 session->ip4_hdr.ip_hl = 5;
2014                 session->ip4_hdr.ip_len = rte_cpu_to_be_16(
2015                                                 sizeof(session->ip4_hdr));
2016                 session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
2017                 session->ip4_hdr.ip_id = 0;
2018                 session->ip4_hdr.ip_off = 0;
2019                 session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
2020                 session->ip4_hdr.ip_p = (ipsec_xform->proto ==
2021                                 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP
2022                                 : IPPROTO_AH;
2023                 session->ip4_hdr.ip_sum = 0;
2024                 session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
2025                 session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
2026                 session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
2027                                                 (void *)&session->ip4_hdr,
2028                                                 sizeof(struct ip));
2029
2030                 session->encap_pdb.options =
2031                         (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
2032                         PDBOPTS_ESP_OIHI_PDB_INL |
2033                         PDBOPTS_ESP_IVSRC |
2034                         PDBHMO_ESP_ENCAP_DTTL |
2035                         PDBHMO_ESP_SNR;
2036                 session->encap_pdb.spi = ipsec_xform->spi;
2037                 session->encap_pdb.ip_hdr_len = sizeof(struct ip);
2038
2039                 session->dir = DIR_ENC;
2040         } else if (ipsec_xform->direction ==
2041                         RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
2042                 memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
2043                 session->decap_pdb.options = sizeof(struct ip) << 16;
2044                 session->dir = DIR_DEC;
2045         } else
2046                 goto out;
2047         session->ctx_pool = internals->ctx_pool;
2048         rte_spinlock_lock(&internals->lock);
2049         session->inq = dpaa_sec_attach_rxq(internals);
2050         rte_spinlock_unlock(&internals->lock);
2051         if (session->inq == NULL) {
2052                 DPAA_SEC_ERR("unable to attach sec queue");
2053                 goto out;
2054         }
2055
2056
2057         return 0;
2058 out:
2059         rte_free(session->auth_key.data);
2060         rte_free(session->cipher_key.data);
2061         memset(session, 0, sizeof(dpaa_sec_session));
2062         return -1;
2063 }
2064
2065 static int
2066 dpaa_sec_security_session_create(void *dev,
2067                                  struct rte_security_session_conf *conf,
2068                                  struct rte_security_session *sess,
2069                                  struct rte_mempool *mempool)
2070 {
2071         void *sess_private_data;
2072         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
2073         int ret;
2074
2075         if (rte_mempool_get(mempool, &sess_private_data)) {
2076                 DPAA_SEC_ERR("Couldn't get object from session mempool");
2077                 return -ENOMEM;
2078         }
2079
2080         switch (conf->protocol) {
2081         case RTE_SECURITY_PROTOCOL_IPSEC:
2082                 ret = dpaa_sec_set_ipsec_session(cdev, conf,
2083                                 sess_private_data);
2084                 break;
2085         case RTE_SECURITY_PROTOCOL_MACSEC:
2086                 return -ENOTSUP;
2087         default:
2088                 return -EINVAL;
2089         }
2090         if (ret != 0) {
2091                 DPAA_SEC_ERR("failed to configure session parameters");
2092                 /* Return session to mempool */
2093                 rte_mempool_put(mempool, sess_private_data);
2094                 return ret;
2095         }
2096
2097         set_sec_session_private_data(sess, sess_private_data);
2098
2099         return ret;
2100 }
2101
2102 /** Clear the memory of session so it doesn't leave key material behind */
2103 static int
2104 dpaa_sec_security_session_destroy(void *dev __rte_unused,
2105                 struct rte_security_session *sess)
2106 {
2107         PMD_INIT_FUNC_TRACE();
2108         void *sess_priv = get_sec_session_private_data(sess);
2109
2110         dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
2111
2112         if (sess_priv) {
2113                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
2114
2115                 rte_free(s->cipher_key.data);
2116                 rte_free(s->auth_key.data);
2117                 memset(sess, 0, sizeof(dpaa_sec_session));
2118                 set_sec_session_private_data(sess, NULL);
2119                 rte_mempool_put(sess_mp, sess_priv);
2120         }
2121         return 0;
2122 }
2123
2124
2125 static int
2126 dpaa_sec_dev_configure(struct rte_cryptodev *dev,
2127                        struct rte_cryptodev_config *config __rte_unused)
2128 {
2129
2130         char str[20];
2131         struct dpaa_sec_dev_private *internals;
2132
2133         PMD_INIT_FUNC_TRACE();
2134
2135         internals = dev->data->dev_private;
2136         sprintf(str, "ctx_pool_%d", dev->data->dev_id);
2137         if (!internals->ctx_pool) {
2138                 internals->ctx_pool = rte_mempool_create((const char *)str,
2139                                                         CTX_POOL_NUM_BUFS,
2140                                                         CTX_POOL_BUF_SIZE,
2141                                                         CTX_POOL_CACHE_SIZE, 0,
2142                                                         NULL, NULL, NULL, NULL,
2143                                                         SOCKET_ID_ANY, 0);
2144                 if (!internals->ctx_pool) {
2145                         DPAA_SEC_ERR("%s create failed\n", str);
2146                         return -ENOMEM;
2147                 }
2148         } else
2149                 DPAA_SEC_INFO("mempool already created for dev_id : %d",
2150                                 dev->data->dev_id);
2151
2152         return 0;
2153 }
2154
2155 static int
2156 dpaa_sec_dev_start(struct rte_cryptodev *dev __rte_unused)
2157 {
2158         PMD_INIT_FUNC_TRACE();
2159         return 0;
2160 }
2161
2162 static void
2163 dpaa_sec_dev_stop(struct rte_cryptodev *dev __rte_unused)
2164 {
2165         PMD_INIT_FUNC_TRACE();
2166 }
2167
2168 static int
2169 dpaa_sec_dev_close(struct rte_cryptodev *dev)
2170 {
2171         struct dpaa_sec_dev_private *internals;
2172
2173         PMD_INIT_FUNC_TRACE();
2174
2175         if (dev == NULL)
2176                 return -ENOMEM;
2177
2178         internals = dev->data->dev_private;
2179         rte_mempool_free(internals->ctx_pool);
2180         internals->ctx_pool = NULL;
2181
2182         return 0;
2183 }
2184
2185 static void
2186 dpaa_sec_dev_infos_get(struct rte_cryptodev *dev,
2187                        struct rte_cryptodev_info *info)
2188 {
2189         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
2190
2191         PMD_INIT_FUNC_TRACE();
2192         if (info != NULL) {
2193                 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
2194                 info->feature_flags = dev->feature_flags;
2195                 info->capabilities = dpaa_sec_capabilities;
2196                 info->sym.max_nb_sessions = internals->max_nb_sessions;
2197                 info->driver_id = cryptodev_driver_id;
2198         }
2199 }
2200
2201 static struct rte_cryptodev_ops crypto_ops = {
2202         .dev_configure        = dpaa_sec_dev_configure,
2203         .dev_start            = dpaa_sec_dev_start,
2204         .dev_stop             = dpaa_sec_dev_stop,
2205         .dev_close            = dpaa_sec_dev_close,
2206         .dev_infos_get        = dpaa_sec_dev_infos_get,
2207         .queue_pair_setup     = dpaa_sec_queue_pair_setup,
2208         .queue_pair_release   = dpaa_sec_queue_pair_release,
2209         .queue_pair_count     = dpaa_sec_queue_pair_count,
2210         .sym_session_get_size     = dpaa_sec_sym_session_get_size,
2211         .sym_session_configure    = dpaa_sec_sym_session_configure,
2212         .sym_session_clear        = dpaa_sec_sym_session_clear
2213 };
2214
2215 static const struct rte_security_capability *
2216 dpaa_sec_capabilities_get(void *device __rte_unused)
2217 {
2218         return dpaa_sec_security_cap;
2219 }
2220
2221 struct rte_security_ops dpaa_sec_security_ops = {
2222         .session_create = dpaa_sec_security_session_create,
2223         .session_update = NULL,
2224         .session_stats_get = NULL,
2225         .session_destroy = dpaa_sec_security_session_destroy,
2226         .set_pkt_metadata = NULL,
2227         .capabilities_get = dpaa_sec_capabilities_get
2228 };
2229
2230 static int
2231 dpaa_sec_uninit(struct rte_cryptodev *dev)
2232 {
2233         struct dpaa_sec_dev_private *internals;
2234
2235         if (dev == NULL)
2236                 return -ENODEV;
2237
2238         internals = dev->data->dev_private;
2239         rte_free(dev->security_ctx);
2240
2241         /* In case close has been called, internals->ctx_pool would be NULL */
2242         rte_mempool_free(internals->ctx_pool);
2243         rte_free(internals);
2244
2245         DPAA_SEC_INFO("Closing DPAA_SEC device %s on numa socket %u",
2246                       dev->data->name, rte_socket_id());
2247
2248         return 0;
2249 }
2250
2251 static int
2252 dpaa_sec_dev_init(struct rte_cryptodev *cryptodev)
2253 {
2254         struct dpaa_sec_dev_private *internals;
2255         struct rte_security_ctx *security_instance;
2256         struct dpaa_sec_qp *qp;
2257         uint32_t i, flags;
2258         int ret;
2259
2260         PMD_INIT_FUNC_TRACE();
2261
2262         cryptodev->driver_id = cryptodev_driver_id;
2263         cryptodev->dev_ops = &crypto_ops;
2264
2265         cryptodev->enqueue_burst = dpaa_sec_enqueue_burst;
2266         cryptodev->dequeue_burst = dpaa_sec_dequeue_burst;
2267         cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2268                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
2269                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2270                         RTE_CRYPTODEV_FF_SECURITY |
2271                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
2272                         RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
2273                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
2274                         RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
2275                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
2276
2277         internals = cryptodev->data->dev_private;
2278         internals->max_nb_queue_pairs = RTE_DPAA_MAX_NB_SEC_QPS;
2279         internals->max_nb_sessions = RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS;
2280
2281         /*
2282          * For secondary processes, we don't initialise any further as primary
2283          * has already done this work. Only check we don't need a different
2284          * RX function
2285          */
2286         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2287                 DPAA_SEC_WARN("Device already init by primary process");
2288                 return 0;
2289         }
2290
2291         /* Initialize security_ctx only for primary process*/
2292         security_instance = rte_malloc("rte_security_instances_ops",
2293                                 sizeof(struct rte_security_ctx), 0);
2294         if (security_instance == NULL)
2295                 return -ENOMEM;
2296         security_instance->device = (void *)cryptodev;
2297         security_instance->ops = &dpaa_sec_security_ops;
2298         security_instance->sess_cnt = 0;
2299         cryptodev->security_ctx = security_instance;
2300
2301         rte_spinlock_init(&internals->lock);
2302         for (i = 0; i < internals->max_nb_queue_pairs; i++) {
2303                 /* init qman fq for queue pair */
2304                 qp = &internals->qps[i];
2305                 ret = dpaa_sec_init_tx(&qp->outq);
2306                 if (ret) {
2307                         DPAA_SEC_ERR("config tx of queue pair  %d", i);
2308                         goto init_error;
2309                 }
2310         }
2311
2312         flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID |
2313                 QMAN_FQ_FLAG_TO_DCPORTAL;
2314         for (i = 0; i < internals->max_nb_sessions; i++) {
2315                 /* create rx qman fq for sessions*/
2316                 ret = qman_create_fq(0, flags, &internals->inq[i]);
2317                 if (unlikely(ret != 0)) {
2318                         DPAA_SEC_ERR("sec qman_create_fq failed");
2319                         goto init_error;
2320                 }
2321         }
2322
2323         RTE_LOG(INFO, PMD, "%s cryptodev init\n", cryptodev->data->name);
2324         return 0;
2325
2326 init_error:
2327         DPAA_SEC_ERR("driver %s: create failed\n", cryptodev->data->name);
2328
2329         dpaa_sec_uninit(cryptodev);
2330         return -EFAULT;
2331 }
2332
2333 static int
2334 cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv __rte_unused,
2335                                 struct rte_dpaa_device *dpaa_dev)
2336 {
2337         struct rte_cryptodev *cryptodev;
2338         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
2339
2340         int retval;
2341
2342         sprintf(cryptodev_name, "dpaa_sec-%d", dpaa_dev->id.dev_id);
2343
2344         cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id());
2345         if (cryptodev == NULL)
2346                 return -ENOMEM;
2347
2348         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
2349                 cryptodev->data->dev_private = rte_zmalloc_socket(
2350                                         "cryptodev private structure",
2351                                         sizeof(struct dpaa_sec_dev_private),
2352                                         RTE_CACHE_LINE_SIZE,
2353                                         rte_socket_id());
2354
2355                 if (cryptodev->data->dev_private == NULL)
2356                         rte_panic("Cannot allocate memzone for private "
2357                                         "device data");
2358         }
2359
2360         dpaa_dev->crypto_dev = cryptodev;
2361         cryptodev->device = &dpaa_dev->device;
2362
2363         /* init user callbacks */
2364         TAILQ_INIT(&(cryptodev->link_intr_cbs));
2365
2366         /* if sec device version is not configured */
2367         if (!rta_get_sec_era()) {
2368                 const struct device_node *caam_node;
2369
2370                 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
2371                         const uint32_t *prop = of_get_property(caam_node,
2372                                         "fsl,sec-era",
2373                                         NULL);
2374                         if (prop) {
2375                                 rta_set_sec_era(
2376                                         INTL_SEC_ERA(rte_cpu_to_be_32(*prop)));
2377                                 break;
2378                         }
2379                 }
2380         }
2381
2382         /* Invoke PMD device initialization function */
2383         retval = dpaa_sec_dev_init(cryptodev);
2384         if (retval == 0)
2385                 return 0;
2386
2387         /* In case of error, cleanup is done */
2388         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
2389                 rte_free(cryptodev->data->dev_private);
2390
2391         rte_cryptodev_pmd_release_device(cryptodev);
2392
2393         return -ENXIO;
2394 }
2395
2396 static int
2397 cryptodev_dpaa_sec_remove(struct rte_dpaa_device *dpaa_dev)
2398 {
2399         struct rte_cryptodev *cryptodev;
2400         int ret;
2401
2402         cryptodev = dpaa_dev->crypto_dev;
2403         if (cryptodev == NULL)
2404                 return -ENODEV;
2405
2406         ret = dpaa_sec_uninit(cryptodev);
2407         if (ret)
2408                 return ret;
2409
2410         return rte_cryptodev_pmd_destroy(cryptodev);
2411 }
2412
2413 static struct rte_dpaa_driver rte_dpaa_sec_driver = {
2414         .drv_type = FSL_DPAA_CRYPTO,
2415         .driver = {
2416                 .name = "DPAA SEC PMD"
2417         },
2418         .probe = cryptodev_dpaa_sec_probe,
2419         .remove = cryptodev_dpaa_sec_remove,
2420 };
2421
2422 static struct cryptodev_driver dpaa_sec_crypto_drv;
2423
2424 RTE_PMD_REGISTER_DPAA(CRYPTODEV_NAME_DPAA_SEC_PMD, rte_dpaa_sec_driver);
2425 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa_sec_crypto_drv, rte_dpaa_sec_driver.driver,
2426                 cryptodev_driver_id);
2427
2428 RTE_INIT(dpaa_sec_init_log)
2429 {
2430         dpaa_logtype_sec = rte_log_register("pmd.crypto.dpaa");
2431         if (dpaa_logtype_sec >= 0)
2432                 rte_log_set_level(dpaa_logtype_sec, RTE_LOG_NOTICE);
2433 }