drivers/crypto/dpaa_sec/dpaa_sec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2017-2021 NXP
   5  *
   6  */
   7
   8 #include <fcntl.h>
   9 #include <unistd.h>
  10 #include <sched.h>
  11 #include <net/if.h>
  12
  13 #include <rte_byteorder.h>
  14 #include <rte_common.h>
  15 #include <cryptodev_pmd.h>
  16 #include <rte_crypto.h>
  17 #include <rte_cryptodev.h>
  18 #ifdef RTE_LIB_SECURITY
  19 #include <rte_security_driver.h>
  20 #endif
  21 #include <rte_cycles.h>
  22 #include <rte_dev.h>
  23 #include <rte_ip.h>
  24 #include <rte_kvargs.h>
  25 #include <rte_malloc.h>
  26 #include <rte_mbuf.h>
  27 #include <rte_memcpy.h>
  28 #include <rte_string_fns.h>
  29 #include <rte_spinlock.h>
  30
  31 #include <fsl_usd.h>
  32 #include <fsl_qman.h>
  33 #include <dpaa_of.h>
  34
  35 /* RTA header files */
  36 #include <desc/common.h>
  37 #include <desc/algo.h>
  38 #include <desc/ipsec.h>
  39 #include <desc/pdcp.h>
  40 #include <desc/sdap.h>
  41
  42 #include <rte_dpaa_bus.h>
  43 #include <dpaa_sec.h>
  44 #include <dpaa_sec_event.h>
  45 #include <dpaa_sec_log.h>
  46 #include <dpaax_iova_table.h>
  47
  48 static uint8_t cryptodev_driver_id;
  49
  50 static int
  51 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess);
  52
  53 static inline void
  54 dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx)
  55 {
  56         if (!ctx->fd_status) {
  57                 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
  58         } else {
  59                 DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status);
  60                 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
  61         }
  62 }
  63
  64 static inline struct dpaa_sec_op_ctx *
  65 dpaa_sec_alloc_ctx(dpaa_sec_session *ses, int sg_count)
  66 {
  67         struct dpaa_sec_op_ctx *ctx;
  68         int i, retval;
  69
  70         retval = rte_mempool_get(
  71                         ses->qp[rte_lcore_id() % MAX_DPAA_CORES]->ctx_pool,
  72                         (void **)(&ctx));
  73         if (!ctx || retval) {
  74                 DPAA_SEC_DP_WARN("Alloc sec descriptor failed!");
  75                 return NULL;
  76         }
  77         /*
  78          * Clear SG memory. There are 16 SG entries of 16 Bytes each.
  79          * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
  80          * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for
  81          * each packet, memset is costlier than dcbz_64().
  82          */
  83         for (i = 0; i < sg_count && i < MAX_JOB_SG_ENTRIES; i += 4)
  84                 dcbz_64(&ctx->job.sg[i]);
  85
  86         ctx->ctx_pool = ses->qp[rte_lcore_id() % MAX_DPAA_CORES]->ctx_pool;
  87         ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx);
  88
  89         return ctx;
  90 }
  91
  92 static void
  93 ern_sec_fq_handler(struct qman_portal *qm __rte_unused,
  94                    struct qman_fq *fq,
  95                    const struct qm_mr_entry *msg)
  96 {
  97         DPAA_SEC_DP_ERR("sec fq %d error, RC = %x, seqnum = %x\n",
  98                         fq->fqid, msg->ern.rc, msg->ern.seqnum);
  99 }
 100
 101 /* initialize the queue with dest chan as caam chan so that
 102  * all the packets in this queue could be dispatched into caam
 103  */
 104 static int
 105 dpaa_sec_init_rx(struct qman_fq *fq_in, rte_iova_t hwdesc,
 106                  uint32_t fqid_out)
 107 {
 108         struct qm_mcc_initfq fq_opts;
 109         uint32_t flags;
 110         int ret = -1;
 111
 112         /* Clear FQ options */
 113         memset(&fq_opts, 0x00, sizeof(struct qm_mcc_initfq));
 114
 115         flags = QMAN_INITFQ_FLAG_SCHED;
 116         fq_opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_CONTEXTA |
 117                           QM_INITFQ_WE_CONTEXTB;
 118
 119         qm_fqd_context_a_set64(&fq_opts.fqd, hwdesc);
 120         fq_opts.fqd.context_b = fqid_out;
 121         fq_opts.fqd.dest.channel = dpaa_get_qm_channel_caam();
 122         fq_opts.fqd.dest.wq = 0;
 123
 124         fq_in->cb.ern  = ern_sec_fq_handler;
 125
 126         DPAA_SEC_DEBUG("in-%x out-%x", fq_in->fqid, fqid_out);
 127
 128         ret = qman_init_fq(fq_in, flags, &fq_opts);
 129         if (unlikely(ret != 0))
 130                 DPAA_SEC_ERR("qman_init_fq failed %d", ret);
 131
 132         return ret;
 133 }
 134
 135 /* something is put into in_fq and caam put the crypto result into out_fq */
 136 static enum qman_cb_dqrr_result
 137 dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused,
 138                   struct qman_fq *fq __always_unused,
 139                   const struct qm_dqrr_entry *dqrr)
 140 {
 141         const struct qm_fd *fd;
 142         struct dpaa_sec_job *job;
 143         struct dpaa_sec_op_ctx *ctx;
 144
 145         if (DPAA_PER_LCORE_DPAA_SEC_OP_NB >= DPAA_SEC_BURST)
 146                 return qman_cb_dqrr_defer;
 147
 148         if (!(dqrr->stat & QM_DQRR_STAT_FD_VALID))
 149                 return qman_cb_dqrr_consume;
 150
 151         fd = &dqrr->fd;
 152         /* sg is embedded in an op ctx,
 153          * sg[0] is for output
 154          * sg[1] for input
 155          */
 156         job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd));
 157
 158         ctx = container_of(job, struct dpaa_sec_op_ctx, job);
 159         ctx->fd_status = fd->status;
 160         if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 161                 struct qm_sg_entry *sg_out;
 162                 uint32_t len;
 163                 struct rte_mbuf *mbuf = (ctx->op->sym->m_dst == NULL) ?
 164                                 ctx->op->sym->m_src : ctx->op->sym->m_dst;
 165
 166                 sg_out = &job->sg[0];
 167                 hw_sg_to_cpu(sg_out);
 168                 len = sg_out->length;
 169                 mbuf->pkt_len = len;
 170                 while (mbuf->next != NULL) {
 171                         len -= mbuf->data_len;
 172                         mbuf = mbuf->next;
 173                 }
 174                 mbuf->data_len = len;
 175         }
 176         DPAA_PER_LCORE_RTE_CRYPTO_OP[DPAA_PER_LCORE_DPAA_SEC_OP_NB++] = ctx->op;
 177         dpaa_sec_op_ending(ctx);
 178
 179         return qman_cb_dqrr_consume;
 180 }
 181
 182 /* caam result is put into this queue */
 183 static int
 184 dpaa_sec_init_tx(struct qman_fq *fq)
 185 {
 186         int ret;
 187         struct qm_mcc_initfq opts;
 188         uint32_t flags;
 189
 190         flags = QMAN_FQ_FLAG_NO_ENQUEUE | QMAN_FQ_FLAG_LOCKED |
 191                 QMAN_FQ_FLAG_DYNAMIC_FQID;
 192
 193         ret = qman_create_fq(0, flags, fq);
 194         if (unlikely(ret)) {
 195                 DPAA_SEC_ERR("qman_create_fq failed");
 196                 return ret;
 197         }
 198
 199         memset(&opts, 0, sizeof(opts));
 200         opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
 201                        QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB;
 202
 203         /* opts.fqd.dest.channel = dpaa_sec_pool_chan; */
 204
 205         fq->cb.dqrr = dqrr_out_fq_cb_rx;
 206         fq->cb.ern  = ern_sec_fq_handler;
 207
 208         ret = qman_init_fq(fq, 0, &opts);
 209         if (unlikely(ret)) {
 210                 DPAA_SEC_ERR("unable to init caam source fq!");
 211                 return ret;
 212         }
 213
 214         return ret;
 215 }
 216
 217 static inline int is_aead(dpaa_sec_session *ses)
 218 {
 219         return ((ses->cipher_alg == 0) &&
 220                 (ses->auth_alg == 0) &&
 221                 (ses->aead_alg != 0));
 222 }
 223
 224 static inline int is_encode(dpaa_sec_session *ses)
 225 {
 226         return ses->dir == DIR_ENC;
 227 }
 228
 229 static inline int is_decode(dpaa_sec_session *ses)
 230 {
 231         return ses->dir == DIR_DEC;
 232 }
 233
 234 #ifdef RTE_LIB_SECURITY
 235 static int
 236 dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses)
 237 {
 238         struct alginfo authdata = {0}, cipherdata = {0};
 239         struct sec_cdb *cdb = &ses->cdb;
 240         struct alginfo *p_authdata = NULL;
 241         int32_t shared_desc_len = 0;
 242 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 243         int swap = false;
 244 #else
 245         int swap = true;
 246 #endif
 247
 248         cipherdata.key = (size_t)ses->cipher_key.data;
 249         cipherdata.keylen = ses->cipher_key.length;
 250         cipherdata.key_enc_flags = 0;
 251         cipherdata.key_type = RTA_DATA_IMM;
 252         cipherdata.algtype = ses->cipher_key.alg;
 253         cipherdata.algmode = ses->cipher_key.algmode;
 254
 255         if (ses->auth_alg) {
 256                 authdata.key = (size_t)ses->auth_key.data;
 257                 authdata.keylen = ses->auth_key.length;
 258                 authdata.key_enc_flags = 0;
 259                 authdata.key_type = RTA_DATA_IMM;
 260                 authdata.algtype = ses->auth_key.alg;
 261                 authdata.algmode = ses->auth_key.algmode;
 262
 263                 p_authdata = &authdata;
 264         }
 265
 266         if (ses->pdcp.sdap_enabled) {
 267                 int nb_keys_to_inline =
 268                                 rta_inline_pdcp_sdap_query(authdata.algtype,
 269                                         cipherdata.algtype,
 270                                         ses->pdcp.sn_size,
 271                                         ses->pdcp.hfn_ovd);
 272                 if (nb_keys_to_inline >= 1) {
 273                         cipherdata.key = (size_t)rte_dpaa_mem_vtop((void *)
 274                                                 (size_t)cipherdata.key);
 275                         cipherdata.key_type = RTA_DATA_PTR;
 276                 }
 277                 if (nb_keys_to_inline >= 2) {
 278                         authdata.key = (size_t)rte_dpaa_mem_vtop((void *)
 279                                                 (size_t)authdata.key);
 280                         authdata.key_type = RTA_DATA_PTR;
 281                 }
 282         } else {
 283                 if (rta_inline_pdcp_query(authdata.algtype,
 284                                         cipherdata.algtype,
 285                                         ses->pdcp.sn_size,
 286                                         ses->pdcp.hfn_ovd)) {
 287                         cipherdata.key = (size_t)rte_dpaa_mem_vtop((void *)
 288                                                 (size_t)cipherdata.key);
 289                         cipherdata.key_type = RTA_DATA_PTR;
 290                 }
 291         }
 292
 293         if (ses->pdcp.domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
 294                 if (ses->dir == DIR_ENC)
 295                         shared_desc_len = cnstr_shdsc_pdcp_c_plane_encap(
 296                                         cdb->sh_desc, 1, swap,
 297                                         ses->pdcp.hfn,
 298                                         ses->pdcp.sn_size,
 299                                         ses->pdcp.bearer,
 300                                         ses->pdcp.pkt_dir,
 301                                         ses->pdcp.hfn_threshold,
 302                                         &cipherdata, &authdata,
 303                                         0);
 304                 else if (ses->dir == DIR_DEC)
 305                         shared_desc_len = cnstr_shdsc_pdcp_c_plane_decap(
 306                                         cdb->sh_desc, 1, swap,
 307                                         ses->pdcp.hfn,
 308                                         ses->pdcp.sn_size,
 309                                         ses->pdcp.bearer,
 310                                         ses->pdcp.pkt_dir,
 311                                         ses->pdcp.hfn_threshold,
 312                                         &cipherdata, &authdata,
 313                                         0);
 314         } else if (ses->pdcp.domain == RTE_SECURITY_PDCP_MODE_SHORT_MAC) {
 315                 shared_desc_len = cnstr_shdsc_pdcp_short_mac(cdb->sh_desc,
 316                                                      1, swap, &authdata);
 317         } else {
 318                 if (ses->dir == DIR_ENC) {
 319                         if (ses->pdcp.sdap_enabled)
 320                                 shared_desc_len =
 321                                         cnstr_shdsc_pdcp_sdap_u_plane_encap(
 322                                                 cdb->sh_desc, 1, swap,
 323                                                 ses->pdcp.sn_size,
 324                                                 ses->pdcp.hfn,
 325                                                 ses->pdcp.bearer,
 326                                                 ses->pdcp.pkt_dir,
 327                                                 ses->pdcp.hfn_threshold,
 328                                                 &cipherdata, p_authdata, 0);
 329                         else
 330                                 shared_desc_len =
 331                                         cnstr_shdsc_pdcp_u_plane_encap(
 332                                                 cdb->sh_desc, 1, swap,
 333                                                 ses->pdcp.sn_size,
 334                                                 ses->pdcp.hfn,
 335                                                 ses->pdcp.bearer,
 336                                                 ses->pdcp.pkt_dir,
 337                                                 ses->pdcp.hfn_threshold,
 338                                                 &cipherdata, p_authdata, 0);
 339                 } else if (ses->dir == DIR_DEC) {
 340                         if (ses->pdcp.sdap_enabled)
 341                                 shared_desc_len =
 342                                         cnstr_shdsc_pdcp_sdap_u_plane_decap(
 343                                                 cdb->sh_desc, 1, swap,
 344                                                 ses->pdcp.sn_size,
 345                                                 ses->pdcp.hfn,
 346                                                 ses->pdcp.bearer,
 347                                                 ses->pdcp.pkt_dir,
 348                                                 ses->pdcp.hfn_threshold,
 349                                                 &cipherdata, p_authdata, 0);
 350                         else
 351                                 shared_desc_len =
 352                                         cnstr_shdsc_pdcp_u_plane_decap(
 353                                                 cdb->sh_desc, 1, swap,
 354                                                 ses->pdcp.sn_size,
 355                                                 ses->pdcp.hfn,
 356                                                 ses->pdcp.bearer,
 357                                                 ses->pdcp.pkt_dir,
 358                                                 ses->pdcp.hfn_threshold,
 359                                                 &cipherdata, p_authdata, 0);
 360                 }
 361         }
 362         return shared_desc_len;
 363 }
 364
 365 /* prepare ipsec proto command block of the session */
 366 static int
 367 dpaa_sec_prep_ipsec_cdb(dpaa_sec_session *ses)
 368 {
 369         struct alginfo cipherdata = {0}, authdata = {0};
 370         struct sec_cdb *cdb = &ses->cdb;
 371         int32_t shared_desc_len = 0;
 372         int err;
 373 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 374         int swap = false;
 375 #else
 376         int swap = true;
 377 #endif
 378
 379         cipherdata.key = (size_t)ses->cipher_key.data;
 380         cipherdata.keylen = ses->cipher_key.length;
 381         cipherdata.key_enc_flags = 0;
 382         cipherdata.key_type = RTA_DATA_IMM;
 383         cipherdata.algtype = ses->cipher_key.alg;
 384         cipherdata.algmode = ses->cipher_key.algmode;
 385
 386         if (ses->auth_key.length) {
 387                 authdata.key = (size_t)ses->auth_key.data;
 388                 authdata.keylen = ses->auth_key.length;
 389                 authdata.key_enc_flags = 0;
 390                 authdata.key_type = RTA_DATA_IMM;
 391                 authdata.algtype = ses->auth_key.alg;
 392                 authdata.algmode = ses->auth_key.algmode;
 393         }
 394
 395         cdb->sh_desc[0] = cipherdata.keylen;
 396         cdb->sh_desc[1] = authdata.keylen;
 397         err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
 398                                DESC_JOB_IO_LEN,
 399                                (unsigned int *)cdb->sh_desc,
 400                                &cdb->sh_desc[2], 2);
 401
 402         if (err < 0) {
 403                 DPAA_SEC_ERR("Crypto: Incorrect key lengths");
 404                 return err;
 405         }
 406         if (cdb->sh_desc[2] & 1)
 407                 cipherdata.key_type = RTA_DATA_IMM;
 408         else {
 409                 cipherdata.key = (size_t)rte_dpaa_mem_vtop(
 410                                         (void *)(size_t)cipherdata.key);
 411                 cipherdata.key_type = RTA_DATA_PTR;
 412         }
 413         if (cdb->sh_desc[2] & (1<<1))
 414                 authdata.key_type = RTA_DATA_IMM;
 415         else {
 416                 authdata.key = (size_t)rte_dpaa_mem_vtop(
 417                                         (void *)(size_t)authdata.key);
 418                 authdata.key_type = RTA_DATA_PTR;
 419         }
 420
 421         cdb->sh_desc[0] = 0;
 422         cdb->sh_desc[1] = 0;
 423         cdb->sh_desc[2] = 0;
 424         if (ses->dir == DIR_ENC) {
 425                 shared_desc_len = cnstr_shdsc_ipsec_new_encap(
 426                                 cdb->sh_desc,
 427                                 true, swap, SHR_SERIAL,
 428                                 &ses->encap_pdb,
 429                                 (uint8_t *)&ses->ip4_hdr,
 430                                 &cipherdata, &authdata);
 431         } else if (ses->dir == DIR_DEC) {
 432                 shared_desc_len = cnstr_shdsc_ipsec_new_decap(
 433                                 cdb->sh_desc,
 434                                 true, swap, SHR_SERIAL,
 435                                 &ses->decap_pdb,
 436                                 &cipherdata, &authdata);
 437         }
 438         return shared_desc_len;
 439 }
 440 #endif
 441 /* prepare command block of the session */
 442 static int
 443 dpaa_sec_prep_cdb(dpaa_sec_session *ses)
 444 {
 445         struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0};
 446         int32_t shared_desc_len = 0;
 447         struct sec_cdb *cdb = &ses->cdb;
 448         int err;
 449 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 450         int swap = false;
 451 #else
 452         int swap = true;
 453 #endif
 454
 455         memset(cdb, 0, sizeof(struct sec_cdb));
 456
 457         switch (ses->ctxt) {
 458 #ifdef RTE_LIB_SECURITY
 459         case DPAA_SEC_IPSEC:
 460                 shared_desc_len = dpaa_sec_prep_ipsec_cdb(ses);
 461                 break;
 462         case DPAA_SEC_PDCP:
 463                 shared_desc_len = dpaa_sec_prep_pdcp_cdb(ses);
 464                 break;
 465 #endif
 466         case DPAA_SEC_CIPHER:
 467                 alginfo_c.key = (size_t)ses->cipher_key.data;
 468                 alginfo_c.keylen = ses->cipher_key.length;
 469                 alginfo_c.key_enc_flags = 0;
 470                 alginfo_c.key_type = RTA_DATA_IMM;
 471                 alginfo_c.algtype = ses->cipher_key.alg;
 472                 alginfo_c.algmode = ses->cipher_key.algmode;
 473
 474                 switch (ses->cipher_alg) {
 475                 case RTE_CRYPTO_CIPHER_AES_CBC:
 476                 case RTE_CRYPTO_CIPHER_3DES_CBC:
 477                 case RTE_CRYPTO_CIPHER_DES_CBC:
 478                 case RTE_CRYPTO_CIPHER_AES_CTR:
 479                 case RTE_CRYPTO_CIPHER_3DES_CTR:
 480                         shared_desc_len = cnstr_shdsc_blkcipher(
 481                                         cdb->sh_desc, true,
 482                                         swap, SHR_NEVER, &alginfo_c,
 483                                         ses->iv.length,
 484                                         ses->dir);
 485                         break;
 486                 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
 487                         shared_desc_len = cnstr_shdsc_snow_f8(
 488                                         cdb->sh_desc, true, swap,
 489                                         &alginfo_c,
 490                                         ses->dir);
 491                         break;
 492                 case RTE_CRYPTO_CIPHER_ZUC_EEA3:
 493                         shared_desc_len = cnstr_shdsc_zuce(
 494                                         cdb->sh_desc, true, swap,
 495                                         &alginfo_c,
 496                                         ses->dir);
 497                         break;
 498                 default:
 499                         DPAA_SEC_ERR("unsupported cipher alg %d",
 500                                      ses->cipher_alg);
 501                         return -ENOTSUP;
 502                 }
 503                 break;
 504         case DPAA_SEC_AUTH:
 505                 alginfo_a.key = (size_t)ses->auth_key.data;
 506                 alginfo_a.keylen = ses->auth_key.length;
 507                 alginfo_a.key_enc_flags = 0;
 508                 alginfo_a.key_type = RTA_DATA_IMM;
 509                 alginfo_a.algtype = ses->auth_key.alg;
 510                 alginfo_a.algmode = ses->auth_key.algmode;
 511                 switch (ses->auth_alg) {
 512                 case RTE_CRYPTO_AUTH_MD5:
 513                 case RTE_CRYPTO_AUTH_SHA1:
 514                 case RTE_CRYPTO_AUTH_SHA224:
 515                 case RTE_CRYPTO_AUTH_SHA256:
 516                 case RTE_CRYPTO_AUTH_SHA384:
 517                 case RTE_CRYPTO_AUTH_SHA512:
 518                         shared_desc_len = cnstr_shdsc_hash(
 519                                                 cdb->sh_desc, true,
 520                                                 swap, SHR_NEVER, &alginfo_a,
 521                                                 !ses->dir,
 522                                                 ses->digest_length);
 523                         break;
 524                 case RTE_CRYPTO_AUTH_MD5_HMAC:
 525                 case RTE_CRYPTO_AUTH_SHA1_HMAC:
 526                 case RTE_CRYPTO_AUTH_SHA224_HMAC:
 527                 case RTE_CRYPTO_AUTH_SHA256_HMAC:
 528                 case RTE_CRYPTO_AUTH_SHA384_HMAC:
 529                 case RTE_CRYPTO_AUTH_SHA512_HMAC:
 530                         shared_desc_len = cnstr_shdsc_hmac(
 531                                                 cdb->sh_desc, true,
 532                                                 swap, SHR_NEVER, &alginfo_a,
 533                                                 !ses->dir,
 534                                                 ses->digest_length);
 535                         break;
 536                 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
 537                         shared_desc_len = cnstr_shdsc_snow_f9(
 538                                                 cdb->sh_desc, true, swap,
 539                                                 &alginfo_a,
 540                                                 !ses->dir,
 541                                                 ses->digest_length);
 542                         break;
 543                 case RTE_CRYPTO_AUTH_ZUC_EIA3:
 544                         shared_desc_len = cnstr_shdsc_zuca(
 545                                                 cdb->sh_desc, true, swap,
 546                                                 &alginfo_a,
 547                                                 !ses->dir,
 548                                                 ses->digest_length);
 549                         break;
 550                 case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
 551                 case RTE_CRYPTO_AUTH_AES_CMAC:
 552                         shared_desc_len = cnstr_shdsc_aes_mac(
 553                                                 cdb->sh_desc,
 554                                                 true, swap, SHR_NEVER,
 555                                                 &alginfo_a,
 556                                                 !ses->dir,
 557                                                 ses->digest_length);
 558                         break;
 559                 default:
 560                         DPAA_SEC_ERR("unsupported auth alg %u", ses->auth_alg);
 561                 }
 562                 break;
 563         case DPAA_SEC_AEAD:
 564                 if (alginfo.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 565                         DPAA_SEC_ERR("not supported aead alg");
 566                         return -ENOTSUP;
 567                 }
 568                 alginfo.key = (size_t)ses->aead_key.data;
 569                 alginfo.keylen = ses->aead_key.length;
 570                 alginfo.key_enc_flags = 0;
 571                 alginfo.key_type = RTA_DATA_IMM;
 572                 alginfo.algtype = ses->aead_key.alg;
 573                 alginfo.algmode = ses->aead_key.algmode;
 574
 575                 if (ses->dir == DIR_ENC)
 576                         shared_desc_len = cnstr_shdsc_gcm_encap(
 577                                         cdb->sh_desc, true, swap, SHR_NEVER,
 578                                         &alginfo,
 579                                         ses->iv.length,
 580                                         ses->digest_length);
 581                 else
 582                         shared_desc_len = cnstr_shdsc_gcm_decap(
 583                                         cdb->sh_desc, true, swap, SHR_NEVER,
 584                                         &alginfo,
 585                                         ses->iv.length,
 586                                         ses->digest_length);
 587                 break;
 588         case DPAA_SEC_CIPHER_HASH:
 589                 alginfo_c.key = (size_t)ses->cipher_key.data;
 590                 alginfo_c.keylen = ses->cipher_key.length;
 591                 alginfo_c.key_enc_flags = 0;
 592                 alginfo_c.key_type = RTA_DATA_IMM;
 593                 alginfo_c.algtype = ses->cipher_key.alg;
 594                 alginfo_c.algmode = ses->cipher_key.algmode;
 595
 596                 alginfo_a.key = (size_t)ses->auth_key.data;
 597                 alginfo_a.keylen = ses->auth_key.length;
 598                 alginfo_a.key_enc_flags = 0;
 599                 alginfo_a.key_type = RTA_DATA_IMM;
 600                 alginfo_a.algtype = ses->auth_key.alg;
 601                 alginfo_a.algmode = ses->auth_key.algmode;
 602
 603                 cdb->sh_desc[0] = alginfo_c.keylen;
 604                 cdb->sh_desc[1] = alginfo_a.keylen;
 605                 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
 606                                        DESC_JOB_IO_LEN,
 607                                        (unsigned int *)cdb->sh_desc,
 608                                        &cdb->sh_desc[2], 2);
 609
 610                 if (err < 0) {
 611                         DPAA_SEC_ERR("Crypto: Incorrect key lengths");
 612                         return err;
 613                 }
 614                 if (cdb->sh_desc[2] & 1)
 615                         alginfo_c.key_type = RTA_DATA_IMM;
 616                 else {
 617                         alginfo_c.key = (size_t)rte_dpaa_mem_vtop(
 618                                                 (void *)(size_t)alginfo_c.key);
 619                         alginfo_c.key_type = RTA_DATA_PTR;
 620                 }
 621                 if (cdb->sh_desc[2] & (1<<1))
 622                         alginfo_a.key_type = RTA_DATA_IMM;
 623                 else {
 624                         alginfo_a.key = (size_t)rte_dpaa_mem_vtop(
 625                                                 (void *)(size_t)alginfo_a.key);
 626                         alginfo_a.key_type = RTA_DATA_PTR;
 627                 }
 628                 cdb->sh_desc[0] = 0;
 629                 cdb->sh_desc[1] = 0;
 630                 cdb->sh_desc[2] = 0;
 631                 /* Auth_only_len is set as 0 here and it will be
 632                  * overwritten in fd for each packet.
 633                  */
 634                 shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
 635                                 true, swap, SHR_SERIAL, &alginfo_c, &alginfo_a,
 636                                 ses->iv.length,
 637                                 ses->digest_length, ses->dir);
 638                 break;
 639         case DPAA_SEC_HASH_CIPHER:
 640         default:
 641                 DPAA_SEC_ERR("error: Unsupported session");
 642                 return -ENOTSUP;
 643         }
 644
 645         if (shared_desc_len < 0) {
 646                 DPAA_SEC_ERR("error in preparing command block");
 647                 return shared_desc_len;
 648         }
 649
 650         cdb->sh_hdr.hi.field.idlen = shared_desc_len;
 651         cdb->sh_hdr.hi.word = rte_cpu_to_be_32(cdb->sh_hdr.hi.word);
 652         cdb->sh_hdr.lo.word = rte_cpu_to_be_32(cdb->sh_hdr.lo.word);
 653
 654         return 0;
 655 }
 656
 657 /* qp is lockless, should be accessed by only one thread */
 658 static int
 659 dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops)
 660 {
 661         struct qman_fq *fq;
 662         unsigned int pkts = 0;
 663         int num_rx_bufs, ret;
 664         struct qm_dqrr_entry *dq;
 665         uint32_t vdqcr_flags = 0;
 666
 667         fq = &qp->outq;
 668         /*
 669          * Until request for four buffers, we provide exact number of buffers.
 670          * Otherwise we do not set the QM_VDQCR_EXACT flag.
 671          * Not setting QM_VDQCR_EXACT flag can provide two more buffers than
 672          * requested, so we request two less in this case.
 673          */
 674         if (nb_ops < 4) {
 675                 vdqcr_flags = QM_VDQCR_EXACT;
 676                 num_rx_bufs = nb_ops;
 677         } else {
 678                 num_rx_bufs = nb_ops > DPAA_MAX_DEQUEUE_NUM_FRAMES ?
 679                         (DPAA_MAX_DEQUEUE_NUM_FRAMES - 2) : (nb_ops - 2);
 680         }
 681         ret = qman_set_vdq(fq, num_rx_bufs, vdqcr_flags);
 682         if (ret)
 683                 return 0;
 684
 685         do {
 686                 const struct qm_fd *fd;
 687                 struct dpaa_sec_job *job;
 688                 struct dpaa_sec_op_ctx *ctx;
 689                 struct rte_crypto_op *op;
 690
 691                 dq = qman_dequeue(fq);
 692                 if (!dq)
 693                         continue;
 694
 695                 fd = &dq->fd;
 696                 /* sg is embedded in an op ctx,
 697                  * sg[0] is for output
 698                  * sg[1] for input
 699                  */
 700                 job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd));
 701
 702                 ctx = container_of(job, struct dpaa_sec_op_ctx, job);
 703                 ctx->fd_status = fd->status;
 704                 op = ctx->op;
 705                 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 706                         struct qm_sg_entry *sg_out;
 707                         uint32_t len;
 708                         struct rte_mbuf *mbuf = (op->sym->m_dst == NULL) ?
 709                                                 op->sym->m_src : op->sym->m_dst;
 710
 711                         sg_out = &job->sg[0];
 712                         hw_sg_to_cpu(sg_out);
 713                         len = sg_out->length;
 714                         mbuf->pkt_len = len;
 715                         while (mbuf->next != NULL) {
 716                                 len -= mbuf->data_len;
 717                                 mbuf = mbuf->next;
 718                         }
 719                         mbuf->data_len = len;
 720                 }
 721                 if (!ctx->fd_status) {
 722                         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 723                 } else {
 724                         DPAA_SEC_DP_WARN("SEC return err:0x%x", ctx->fd_status);
 725                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 726                 }
 727                 ops[pkts++] = op;
 728
 729                 /* report op status to sym->op and then free the ctx memeory */
 730                 rte_mempool_put(ctx->ctx_pool, (void *)ctx);
 731
 732                 qman_dqrr_consume(fq, dq);
 733         } while (fq->flags & QMAN_FQ_STATE_VDQCR);
 734
 735         return pkts;
 736 }
 737
 738 static inline struct dpaa_sec_job *
 739 build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 740 {
 741         struct rte_crypto_sym_op *sym = op->sym;
 742         struct rte_mbuf *mbuf = sym->m_src;
 743         struct dpaa_sec_job *cf;
 744         struct dpaa_sec_op_ctx *ctx;
 745         struct qm_sg_entry *sg, *out_sg, *in_sg;
 746         phys_addr_t start_addr;
 747         uint8_t *old_digest, extra_segs;
 748         int data_len, data_offset;
 749
 750         data_len = sym->auth.data.length;
 751         data_offset = sym->auth.data.offset;
 752
 753         if (ses->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
 754             ses->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
 755                 if ((data_len & 7) || (data_offset & 7)) {
 756                         DPAA_SEC_ERR("AUTH: len/offset must be full bytes");
 757                         return NULL;
 758                 }
 759
 760                 data_len = data_len >> 3;
 761                 data_offset = data_offset >> 3;
 762         }
 763
 764         if (is_decode(ses))
 765                 extra_segs = 3;
 766         else
 767                 extra_segs = 2;
 768
 769         if (mbuf->nb_segs > MAX_SG_ENTRIES) {
 770                 DPAA_SEC_DP_ERR("Auth: Max sec segs supported is %d",
 771                                 MAX_SG_ENTRIES);
 772                 return NULL;
 773         }
 774         ctx = dpaa_sec_alloc_ctx(ses, mbuf->nb_segs + extra_segs);
 775         if (!ctx)
 776                 return NULL;
 777
 778         cf = &ctx->job;
 779         ctx->op = op;
 780         old_digest = ctx->digest;
 781
 782         /* output */
 783         out_sg = &cf->sg[0];
 784         qm_sg_entry_set64(out_sg, sym->auth.digest.phys_addr);
 785         out_sg->length = ses->digest_length;
 786         cpu_to_hw_sg(out_sg);
 787
 788         /* input */
 789         in_sg = &cf->sg[1];
 790         /* need to extend the input to a compound frame */
 791         in_sg->extension = 1;
 792         in_sg->final = 1;
 793         in_sg->length = data_len;
 794         qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
 795
 796         /* 1st seg */
 797         sg = in_sg + 1;
 798
 799         if (ses->iv.length) {
 800                 uint8_t *iv_ptr;
 801
 802                 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 803                                                    ses->iv.offset);
 804
 805                 if (ses->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
 806                         iv_ptr = conv_to_snow_f9_iv(iv_ptr);
 807                         sg->length = 12;
 808                 } else if (ses->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
 809                         iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
 810                         sg->length = 8;
 811                 } else {
 812                         sg->length = ses->iv.length;
 813                 }
 814                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(iv_ptr));
 815                 in_sg->length += sg->length;
 816                 cpu_to_hw_sg(sg);
 817                 sg++;
 818         }
 819
 820         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
 821         sg->offset = data_offset;
 822
 823         if (data_len <= (mbuf->data_len - data_offset)) {
 824                 sg->length = data_len;
 825         } else {
 826                 sg->length = mbuf->data_len - data_offset;
 827
 828                 /* remaining i/p segs */
 829                 while ((data_len = data_len - sg->length) &&
 830                        (mbuf = mbuf->next)) {
 831                         cpu_to_hw_sg(sg);
 832                         sg++;
 833                         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
 834                         if (data_len > mbuf->data_len)
 835                                 sg->length = mbuf->data_len;
 836                         else
 837                                 sg->length = data_len;
 838                 }
 839         }
 840
 841         if (is_decode(ses)) {
 842                 /* Digest verification case */
 843                 cpu_to_hw_sg(sg);
 844                 sg++;
 845                 rte_memcpy(old_digest, sym->auth.digest.data,
 846                                 ses->digest_length);
 847                 start_addr = rte_dpaa_mem_vtop(old_digest);
 848                 qm_sg_entry_set64(sg, start_addr);
 849                 sg->length = ses->digest_length;
 850                 in_sg->length += ses->digest_length;
 851         }
 852         sg->final = 1;
 853         cpu_to_hw_sg(sg);
 854         cpu_to_hw_sg(in_sg);
 855
 856         return cf;
 857 }
 858
 859 /**
 860  * packet looks like:
 861  *              |<----data_len------->|
 862  *    |ip_header|ah_header|icv|payload|
 863  *              ^
 864  *              |
 865  *         mbuf->pkt.data
 866  */
 867 static inline struct dpaa_sec_job *
 868 build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
 869 {
 870         struct rte_crypto_sym_op *sym = op->sym;
 871         struct rte_mbuf *mbuf = sym->m_src;
 872         struct dpaa_sec_job *cf;
 873         struct dpaa_sec_op_ctx *ctx;
 874         struct qm_sg_entry *sg, *in_sg;
 875         rte_iova_t start_addr;
 876         uint8_t *old_digest;
 877         int data_len, data_offset;
 878
 879         data_len = sym->auth.data.length;
 880         data_offset = sym->auth.data.offset;
 881
 882         if (ses->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
 883             ses->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
 884                 if ((data_len & 7) || (data_offset & 7)) {
 885                         DPAA_SEC_ERR("AUTH: len/offset must be full bytes");
 886                         return NULL;
 887                 }
 888
 889                 data_len = data_len >> 3;
 890                 data_offset = data_offset >> 3;
 891         }
 892
 893         ctx = dpaa_sec_alloc_ctx(ses, 4);
 894         if (!ctx)
 895                 return NULL;
 896
 897         cf = &ctx->job;
 898         ctx->op = op;
 899         old_digest = ctx->digest;
 900
 901         start_addr = rte_pktmbuf_iova(mbuf);
 902         /* output */
 903         sg = &cf->sg[0];
 904         qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
 905         sg->length = ses->digest_length;
 906         cpu_to_hw_sg(sg);
 907
 908         /* input */
 909         in_sg = &cf->sg[1];
 910         /* need to extend the input to a compound frame */
 911         in_sg->extension = 1;
 912         in_sg->final = 1;
 913         in_sg->length = data_len;
 914         qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
 915         sg = &cf->sg[2];
 916
 917         if (ses->iv.length) {
 918                 uint8_t *iv_ptr;
 919
 920                 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 921                                                    ses->iv.offset);
 922
 923                 if (ses->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
 924                         iv_ptr = conv_to_snow_f9_iv(iv_ptr);
 925                         sg->length = 12;
 926                 } else if (ses->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
 927                         iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
 928                         sg->length = 8;
 929                 } else {
 930                         sg->length = ses->iv.length;
 931                 }
 932                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(iv_ptr));
 933                 in_sg->length += sg->length;
 934                 cpu_to_hw_sg(sg);
 935                 sg++;
 936         }
 937
 938         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
 939         sg->offset = data_offset;
 940         sg->length = data_len;
 941
 942         if (is_decode(ses)) {
 943                 /* Digest verification case */
 944                 cpu_to_hw_sg(sg);
 945                 /* hash result or digest, save digest first */
 946                 rte_memcpy(old_digest, sym->auth.digest.data,
 947                                 ses->digest_length);
 948                 /* let's check digest by hw */
 949                 start_addr = rte_dpaa_mem_vtop(old_digest);
 950                 sg++;
 951                 qm_sg_entry_set64(sg, start_addr);
 952                 sg->length = ses->digest_length;
 953                 in_sg->length += ses->digest_length;
 954         }
 955         sg->final = 1;
 956         cpu_to_hw_sg(sg);
 957         cpu_to_hw_sg(in_sg);
 958
 959         return cf;
 960 }
 961
 962 static inline struct dpaa_sec_job *
 963 build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 964 {
 965         struct rte_crypto_sym_op *sym = op->sym;
 966         struct dpaa_sec_job *cf;
 967         struct dpaa_sec_op_ctx *ctx;
 968         struct qm_sg_entry *sg, *out_sg, *in_sg;
 969         struct rte_mbuf *mbuf;
 970         uint8_t req_segs;
 971         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 972                         ses->iv.offset);
 973         int data_len, data_offset;
 974
 975         data_len = sym->cipher.data.length;
 976         data_offset = sym->cipher.data.offset;
 977
 978         if (ses->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
 979                 ses->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
 980                 if ((data_len & 7) || (data_offset & 7)) {
 981                         DPAA_SEC_ERR("CIPHER: len/offset must be full bytes");
 982                         return NULL;
 983                 }
 984
 985                 data_len = data_len >> 3;
 986                 data_offset = data_offset >> 3;
 987         }
 988
 989         if (sym->m_dst) {
 990                 mbuf = sym->m_dst;
 991                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3;
 992         } else {
 993                 mbuf = sym->m_src;
 994                 req_segs = mbuf->nb_segs * 2 + 3;
 995         }
 996         if (mbuf->nb_segs > MAX_SG_ENTRIES) {
 997                 DPAA_SEC_DP_ERR("Cipher: Max sec segs supported is %d",
 998                                 MAX_SG_ENTRIES);
 999                 return NULL;
1000         }
1001
1002         ctx = dpaa_sec_alloc_ctx(ses, req_segs);
1003         if (!ctx)
1004                 return NULL;
1005
1006         cf = &ctx->job;
1007         ctx->op = op;
1008
1009         /* output */
1010         out_sg = &cf->sg[0];
1011         out_sg->extension = 1;
1012         out_sg->length = data_len;
1013         qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
1014         cpu_to_hw_sg(out_sg);
1015
1016         /* 1st seg */
1017         sg = &cf->sg[2];
1018         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1019         sg->length = mbuf->data_len - data_offset;
1020         sg->offset = data_offset;
1021
1022         /* Successive segs */
1023         mbuf = mbuf->next;
1024         while (mbuf) {
1025                 cpu_to_hw_sg(sg);
1026                 sg++;
1027                 qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1028                 sg->length = mbuf->data_len;
1029                 mbuf = mbuf->next;
1030         }
1031         sg->final = 1;
1032         cpu_to_hw_sg(sg);
1033
1034         /* input */
1035         mbuf = sym->m_src;
1036         in_sg = &cf->sg[1];
1037         in_sg->extension = 1;
1038         in_sg->final = 1;
1039         in_sg->length = data_len + ses->iv.length;
1040
1041         sg++;
1042         qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
1043         cpu_to_hw_sg(in_sg);
1044
1045         /* IV */
1046         qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
1047         sg->length = ses->iv.length;
1048         cpu_to_hw_sg(sg);
1049
1050         /* 1st seg */
1051         sg++;
1052         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1053         sg->length = mbuf->data_len - data_offset;
1054         sg->offset = data_offset;
1055
1056         /* Successive segs */
1057         mbuf = mbuf->next;
1058         while (mbuf) {
1059                 cpu_to_hw_sg(sg);
1060                 sg++;
1061                 qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1062                 sg->length = mbuf->data_len;
1063                 mbuf = mbuf->next;
1064         }
1065         sg->final = 1;
1066         cpu_to_hw_sg(sg);
1067
1068         return cf;
1069 }
1070
1071 static inline struct dpaa_sec_job *
1072 build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
1073 {
1074         struct rte_crypto_sym_op *sym = op->sym;
1075         struct dpaa_sec_job *cf;
1076         struct dpaa_sec_op_ctx *ctx;
1077         struct qm_sg_entry *sg;
1078         rte_iova_t src_start_addr, dst_start_addr;
1079         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1080                         ses->iv.offset);
1081         int data_len, data_offset;
1082
1083         data_len = sym->cipher.data.length;
1084         data_offset = sym->cipher.data.offset;
1085
1086         if (ses->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
1087                 ses->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
1088                 if ((data_len & 7) || (data_offset & 7)) {
1089                         DPAA_SEC_ERR("CIPHER: len/offset must be full bytes");
1090                         return NULL;
1091                 }
1092
1093                 data_len = data_len >> 3;
1094                 data_offset = data_offset >> 3;
1095         }
1096
1097         ctx = dpaa_sec_alloc_ctx(ses, 4);
1098         if (!ctx)
1099                 return NULL;
1100
1101         cf = &ctx->job;
1102         ctx->op = op;
1103
1104         src_start_addr = rte_pktmbuf_iova(sym->m_src);
1105
1106         if (sym->m_dst)
1107                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
1108         else
1109                 dst_start_addr = src_start_addr;
1110
1111         /* output */
1112         sg = &cf->sg[0];
1113         qm_sg_entry_set64(sg, dst_start_addr + data_offset);
1114         sg->length = data_len + ses->iv.length;
1115         cpu_to_hw_sg(sg);
1116
1117         /* input */
1118         sg = &cf->sg[1];
1119
1120         /* need to extend the input to a compound frame */
1121         sg->extension = 1;
1122         sg->final = 1;
1123         sg->length = data_len + ses->iv.length;
1124         qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(&cf->sg[2]));
1125         cpu_to_hw_sg(sg);
1126
1127         sg = &cf->sg[2];
1128         qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
1129         sg->length = ses->iv.length;
1130         cpu_to_hw_sg(sg);
1131
1132         sg++;
1133         qm_sg_entry_set64(sg, src_start_addr + data_offset);
1134         sg->length = data_len;
1135         sg->final = 1;
1136         cpu_to_hw_sg(sg);
1137
1138         return cf;
1139 }
1140
1141 static inline struct dpaa_sec_job *
1142 build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
1143 {
1144         struct rte_crypto_sym_op *sym = op->sym;
1145         struct dpaa_sec_job *cf;
1146         struct dpaa_sec_op_ctx *ctx;
1147         struct qm_sg_entry *sg, *out_sg, *in_sg;
1148         struct rte_mbuf *mbuf;
1149         uint8_t req_segs;
1150         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1151                         ses->iv.offset);
1152
1153         if (sym->m_dst) {
1154                 mbuf = sym->m_dst;
1155                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4;
1156         } else {
1157                 mbuf = sym->m_src;
1158                 req_segs = mbuf->nb_segs * 2 + 4;
1159         }
1160
1161         if (ses->auth_only_len)
1162                 req_segs++;
1163
1164         if (mbuf->nb_segs > MAX_SG_ENTRIES) {
1165                 DPAA_SEC_DP_ERR("AEAD: Max sec segs supported is %d",
1166                                 MAX_SG_ENTRIES);
1167                 return NULL;
1168         }
1169
1170         ctx = dpaa_sec_alloc_ctx(ses, req_segs);
1171         if (!ctx)
1172                 return NULL;
1173
1174         cf = &ctx->job;
1175         ctx->op = op;
1176
1177         rte_prefetch0(cf->sg);
1178
1179         /* output */
1180         out_sg = &cf->sg[0];
1181         out_sg->extension = 1;
1182         if (is_encode(ses))
1183                 out_sg->length = sym->aead.data.length + ses->digest_length;
1184         else
1185                 out_sg->length = sym->aead.data.length;
1186
1187         /* output sg entries */
1188         sg = &cf->sg[2];
1189         qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(sg));
1190         cpu_to_hw_sg(out_sg);
1191
1192         /* 1st seg */
1193         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1194         sg->length = mbuf->data_len - sym->aead.data.offset;
1195         sg->offset = sym->aead.data.offset;
1196
1197         /* Successive segs */
1198         mbuf = mbuf->next;
1199         while (mbuf) {
1200                 cpu_to_hw_sg(sg);
1201                 sg++;
1202                 qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1203                 sg->length = mbuf->data_len;
1204                 mbuf = mbuf->next;
1205         }
1206         sg->length -= ses->digest_length;
1207
1208         if (is_encode(ses)) {
1209                 cpu_to_hw_sg(sg);
1210                 /* set auth output */
1211                 sg++;
1212                 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr);
1213                 sg->length = ses->digest_length;
1214         }
1215         sg->final = 1;
1216         cpu_to_hw_sg(sg);
1217
1218         /* input */
1219         mbuf = sym->m_src;
1220         in_sg = &cf->sg[1];
1221         in_sg->extension = 1;
1222         in_sg->final = 1;
1223         if (is_encode(ses))
1224                 in_sg->length = ses->iv.length + sym->aead.data.length
1225                                                         + ses->auth_only_len;
1226         else
1227                 in_sg->length = ses->iv.length + sym->aead.data.length
1228                                 + ses->auth_only_len + ses->digest_length;
1229
1230         /* input sg entries */
1231         sg++;
1232         qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
1233         cpu_to_hw_sg(in_sg);
1234
1235         /* 1st seg IV */
1236         qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
1237         sg->length = ses->iv.length;
1238         cpu_to_hw_sg(sg);
1239
1240         /* 2nd seg auth only */
1241         if (ses->auth_only_len) {
1242                 sg++;
1243                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(sym->aead.aad.data));
1244                 sg->length = ses->auth_only_len;
1245                 cpu_to_hw_sg(sg);
1246         }
1247
1248         /* 3rd seg */
1249         sg++;
1250         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1251         sg->length = mbuf->data_len - sym->aead.data.offset;
1252         sg->offset = sym->aead.data.offset;
1253
1254         /* Successive segs */
1255         mbuf = mbuf->next;
1256         while (mbuf) {
1257                 cpu_to_hw_sg(sg);
1258                 sg++;
1259                 qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1260                 sg->length = mbuf->data_len;
1261                 mbuf = mbuf->next;
1262         }
1263
1264         if (is_decode(ses)) {
1265                 cpu_to_hw_sg(sg);
1266                 sg++;
1267                 memcpy(ctx->digest, sym->aead.digest.data,
1268                         ses->digest_length);
1269                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest));
1270                 sg->length = ses->digest_length;
1271         }
1272         sg->final = 1;
1273         cpu_to_hw_sg(sg);
1274
1275         return cf;
1276 }
1277
1278 static inline struct dpaa_sec_job *
1279 build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses)
1280 {
1281         struct rte_crypto_sym_op *sym = op->sym;
1282         struct dpaa_sec_job *cf;
1283         struct dpaa_sec_op_ctx *ctx;
1284         struct qm_sg_entry *sg;
1285         uint32_t length = 0;
1286         rte_iova_t src_start_addr, dst_start_addr;
1287         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1288                         ses->iv.offset);
1289
1290         src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
1291
1292         if (sym->m_dst)
1293                 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
1294         else
1295                 dst_start_addr = src_start_addr;
1296
1297         ctx = dpaa_sec_alloc_ctx(ses, 7);
1298         if (!ctx)
1299                 return NULL;
1300
1301         cf = &ctx->job;
1302         ctx->op = op;
1303
1304         /* input */
1305         rte_prefetch0(cf->sg);
1306         sg = &cf->sg[2];
1307         qm_sg_entry_set64(&cf->sg[1], rte_dpaa_mem_vtop(sg));
1308         if (is_encode(ses)) {
1309                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
1310                 sg->length = ses->iv.length;
1311                 length += sg->length;
1312                 cpu_to_hw_sg(sg);
1313
1314                 sg++;
1315                 if (ses->auth_only_len) {
1316                         qm_sg_entry_set64(sg,
1317                                           rte_dpaa_mem_vtop(sym->aead.aad.data));
1318                         sg->length = ses->auth_only_len;
1319                         length += sg->length;
1320                         cpu_to_hw_sg(sg);
1321                         sg++;
1322                 }
1323                 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
1324                 sg->length = sym->aead.data.length;
1325                 length += sg->length;
1326                 sg->final = 1;
1327                 cpu_to_hw_sg(sg);
1328         } else {
1329                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
1330                 sg->length = ses->iv.length;
1331                 length += sg->length;
1332                 cpu_to_hw_sg(sg);
1333
1334                 sg++;
1335                 if (ses->auth_only_len) {
1336                         qm_sg_entry_set64(sg,
1337                                           rte_dpaa_mem_vtop(sym->aead.aad.data));
1338                         sg->length = ses->auth_only_len;
1339                         length += sg->length;
1340                         cpu_to_hw_sg(sg);
1341                         sg++;
1342                 }
1343                 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
1344                 sg->length = sym->aead.data.length;
1345                 length += sg->length;
1346                 cpu_to_hw_sg(sg);
1347
1348                 memcpy(ctx->digest, sym->aead.digest.data,
1349                        ses->digest_length);
1350                 sg++;
1351
1352                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest));
1353                 sg->length = ses->digest_length;
1354                 length += sg->length;
1355                 sg->final = 1;
1356                 cpu_to_hw_sg(sg);
1357         }
1358         /* input compound frame */
1359         cf->sg[1].length = length;
1360         cf->sg[1].extension = 1;
1361         cf->sg[1].final = 1;
1362         cpu_to_hw_sg(&cf->sg[1]);
1363
1364         /* output */
1365         sg++;
1366         qm_sg_entry_set64(&cf->sg[0], rte_dpaa_mem_vtop(sg));
1367         qm_sg_entry_set64(sg,
1368                 dst_start_addr + sym->aead.data.offset);
1369         sg->length = sym->aead.data.length;
1370         length = sg->length;
1371         if (is_encode(ses)) {
1372                 cpu_to_hw_sg(sg);
1373                 /* set auth output */
1374                 sg++;
1375                 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr);
1376                 sg->length = ses->digest_length;
1377                 length += sg->length;
1378         }
1379         sg->final = 1;
1380         cpu_to_hw_sg(sg);
1381
1382         /* output compound frame */
1383         cf->sg[0].length = length;
1384         cf->sg[0].extension = 1;
1385         cpu_to_hw_sg(&cf->sg[0]);
1386
1387         return cf;
1388 }
1389
1390 static inline struct dpaa_sec_job *
1391 build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
1392 {
1393         struct rte_crypto_sym_op *sym = op->sym;
1394         struct dpaa_sec_job *cf;
1395         struct dpaa_sec_op_ctx *ctx;
1396         struct qm_sg_entry *sg, *out_sg, *in_sg;
1397         struct rte_mbuf *mbuf;
1398         uint8_t req_segs;
1399         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1400                         ses->iv.offset);
1401
1402         if (sym->m_dst) {
1403                 mbuf = sym->m_dst;
1404                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4;
1405         } else {
1406                 mbuf = sym->m_src;
1407                 req_segs = mbuf->nb_segs * 2 + 4;
1408         }
1409
1410         if (mbuf->nb_segs > MAX_SG_ENTRIES) {
1411                 DPAA_SEC_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
1412                                 MAX_SG_ENTRIES);
1413                 return NULL;
1414         }
1415
1416         ctx = dpaa_sec_alloc_ctx(ses, req_segs);
1417         if (!ctx)
1418                 return NULL;
1419
1420         cf = &ctx->job;
1421         ctx->op = op;
1422
1423         rte_prefetch0(cf->sg);
1424
1425         /* output */
1426         out_sg = &cf->sg[0];
1427         out_sg->extension = 1;
1428         if (is_encode(ses))
1429                 out_sg->length = sym->auth.data.length + ses->digest_length;
1430         else
1431                 out_sg->length = sym->auth.data.length;
1432
1433         /* output sg entries */
1434         sg = &cf->sg[2];
1435         qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(sg));
1436         cpu_to_hw_sg(out_sg);
1437
1438         /* 1st seg */
1439         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1440         sg->length = mbuf->data_len - sym->auth.data.offset;
1441         sg->offset = sym->auth.data.offset;
1442
1443         /* Successive segs */
1444         mbuf = mbuf->next;
1445         while (mbuf) {
1446                 cpu_to_hw_sg(sg);
1447                 sg++;
1448                 qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1449                 sg->length = mbuf->data_len;
1450                 mbuf = mbuf->next;
1451         }
1452         sg->length -= ses->digest_length;
1453
1454         if (is_encode(ses)) {
1455                 cpu_to_hw_sg(sg);
1456                 /* set auth output */
1457                 sg++;
1458                 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
1459                 sg->length = ses->digest_length;
1460         }
1461         sg->final = 1;
1462         cpu_to_hw_sg(sg);
1463
1464         /* input */
1465         mbuf = sym->m_src;
1466         in_sg = &cf->sg[1];
1467         in_sg->extension = 1;
1468         in_sg->final = 1;
1469         if (is_encode(ses))
1470                 in_sg->length = ses->iv.length + sym->auth.data.length;
1471         else
1472                 in_sg->length = ses->iv.length + sym->auth.data.length
1473                                                 + ses->digest_length;
1474
1475         /* input sg entries */
1476         sg++;
1477         qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
1478         cpu_to_hw_sg(in_sg);
1479
1480         /* 1st seg IV */
1481         qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
1482         sg->length = ses->iv.length;
1483         cpu_to_hw_sg(sg);
1484
1485         /* 2nd seg */
1486         sg++;
1487         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1488         sg->length = mbuf->data_len - sym->auth.data.offset;
1489         sg->offset = sym->auth.data.offset;
1490
1491         /* Successive segs */
1492         mbuf = mbuf->next;
1493         while (mbuf) {
1494                 cpu_to_hw_sg(sg);
1495                 sg++;
1496                 qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1497                 sg->length = mbuf->data_len;
1498                 mbuf = mbuf->next;
1499         }
1500
1501         sg->length -= ses->digest_length;
1502         if (is_decode(ses)) {
1503                 cpu_to_hw_sg(sg);
1504                 sg++;
1505                 memcpy(ctx->digest, sym->auth.digest.data,
1506                         ses->digest_length);
1507                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest));
1508                 sg->length = ses->digest_length;
1509         }
1510         sg->final = 1;
1511         cpu_to_hw_sg(sg);
1512
1513         return cf;
1514 }
1515
1516 static inline struct dpaa_sec_job *
1517 build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses)
1518 {
1519         struct rte_crypto_sym_op *sym = op->sym;
1520         struct dpaa_sec_job *cf;
1521         struct dpaa_sec_op_ctx *ctx;
1522         struct qm_sg_entry *sg;
1523         rte_iova_t src_start_addr, dst_start_addr;
1524         uint32_t length = 0;
1525         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1526                         ses->iv.offset);
1527
1528         src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
1529         if (sym->m_dst)
1530                 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
1531         else
1532                 dst_start_addr = src_start_addr;
1533
1534         ctx = dpaa_sec_alloc_ctx(ses, 7);
1535         if (!ctx)
1536                 return NULL;
1537
1538         cf = &ctx->job;
1539         ctx->op = op;
1540
1541         /* input */
1542         rte_prefetch0(cf->sg);
1543         sg = &cf->sg[2];
1544         qm_sg_entry_set64(&cf->sg[1], rte_dpaa_mem_vtop(sg));
1545         if (is_encode(ses)) {
1546                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
1547                 sg->length = ses->iv.length;
1548                 length += sg->length;
1549                 cpu_to_hw_sg(sg);
1550
1551                 sg++;
1552                 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
1553                 sg->length = sym->auth.data.length;
1554                 length += sg->length;
1555                 sg->final = 1;
1556                 cpu_to_hw_sg(sg);
1557         } else {
1558                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
1559                 sg->length = ses->iv.length;
1560                 length += sg->length;
1561                 cpu_to_hw_sg(sg);
1562
1563                 sg++;
1564
1565                 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
1566                 sg->length = sym->auth.data.length;
1567                 length += sg->length;
1568                 cpu_to_hw_sg(sg);
1569
1570                 memcpy(ctx->digest, sym->auth.digest.data,
1571                        ses->digest_length);
1572                 sg++;
1573
1574                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest));
1575                 sg->length = ses->digest_length;
1576                 length += sg->length;
1577                 sg->final = 1;
1578                 cpu_to_hw_sg(sg);
1579         }
1580         /* input compound frame */
1581         cf->sg[1].length = length;
1582         cf->sg[1].extension = 1;
1583         cf->sg[1].final = 1;
1584         cpu_to_hw_sg(&cf->sg[1]);
1585
1586         /* output */
1587         sg++;
1588         qm_sg_entry_set64(&cf->sg[0], rte_dpaa_mem_vtop(sg));
1589         qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset);
1590         sg->length = sym->cipher.data.length;
1591         length = sg->length;
1592         if (is_encode(ses)) {
1593                 cpu_to_hw_sg(sg);
1594                 /* set auth output */
1595                 sg++;
1596                 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
1597                 sg->length = ses->digest_length;
1598                 length += sg->length;
1599         }
1600         sg->final = 1;
1601         cpu_to_hw_sg(sg);
1602
1603         /* output compound frame */
1604         cf->sg[0].length = length;
1605         cf->sg[0].extension = 1;
1606         cpu_to_hw_sg(&cf->sg[0]);
1607
1608         return cf;
1609 }
1610
1611 #ifdef RTE_LIB_SECURITY
1612 static inline struct dpaa_sec_job *
1613 build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses)
1614 {
1615         struct rte_crypto_sym_op *sym = op->sym;
1616         struct dpaa_sec_job *cf;
1617         struct dpaa_sec_op_ctx *ctx;
1618         struct qm_sg_entry *sg;
1619         phys_addr_t src_start_addr, dst_start_addr;
1620
1621         ctx = dpaa_sec_alloc_ctx(ses, 2);
1622         if (!ctx)
1623                 return NULL;
1624         cf = &ctx->job;
1625         ctx->op = op;
1626
1627         src_start_addr = rte_pktmbuf_iova(sym->m_src);
1628
1629         if (sym->m_dst)
1630                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
1631         else
1632                 dst_start_addr = src_start_addr;
1633
1634         /* input */
1635         sg = &cf->sg[1];
1636         qm_sg_entry_set64(sg, src_start_addr);
1637         sg->length = sym->m_src->pkt_len;
1638         sg->final = 1;
1639         cpu_to_hw_sg(sg);
1640
1641         sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
1642         /* output */
1643         sg = &cf->sg[0];
1644         qm_sg_entry_set64(sg, dst_start_addr);
1645         sg->length = sym->m_src->buf_len - sym->m_src->data_off;
1646         cpu_to_hw_sg(sg);
1647
1648         return cf;
1649 }
1650
1651 static inline struct dpaa_sec_job *
1652 build_proto_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
1653 {
1654         struct rte_crypto_sym_op *sym = op->sym;
1655         struct dpaa_sec_job *cf;
1656         struct dpaa_sec_op_ctx *ctx;
1657         struct qm_sg_entry *sg, *out_sg, *in_sg;
1658         struct rte_mbuf *mbuf;
1659         uint8_t req_segs;
1660         uint32_t in_len = 0, out_len = 0;
1661
1662         if (sym->m_dst)
1663                 mbuf = sym->m_dst;
1664         else
1665                 mbuf = sym->m_src;
1666
1667         req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 2;
1668         if (mbuf->nb_segs > MAX_SG_ENTRIES) {
1669                 DPAA_SEC_DP_ERR("Proto: Max sec segs supported is %d",
1670                                 MAX_SG_ENTRIES);
1671                 return NULL;
1672         }
1673
1674         ctx = dpaa_sec_alloc_ctx(ses, req_segs);
1675         if (!ctx)
1676                 return NULL;
1677         cf = &ctx->job;
1678         ctx->op = op;
1679         /* output */
1680         out_sg = &cf->sg[0];
1681         out_sg->extension = 1;
1682         qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
1683
1684         /* 1st seg */
1685         sg = &cf->sg[2];
1686         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1687         sg->offset = 0;
1688
1689         /* Successive segs */
1690         while (mbuf->next) {
1691                 sg->length = mbuf->data_len;
1692                 out_len += sg->length;
1693                 mbuf = mbuf->next;
1694                 cpu_to_hw_sg(sg);
1695                 sg++;
1696                 qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1697                 sg->offset = 0;
1698         }
1699         sg->length = mbuf->buf_len - mbuf->data_off;
1700         out_len += sg->length;
1701         sg->final = 1;
1702         cpu_to_hw_sg(sg);
1703
1704         out_sg->length = out_len;
1705         cpu_to_hw_sg(out_sg);
1706
1707         /* input */
1708         mbuf = sym->m_src;
1709         in_sg = &cf->sg[1];
1710         in_sg->extension = 1;
1711         in_sg->final = 1;
1712         in_len = mbuf->data_len;
1713
1714         sg++;
1715         qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
1716
1717         /* 1st seg */
1718         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1719         sg->length = mbuf->data_len;
1720         sg->offset = 0;
1721
1722         /* Successive segs */
1723         mbuf = mbuf->next;
1724         while (mbuf) {
1725                 cpu_to_hw_sg(sg);
1726                 sg++;
1727                 qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1728                 sg->length = mbuf->data_len;
1729                 sg->offset = 0;
1730                 in_len += sg->length;
1731                 mbuf = mbuf->next;
1732         }
1733         sg->final = 1;
1734         cpu_to_hw_sg(sg);
1735
1736         in_sg->length = in_len;
1737         cpu_to_hw_sg(in_sg);
1738
1739         sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
1740
1741         return cf;
1742 }
1743 #endif
1744
1745 static uint16_t
1746 dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1747                        uint16_t nb_ops)
1748 {
1749         /* Function to transmit the frames to given device and queuepair */
1750         uint32_t loop;
1751         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp;
1752         uint16_t num_tx = 0;
1753         struct qm_fd fds[DPAA_SEC_BURST], *fd;
1754         uint32_t frames_to_send;
1755         struct rte_crypto_op *op;
1756         struct dpaa_sec_job *cf;
1757         dpaa_sec_session *ses;
1758         uint16_t auth_hdr_len, auth_tail_len;
1759         uint32_t index, flags[DPAA_SEC_BURST] = {0};
1760         struct qman_fq *inq[DPAA_SEC_BURST];
1761
1762         if (unlikely(!DPAA_PER_LCORE_PORTAL)) {
1763                 if (rte_dpaa_portal_init((void *)0)) {
1764                         DPAA_SEC_ERR("Failure in affining portal");
1765                         return 0;
1766                 }
1767         }
1768
1769         while (nb_ops) {
1770                 frames_to_send = (nb_ops > DPAA_SEC_BURST) ?
1771                                 DPAA_SEC_BURST : nb_ops;
1772                 for (loop = 0; loop < frames_to_send; loop++) {
1773                         op = *(ops++);
1774                         if (*dpaa_seqn(op->sym->m_src) != 0) {
1775                                 index = *dpaa_seqn(op->sym->m_src) - 1;
1776                                 if (DPAA_PER_LCORE_DQRR_HELD & (1 << index)) {
1777                                         /* QM_EQCR_DCA_IDXMASK = 0x0f */
1778                                         flags[loop] = ((index & 0x0f) << 8);
1779                                         flags[loop] |= QMAN_ENQUEUE_FLAG_DCA;
1780                                         DPAA_PER_LCORE_DQRR_SIZE--;
1781                                         DPAA_PER_LCORE_DQRR_HELD &=
1782                                                                 ~(1 << index);
1783                                 }
1784                         }
1785
1786                         switch (op->sess_type) {
1787                         case RTE_CRYPTO_OP_WITH_SESSION:
1788                                 ses = (dpaa_sec_session *)
1789                                         get_sym_session_private_data(
1790                                                         op->sym->session,
1791                                                         cryptodev_driver_id);
1792                                 break;
1793 #ifdef RTE_LIB_SECURITY
1794                         case RTE_CRYPTO_OP_SECURITY_SESSION:
1795                                 ses = (dpaa_sec_session *)
1796                                         get_sec_session_private_data(
1797                                                         op->sym->sec_session);
1798                                 break;
1799 #endif
1800                         default:
1801                                 DPAA_SEC_DP_ERR(
1802                                         "sessionless crypto op not supported");
1803                                 frames_to_send = loop;
1804                                 nb_ops = loop;
1805                                 goto send_pkts;
1806                         }
1807
1808                         if (!ses) {
1809                                 DPAA_SEC_DP_ERR("session not available");
1810                                 frames_to_send = loop;
1811                                 nb_ops = loop;
1812                                 goto send_pkts;
1813                         }
1814
1815                         if (unlikely(!ses->qp[rte_lcore_id() % MAX_DPAA_CORES])) {
1816                                 if (dpaa_sec_attach_sess_q(qp, ses)) {
1817                                         frames_to_send = loop;
1818                                         nb_ops = loop;
1819                                         goto send_pkts;
1820                                 }
1821                         } else if (unlikely(ses->qp[rte_lcore_id() %
1822                                                 MAX_DPAA_CORES] != qp)) {
1823                                 DPAA_SEC_DP_ERR("Old:sess->qp = %p"
1824                                         " New qp = %p\n",
1825                                         ses->qp[rte_lcore_id() %
1826                                         MAX_DPAA_CORES], qp);
1827                                 frames_to_send = loop;
1828                                 nb_ops = loop;
1829                                 goto send_pkts;
1830                         }
1831
1832                         auth_hdr_len = op->sym->auth.data.length -
1833                                                 op->sym->cipher.data.length;
1834                         auth_tail_len = 0;
1835
1836                         if (rte_pktmbuf_is_contiguous(op->sym->m_src) &&
1837                                   ((op->sym->m_dst == NULL) ||
1838                                    rte_pktmbuf_is_contiguous(op->sym->m_dst))) {
1839                                 switch (ses->ctxt) {
1840 #ifdef RTE_LIB_SECURITY
1841                                 case DPAA_SEC_PDCP:
1842                                 case DPAA_SEC_IPSEC:
1843                                         cf = build_proto(op, ses);
1844                                         break;
1845 #endif
1846                                 case DPAA_SEC_AUTH:
1847                                         cf = build_auth_only(op, ses);
1848                                         break;
1849                                 case DPAA_SEC_CIPHER:
1850                                         cf = build_cipher_only(op, ses);
1851                                         break;
1852                                 case DPAA_SEC_AEAD:
1853                                         cf = build_cipher_auth_gcm(op, ses);
1854                                         auth_hdr_len = ses->auth_only_len;
1855                                         break;
1856                                 case DPAA_SEC_CIPHER_HASH:
1857                                         auth_hdr_len =
1858                                                 op->sym->cipher.data.offset
1859                                                 - op->sym->auth.data.offset;
1860                                         auth_tail_len =
1861                                                 op->sym->auth.data.length
1862                                                 - op->sym->cipher.data.length
1863                                                 - auth_hdr_len;
1864                                         cf = build_cipher_auth(op, ses);
1865                                         break;
1866                                 default:
1867                                         DPAA_SEC_DP_ERR("not supported ops");
1868                                         frames_to_send = loop;
1869                                         nb_ops = loop;
1870                                         goto send_pkts;
1871                                 }
1872                         } else {
1873                                 switch (ses->ctxt) {
1874 #ifdef RTE_LIB_SECURITY
1875                                 case DPAA_SEC_PDCP:
1876                                 case DPAA_SEC_IPSEC:
1877                                         cf = build_proto_sg(op, ses);
1878                                         break;
1879 #endif
1880                                 case DPAA_SEC_AUTH:
1881                                         cf = build_auth_only_sg(op, ses);
1882                                         break;
1883                                 case DPAA_SEC_CIPHER:
1884                                         cf = build_cipher_only_sg(op, ses);
1885                                         break;
1886                                 case DPAA_SEC_AEAD:
1887                                         cf = build_cipher_auth_gcm_sg(op, ses);
1888                                         auth_hdr_len = ses->auth_only_len;
1889                                         break;
1890                                 case DPAA_SEC_CIPHER_HASH:
1891                                         auth_hdr_len =
1892                                                 op->sym->cipher.data.offset
1893                                                 - op->sym->auth.data.offset;
1894                                         auth_tail_len =
1895                                                 op->sym->auth.data.length
1896                                                 - op->sym->cipher.data.length
1897                                                 - auth_hdr_len;
1898                                         cf = build_cipher_auth_sg(op, ses);
1899                                         break;
1900                                 default:
1901                                         DPAA_SEC_DP_ERR("not supported ops");
1902                                         frames_to_send = loop;
1903                                         nb_ops = loop;
1904                                         goto send_pkts;
1905                                 }
1906                         }
1907                         if (unlikely(!cf)) {
1908                                 frames_to_send = loop;
1909                                 nb_ops = loop;
1910                                 goto send_pkts;
1911                         }
1912
1913                         fd = &fds[loop];
1914                         inq[loop] = ses->inq[rte_lcore_id() % MAX_DPAA_CORES];
1915                         fd->opaque_addr = 0;
1916                         fd->cmd = 0;
1917                         qm_fd_addr_set64(fd, rte_dpaa_mem_vtop(cf->sg));
1918                         fd->_format1 = qm_fd_compound;
1919                         fd->length29 = 2 * sizeof(struct qm_sg_entry);
1920
1921                         /* Auth_only_len is set as 0 in descriptor and it is
1922                          * overwritten here in the fd.cmd which will update
1923                          * the DPOVRD reg.
1924                          */
1925                         if (auth_hdr_len || auth_tail_len) {
1926                                 fd->cmd = 0x80000000;
1927                                 fd->cmd |=
1928                                         ((auth_tail_len << 16) | auth_hdr_len);
1929                         }
1930
1931 #ifdef RTE_LIB_SECURITY
1932                         /* In case of PDCP, per packet HFN is stored in
1933                          * mbuf priv after sym_op.
1934                          */
1935                         if ((ses->ctxt == DPAA_SEC_PDCP) && ses->pdcp.hfn_ovd) {
1936                                 fd->cmd = 0x80000000 |
1937                                         *((uint32_t *)((uint8_t *)op +
1938                                         ses->pdcp.hfn_ovd_offset));
1939                                 DPAA_SEC_DP_DEBUG("Per packet HFN: %x, ovd:%u\n",
1940                                         *((uint32_t *)((uint8_t *)op +
1941                                         ses->pdcp.hfn_ovd_offset)),
1942                                         ses->pdcp.hfn_ovd);
1943                         }
1944 #endif
1945                 }
1946 send_pkts:
1947                 loop = 0;
1948                 while (loop < frames_to_send) {
1949                         loop += qman_enqueue_multi_fq(&inq[loop], &fds[loop],
1950                                         &flags[loop], frames_to_send - loop);
1951                 }
1952                 nb_ops -= frames_to_send;
1953                 num_tx += frames_to_send;
1954         }
1955
1956         dpaa_qp->tx_pkts += num_tx;
1957         dpaa_qp->tx_errs += nb_ops - num_tx;
1958
1959         return num_tx;
1960 }
1961
1962 static uint16_t
1963 dpaa_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops,
1964                        uint16_t nb_ops)
1965 {
1966         uint16_t num_rx;
1967         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp;
1968
1969         if (unlikely(!DPAA_PER_LCORE_PORTAL)) {
1970                 if (rte_dpaa_portal_init((void *)0)) {
1971                         DPAA_SEC_ERR("Failure in affining portal");
1972                         return 0;
1973                 }
1974         }
1975
1976         num_rx = dpaa_sec_deq(dpaa_qp, ops, nb_ops);
1977
1978         dpaa_qp->rx_pkts += num_rx;
1979         dpaa_qp->rx_errs += nb_ops - num_rx;
1980
1981         DPAA_SEC_DP_DEBUG("SEC Received %d Packets\n", num_rx);
1982
1983         return num_rx;
1984 }
1985
1986 /** Release queue pair */
1987 static int
1988 dpaa_sec_queue_pair_release(struct rte_cryptodev *dev,
1989                             uint16_t qp_id)
1990 {
1991         struct dpaa_sec_dev_private *internals;
1992         struct dpaa_sec_qp *qp = NULL;
1993
1994         PMD_INIT_FUNC_TRACE();
1995
1996         DPAA_SEC_DEBUG("dev =%p, queue =%d", dev, qp_id);
1997
1998         internals = dev->data->dev_private;
1999         if (qp_id >= internals->max_nb_queue_pairs) {
2000                 DPAA_SEC_ERR("Max supported qpid %d",
2001                              internals->max_nb_queue_pairs);
2002                 return -EINVAL;
2003         }
2004
2005         qp = &internals->qps[qp_id];
2006         rte_mempool_free(qp->ctx_pool);
2007         qp->internals = NULL;
2008         dev->data->queue_pairs[qp_id] = NULL;
2009
2010         return 0;
2011 }
2012
2013 /** Setup a queue pair */
2014 static int
2015 dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
2016                 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
2017                 __rte_unused int socket_id)
2018 {
2019         struct dpaa_sec_dev_private *internals;
2020         struct dpaa_sec_qp *qp = NULL;
2021         char str[20];
2022
2023         DPAA_SEC_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);
2024
2025         internals = dev->data->dev_private;
2026         if (qp_id >= internals->max_nb_queue_pairs) {
2027                 DPAA_SEC_ERR("Max supported qpid %d",
2028                              internals->max_nb_queue_pairs);
2029                 return -EINVAL;
2030         }
2031
2032         qp = &internals->qps[qp_id];
2033         qp->internals = internals;
2034         snprintf(str, sizeof(str), "ctx_pool_d%d_qp%d",
2035                         dev->data->dev_id, qp_id);
2036         if (!qp->ctx_pool) {
2037                 qp->ctx_pool = rte_mempool_create((const char *)str,
2038                                                         CTX_POOL_NUM_BUFS,
2039                                                         CTX_POOL_BUF_SIZE,
2040                                                         CTX_POOL_CACHE_SIZE, 0,
2041                                                         NULL, NULL, NULL, NULL,
2042                                                         SOCKET_ID_ANY, 0);
2043                 if (!qp->ctx_pool) {
2044                         DPAA_SEC_ERR("%s create failed\n", str);
2045                         return -ENOMEM;
2046                 }
2047         } else
2048                 DPAA_SEC_INFO("mempool already created for dev_id : %d, qp: %d",
2049                                 dev->data->dev_id, qp_id);
2050         dev->data->queue_pairs[qp_id] = qp;
2051
2052         return 0;
2053 }
2054
2055 /** Returns the size of session structure */
2056 static unsigned int
2057 dpaa_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
2058 {
2059         PMD_INIT_FUNC_TRACE();
2060
2061         return sizeof(dpaa_sec_session);
2062 }
2063
2064 static int
2065 dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused,
2066                      struct rte_crypto_sym_xform *xform,
2067                      dpaa_sec_session *session)
2068 {
2069         session->ctxt = DPAA_SEC_CIPHER;
2070         session->cipher_alg = xform->cipher.algo;
2071         session->iv.length = xform->cipher.iv.length;
2072         session->iv.offset = xform->cipher.iv.offset;
2073         session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
2074                                                RTE_CACHE_LINE_SIZE);
2075         if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
2076                 DPAA_SEC_ERR("No Memory for cipher key");
2077                 return -ENOMEM;
2078         }
2079         session->cipher_key.length = xform->cipher.key.length;
2080
2081         memcpy(session->cipher_key.data, xform->cipher.key.data,
2082                xform->cipher.key.length);
2083         switch (xform->cipher.algo) {
2084         case RTE_CRYPTO_CIPHER_AES_CBC:
2085                 session->cipher_key.alg = OP_ALG_ALGSEL_AES;
2086                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2087                 break;
2088         case RTE_CRYPTO_CIPHER_DES_CBC:
2089                 session->cipher_key.alg = OP_ALG_ALGSEL_DES;
2090                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2091                 break;
2092         case RTE_CRYPTO_CIPHER_3DES_CBC:
2093                 session->cipher_key.alg = OP_ALG_ALGSEL_3DES;
2094                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2095                 break;
2096         case RTE_CRYPTO_CIPHER_AES_CTR:
2097                 session->cipher_key.alg = OP_ALG_ALGSEL_AES;
2098                 session->cipher_key.algmode = OP_ALG_AAI_CTR;
2099                 break;
2100         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2101                 session->cipher_key.alg = OP_ALG_ALGSEL_SNOW_F8;
2102                 break;
2103         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2104                 session->cipher_key.alg = OP_ALG_ALGSEL_ZUCE;
2105                 break;
2106         default:
2107                 DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u",
2108                               xform->cipher.algo);
2109                 return -ENOTSUP;
2110         }
2111         session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2112                         DIR_ENC : DIR_DEC;
2113
2114         return 0;
2115 }
2116
2117 static int
2118 dpaa_sec_auth_init(struct rte_cryptodev *dev __rte_unused,
2119                    struct rte_crypto_sym_xform *xform,
2120                    dpaa_sec_session *session)
2121 {
2122         session->ctxt = DPAA_SEC_AUTH;
2123         session->auth_alg = xform->auth.algo;
2124         session->auth_key.length = xform->auth.key.length;
2125         if (xform->auth.key.length) {
2126                 session->auth_key.data =
2127                                 rte_zmalloc(NULL, xform->auth.key.length,
2128                                              RTE_CACHE_LINE_SIZE);
2129                 if (session->auth_key.data == NULL) {
2130                         DPAA_SEC_ERR("No Memory for auth key");
2131                         return -ENOMEM;
2132                 }
2133                 memcpy(session->auth_key.data, xform->auth.key.data,
2134                                 xform->auth.key.length);
2135
2136         }
2137         session->digest_length = xform->auth.digest_length;
2138         if (session->cipher_alg == RTE_CRYPTO_CIPHER_NULL) {
2139                 session->iv.offset = xform->auth.iv.offset;
2140                 session->iv.length = xform->auth.iv.length;
2141         }
2142
2143         switch (xform->auth.algo) {
2144         case RTE_CRYPTO_AUTH_SHA1:
2145                 session->auth_key.alg = OP_ALG_ALGSEL_SHA1;
2146                 session->auth_key.algmode = OP_ALG_AAI_HASH;
2147                 break;
2148         case RTE_CRYPTO_AUTH_SHA1_HMAC:
2149                 session->auth_key.alg = OP_ALG_ALGSEL_SHA1;
2150                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2151                 break;
2152         case RTE_CRYPTO_AUTH_MD5:
2153                 session->auth_key.alg = OP_ALG_ALGSEL_MD5;
2154                 session->auth_key.algmode = OP_ALG_AAI_HASH;
2155                 break;
2156         case RTE_CRYPTO_AUTH_MD5_HMAC:
2157                 session->auth_key.alg = OP_ALG_ALGSEL_MD5;
2158                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2159                 break;
2160         case RTE_CRYPTO_AUTH_SHA224:
2161                 session->auth_key.alg = OP_ALG_ALGSEL_SHA224;
2162                 session->auth_key.algmode = OP_ALG_AAI_HASH;
2163                 break;
2164         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2165                 session->auth_key.alg = OP_ALG_ALGSEL_SHA224;
2166                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2167                 break;
2168         case RTE_CRYPTO_AUTH_SHA256:
2169                 session->auth_key.alg = OP_ALG_ALGSEL_SHA256;
2170                 session->auth_key.algmode = OP_ALG_AAI_HASH;
2171                 break;
2172         case RTE_CRYPTO_AUTH_SHA256_HMAC:
2173                 session->auth_key.alg = OP_ALG_ALGSEL_SHA256;
2174                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2175                 break;
2176         case RTE_CRYPTO_AUTH_SHA384:
2177                 session->auth_key.alg = OP_ALG_ALGSEL_SHA384;
2178                 session->auth_key.algmode = OP_ALG_AAI_HASH;
2179                 break;
2180         case RTE_CRYPTO_AUTH_SHA384_HMAC:
2181                 session->auth_key.alg = OP_ALG_ALGSEL_SHA384;
2182                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2183                 break;
2184         case RTE_CRYPTO_AUTH_SHA512:
2185                 session->auth_key.alg = OP_ALG_ALGSEL_SHA512;
2186                 session->auth_key.algmode = OP_ALG_AAI_HASH;
2187                 break;
2188         case RTE_CRYPTO_AUTH_SHA512_HMAC:
2189                 session->auth_key.alg = OP_ALG_ALGSEL_SHA512;
2190                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2191                 break;
2192         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2193                 session->auth_key.alg = OP_ALG_ALGSEL_SNOW_F9;
2194                 session->auth_key.algmode = OP_ALG_AAI_F9;
2195                 break;
2196         case RTE_CRYPTO_AUTH_ZUC_EIA3:
2197                 session->auth_key.alg = OP_ALG_ALGSEL_ZUCA;
2198                 session->auth_key.algmode = OP_ALG_AAI_F9;
2199                 break;
2200         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2201                 session->auth_key.alg = OP_ALG_ALGSEL_AES;
2202                 session->auth_key.algmode = OP_ALG_AAI_XCBC_MAC;
2203                 break;
2204         case RTE_CRYPTO_AUTH_AES_CMAC:
2205                 session->auth_key.alg = OP_ALG_ALGSEL_AES;
2206                 session->auth_key.algmode = OP_ALG_AAI_CMAC;
2207                 break;
2208         default:
2209                 DPAA_SEC_ERR("Crypto: Unsupported Auth specified %u",
2210                               xform->auth.algo);
2211                 return -ENOTSUP;
2212         }
2213
2214         session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
2215                         DIR_ENC : DIR_DEC;
2216
2217         return 0;
2218 }
2219
2220 static int
2221 dpaa_sec_chain_init(struct rte_cryptodev *dev __rte_unused,
2222                    struct rte_crypto_sym_xform *xform,
2223                    dpaa_sec_session *session)
2224 {
2225
2226         struct rte_crypto_cipher_xform *cipher_xform;
2227         struct rte_crypto_auth_xform *auth_xform;
2228
2229         session->ctxt = DPAA_SEC_CIPHER_HASH;
2230         if (session->auth_cipher_text) {
2231                 cipher_xform = &xform->cipher;
2232                 auth_xform = &xform->next->auth;
2233         } else {
2234                 cipher_xform = &xform->next->cipher;
2235                 auth_xform = &xform->auth;
2236         }
2237
2238         /* Set IV parameters */
2239         session->iv.offset = cipher_xform->iv.offset;
2240         session->iv.length = cipher_xform->iv.length;
2241
2242         session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length,
2243                                                RTE_CACHE_LINE_SIZE);
2244         if (session->cipher_key.data == NULL && cipher_xform->key.length > 0) {
2245                 DPAA_SEC_ERR("No Memory for cipher key");
2246                 return -ENOMEM;
2247         }
2248         session->cipher_key.length = cipher_xform->key.length;
2249         session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length,
2250                                              RTE_CACHE_LINE_SIZE);
2251         if (session->auth_key.data == NULL && auth_xform->key.length > 0) {
2252                 DPAA_SEC_ERR("No Memory for auth key");
2253                 return -ENOMEM;
2254         }
2255         session->auth_key.length = auth_xform->key.length;
2256         memcpy(session->cipher_key.data, cipher_xform->key.data,
2257                cipher_xform->key.length);
2258         memcpy(session->auth_key.data, auth_xform->key.data,
2259                auth_xform->key.length);
2260
2261         session->digest_length = auth_xform->digest_length;
2262         session->auth_alg = auth_xform->algo;
2263
2264         switch (auth_xform->algo) {
2265         case RTE_CRYPTO_AUTH_SHA1_HMAC:
2266                 session->auth_key.alg = OP_ALG_ALGSEL_SHA1;
2267                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2268                 break;
2269         case RTE_CRYPTO_AUTH_MD5_HMAC:
2270                 session->auth_key.alg = OP_ALG_ALGSEL_MD5;
2271                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2272                 break;
2273         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2274                 session->auth_key.alg = OP_ALG_ALGSEL_SHA224;
2275                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2276                 break;
2277         case RTE_CRYPTO_AUTH_SHA256_HMAC:
2278                 session->auth_key.alg = OP_ALG_ALGSEL_SHA256;
2279                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2280                 break;
2281         case RTE_CRYPTO_AUTH_SHA384_HMAC:
2282                 session->auth_key.alg = OP_ALG_ALGSEL_SHA384;
2283                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2284                 break;
2285         case RTE_CRYPTO_AUTH_SHA512_HMAC:
2286                 session->auth_key.alg = OP_ALG_ALGSEL_SHA512;
2287                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2288                 break;
2289         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2290                 session->auth_key.alg = OP_ALG_ALGSEL_AES;
2291                 session->auth_key.algmode = OP_ALG_AAI_XCBC_MAC;
2292                 break;
2293         case RTE_CRYPTO_AUTH_AES_CMAC:
2294                 session->auth_key.alg = OP_ALG_ALGSEL_AES;
2295                 session->auth_key.algmode = OP_ALG_AAI_CMAC;
2296                 break;
2297         default:
2298                 DPAA_SEC_ERR("Crypto: Unsupported Auth specified %u",
2299                               auth_xform->algo);
2300                 return -ENOTSUP;
2301         }
2302
2303         session->cipher_alg = cipher_xform->algo;
2304
2305         switch (cipher_xform->algo) {
2306         case RTE_CRYPTO_CIPHER_AES_CBC:
2307                 session->cipher_key.alg = OP_ALG_ALGSEL_AES;
2308                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2309                 break;
2310         case RTE_CRYPTO_CIPHER_DES_CBC:
2311                 session->cipher_key.alg = OP_ALG_ALGSEL_DES;
2312                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2313                 break;
2314         case RTE_CRYPTO_CIPHER_3DES_CBC:
2315                 session->cipher_key.alg = OP_ALG_ALGSEL_3DES;
2316                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2317                 break;
2318         case RTE_CRYPTO_CIPHER_AES_CTR:
2319                 session->cipher_key.alg = OP_ALG_ALGSEL_AES;
2320                 session->cipher_key.algmode = OP_ALG_AAI_CTR;
2321                 break;
2322         default:
2323                 DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u",
2324                               cipher_xform->algo);
2325                 return -ENOTSUP;
2326         }
2327         session->dir = (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2328                                 DIR_ENC : DIR_DEC;
2329         return 0;
2330 }
2331
2332 static int
2333 dpaa_sec_aead_init(struct rte_cryptodev *dev __rte_unused,
2334                    struct rte_crypto_sym_xform *xform,
2335                    dpaa_sec_session *session)
2336 {
2337         session->aead_alg = xform->aead.algo;
2338         session->ctxt = DPAA_SEC_AEAD;
2339         session->iv.length = xform->aead.iv.length;
2340         session->iv.offset = xform->aead.iv.offset;
2341         session->auth_only_len = xform->aead.aad_length;
2342         session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length,
2343                                              RTE_CACHE_LINE_SIZE);
2344         if (session->aead_key.data == NULL && xform->aead.key.length > 0) {
2345                 DPAA_SEC_ERR("No Memory for aead key\n");
2346                 return -ENOMEM;
2347         }
2348         session->aead_key.length = xform->aead.key.length;
2349         session->digest_length = xform->aead.digest_length;
2350
2351         memcpy(session->aead_key.data, xform->aead.key.data,
2352                xform->aead.key.length);
2353
2354         switch (session->aead_alg) {
2355         case RTE_CRYPTO_AEAD_AES_GCM:
2356                 session->aead_key.alg = OP_ALG_ALGSEL_AES;
2357                 session->aead_key.algmode = OP_ALG_AAI_GCM;
2358                 break;
2359         default:
2360                 DPAA_SEC_ERR("unsupported AEAD alg %d", session->aead_alg);
2361                 return -ENOTSUP;
2362         }
2363
2364         session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
2365                         DIR_ENC : DIR_DEC;
2366
2367         return 0;
2368 }
2369
2370 static struct qman_fq *
2371 dpaa_sec_attach_rxq(struct dpaa_sec_dev_private *qi)
2372 {
2373         unsigned int i;
2374
2375         for (i = 0; i < RTE_DPAA_MAX_RX_QUEUE; i++) {
2376                 if (qi->inq_attach[i] == 0) {
2377                         qi->inq_attach[i] = 1;
2378                         return &qi->inq[i];
2379                 }
2380         }
2381         DPAA_SEC_WARN("All session in use %u", qi->max_nb_sessions);
2382
2383         return NULL;
2384 }
2385
2386 static int
2387 dpaa_sec_detach_rxq(struct dpaa_sec_dev_private *qi, struct qman_fq *fq)
2388 {
2389         unsigned int i;
2390
2391         for (i = 0; i < RTE_DPAA_MAX_RX_QUEUE; i++) {
2392                 if (&qi->inq[i] == fq) {
2393                         if (qman_retire_fq(fq, NULL) != 0)
2394                                 DPAA_SEC_DEBUG("Queue is not retired\n");
2395                         qman_oos_fq(fq);
2396                         qi->inq_attach[i] = 0;
2397                         return 0;
2398                 }
2399         }
2400         return -1;
2401 }
2402
2403 static int
2404 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess)
2405 {
2406         int ret;
2407
2408         sess->qp[rte_lcore_id() % MAX_DPAA_CORES] = qp;
2409         ret = dpaa_sec_prep_cdb(sess);
2410         if (ret) {
2411                 DPAA_SEC_ERR("Unable to prepare sec cdb");
2412                 return ret;
2413         }
2414         if (unlikely(!DPAA_PER_LCORE_PORTAL)) {
2415                 ret = rte_dpaa_portal_init((void *)0);
2416                 if (ret) {
2417                         DPAA_SEC_ERR("Failure in affining portal");
2418                         return ret;
2419                 }
2420         }
2421         ret = dpaa_sec_init_rx(sess->inq[rte_lcore_id() % MAX_DPAA_CORES],
2422                                rte_dpaa_mem_vtop(&sess->cdb),
2423                                qman_fq_fqid(&qp->outq));
2424         if (ret)
2425                 DPAA_SEC_ERR("Unable to init sec queue");
2426
2427         return ret;
2428 }
2429
2430 static inline void
2431 free_session_data(dpaa_sec_session *s)
2432 {
2433         if (is_aead(s))
2434                 rte_free(s->aead_key.data);
2435         else {
2436                 rte_free(s->auth_key.data);
2437                 rte_free(s->cipher_key.data);
2438         }
2439         memset(s, 0, sizeof(dpaa_sec_session));
2440 }
2441
2442 static int
2443 dpaa_sec_set_session_parameters(struct rte_cryptodev *dev,
2444                             struct rte_crypto_sym_xform *xform, void *sess)
2445 {
2446         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
2447         dpaa_sec_session *session = sess;
2448         uint32_t i;
2449         int ret;
2450
2451         PMD_INIT_FUNC_TRACE();
2452
2453         if (unlikely(sess == NULL)) {
2454                 DPAA_SEC_ERR("invalid session struct");
2455                 return -EINVAL;
2456         }
2457         memset(session, 0, sizeof(dpaa_sec_session));
2458
2459         /* Default IV length = 0 */
2460         session->iv.length = 0;
2461
2462         /* Cipher Only */
2463         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
2464                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
2465                 ret = dpaa_sec_cipher_init(dev, xform, session);
2466
2467         /* Authentication Only */
2468         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
2469                    xform->next == NULL) {
2470                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
2471                 session->ctxt = DPAA_SEC_AUTH;
2472                 ret = dpaa_sec_auth_init(dev, xform, session);
2473
2474         /* Cipher then Authenticate */
2475         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
2476                    xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
2477                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
2478                         session->auth_cipher_text = 1;
2479                         if (xform->cipher.algo == RTE_CRYPTO_CIPHER_NULL)
2480                                 ret = dpaa_sec_auth_init(dev, xform, session);
2481                         else if (xform->next->auth.algo == RTE_CRYPTO_AUTH_NULL)
2482                                 ret = dpaa_sec_cipher_init(dev, xform, session);
2483                         else
2484                                 ret = dpaa_sec_chain_init(dev, xform, session);
2485                 } else {
2486                         DPAA_SEC_ERR("Not supported: Auth then Cipher");
2487                         return -ENOTSUP;
2488                 }
2489         /* Authenticate then Cipher */
2490         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
2491                    xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
2492                 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
2493                         session->auth_cipher_text = 0;
2494                         if (xform->auth.algo == RTE_CRYPTO_AUTH_NULL)
2495                                 ret = dpaa_sec_cipher_init(dev, xform, session);
2496                         else if (xform->next->cipher.algo
2497                                         == RTE_CRYPTO_CIPHER_NULL)
2498                                 ret = dpaa_sec_auth_init(dev, xform, session);
2499                         else
2500                                 ret = dpaa_sec_chain_init(dev, xform, session);
2501                 } else {
2502                         DPAA_SEC_ERR("Not supported: Auth then Cipher");
2503                         return -ENOTSUP;
2504                 }
2505
2506         /* AEAD operation for AES-GCM kind of Algorithms */
2507         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
2508                    xform->next == NULL) {
2509                 ret = dpaa_sec_aead_init(dev, xform, session);
2510
2511         } else {
2512                 DPAA_SEC_ERR("Invalid crypto type");
2513                 return -EINVAL;
2514         }
2515         if (ret) {
2516                 DPAA_SEC_ERR("unable to init session");
2517                 goto err1;
2518         }
2519
2520         rte_spinlock_lock(&internals->lock);
2521         for (i = 0; i < MAX_DPAA_CORES; i++) {
2522                 session->inq[i] = dpaa_sec_attach_rxq(internals);
2523                 if (session->inq[i] == NULL) {
2524                         DPAA_SEC_ERR("unable to attach sec queue");
2525                         rte_spinlock_unlock(&internals->lock);
2526                         ret = -EBUSY;
2527                         goto err1;
2528                 }
2529         }
2530         rte_spinlock_unlock(&internals->lock);
2531
2532         return 0;
2533
2534 err1:
2535         free_session_data(session);
2536         return ret;
2537 }
2538
2539 static int
2540 dpaa_sec_sym_session_configure(struct rte_cryptodev *dev,
2541                 struct rte_crypto_sym_xform *xform,
2542                 struct rte_cryptodev_sym_session *sess,
2543                 struct rte_mempool *mempool)
2544 {
2545         void *sess_private_data;
2546         int ret;
2547
2548         PMD_INIT_FUNC_TRACE();
2549
2550         if (rte_mempool_get(mempool, &sess_private_data)) {
2551                 DPAA_SEC_ERR("Couldn't get object from session mempool");
2552                 return -ENOMEM;
2553         }
2554
2555         ret = dpaa_sec_set_session_parameters(dev, xform, sess_private_data);
2556         if (ret != 0) {
2557                 DPAA_SEC_ERR("failed to configure session parameters");
2558
2559                 /* Return session to mempool */
2560                 rte_mempool_put(mempool, sess_private_data);
2561                 return ret;
2562         }
2563
2564         set_sym_session_private_data(sess, dev->driver_id,
2565                         sess_private_data);
2566
2567
2568         return 0;
2569 }
2570
2571 static inline void
2572 free_session_memory(struct rte_cryptodev *dev, dpaa_sec_session *s)
2573 {
2574         struct dpaa_sec_dev_private *qi = dev->data->dev_private;
2575         struct rte_mempool *sess_mp = rte_mempool_from_obj((void *)s);
2576         uint8_t i;
2577
2578         for (i = 0; i < MAX_DPAA_CORES; i++) {
2579                 if (s->inq[i])
2580                         dpaa_sec_detach_rxq(qi, s->inq[i]);
2581                 s->inq[i] = NULL;
2582                 s->qp[i] = NULL;
2583         }
2584         free_session_data(s);
2585         rte_mempool_put(sess_mp, (void *)s);
2586 }
2587
2588 /** Clear the memory of session so it doesn't leave key material behind */
2589 static void
2590 dpaa_sec_sym_session_clear(struct rte_cryptodev *dev,
2591                 struct rte_cryptodev_sym_session *sess)
2592 {
2593         PMD_INIT_FUNC_TRACE();
2594         uint8_t index = dev->driver_id;
2595         void *sess_priv = get_sym_session_private_data(sess, index);
2596         dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
2597
2598         if (sess_priv) {
2599                 free_session_memory(dev, s);
2600                 set_sym_session_private_data(sess, index, NULL);
2601         }
2602 }
2603
2604 #ifdef RTE_LIB_SECURITY
2605 static int
2606 dpaa_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform,
2607                         struct rte_security_ipsec_xform *ipsec_xform,
2608                         dpaa_sec_session *session)
2609 {
2610         PMD_INIT_FUNC_TRACE();
2611
2612         session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length,
2613                                                RTE_CACHE_LINE_SIZE);
2614         if (session->aead_key.data == NULL && aead_xform->key.length > 0) {
2615                 DPAA_SEC_ERR("No Memory for aead key");
2616                 return -ENOMEM;
2617         }
2618         memcpy(session->aead_key.data, aead_xform->key.data,
2619                aead_xform->key.length);
2620
2621         session->digest_length = aead_xform->digest_length;
2622         session->aead_key.length = aead_xform->key.length;
2623
2624         switch (aead_xform->algo) {
2625         case RTE_CRYPTO_AEAD_AES_GCM:
2626                 switch (session->digest_length) {
2627                 case 8:
2628                         session->aead_key.alg = OP_PCL_IPSEC_AES_GCM8;
2629                         break;
2630                 case 12:
2631                         session->aead_key.alg = OP_PCL_IPSEC_AES_GCM12;
2632                         break;
2633                 case 16:
2634                         session->aead_key.alg = OP_PCL_IPSEC_AES_GCM16;
2635                         break;
2636                 default:
2637                         DPAA_SEC_ERR("Crypto: Undefined GCM digest %d",
2638                                      session->digest_length);
2639                         return -EINVAL;
2640                 }
2641                 if (session->dir == DIR_ENC) {
2642                         memcpy(session->encap_pdb.gcm.salt,
2643                                 (uint8_t *)&(ipsec_xform->salt), 4);
2644                 } else {
2645                         memcpy(session->decap_pdb.gcm.salt,
2646                                 (uint8_t *)&(ipsec_xform->salt), 4);
2647                 }
2648                 session->aead_key.algmode = OP_ALG_AAI_GCM;
2649                 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM;
2650                 break;
2651         default:
2652                 DPAA_SEC_ERR("Crypto: Undefined AEAD specified %u",
2653                               aead_xform->algo);
2654                 return -ENOTSUP;
2655         }
2656         return 0;
2657 }
2658
2659 static int
2660 dpaa_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform,
2661         struct rte_crypto_auth_xform *auth_xform,
2662         struct rte_security_ipsec_xform *ipsec_xform,
2663         dpaa_sec_session *session)
2664 {
2665         if (cipher_xform) {
2666                 session->cipher_key.data = rte_zmalloc(NULL,
2667                                                        cipher_xform->key.length,
2668                                                        RTE_CACHE_LINE_SIZE);
2669                 if (session->cipher_key.data == NULL &&
2670                                 cipher_xform->key.length > 0) {
2671                         DPAA_SEC_ERR("No Memory for cipher key");
2672                         return -ENOMEM;
2673                 }
2674
2675                 session->cipher_key.length = cipher_xform->key.length;
2676                 memcpy(session->cipher_key.data, cipher_xform->key.data,
2677                                 cipher_xform->key.length);
2678                 session->cipher_alg = cipher_xform->algo;
2679         } else {
2680                 session->cipher_key.data = NULL;
2681                 session->cipher_key.length = 0;
2682                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
2683         }
2684
2685         if (auth_xform) {
2686                 session->auth_key.data = rte_zmalloc(NULL,
2687                                                 auth_xform->key.length,
2688                                                 RTE_CACHE_LINE_SIZE);
2689                 if (session->auth_key.data == NULL &&
2690                                 auth_xform->key.length > 0) {
2691                         DPAA_SEC_ERR("No Memory for auth key");
2692                         return -ENOMEM;
2693                 }
2694                 session->auth_key.length = auth_xform->key.length;
2695                 memcpy(session->auth_key.data, auth_xform->key.data,
2696                                 auth_xform->key.length);
2697                 session->auth_alg = auth_xform->algo;
2698                 session->digest_length = auth_xform->digest_length;
2699         } else {
2700                 session->auth_key.data = NULL;
2701                 session->auth_key.length = 0;
2702                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
2703         }
2704
2705         switch (session->auth_alg) {
2706         case RTE_CRYPTO_AUTH_SHA1_HMAC:
2707                 session->auth_key.alg = OP_PCL_IPSEC_HMAC_SHA1_96;
2708                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2709                 break;
2710         case RTE_CRYPTO_AUTH_MD5_HMAC:
2711                 session->auth_key.alg = OP_PCL_IPSEC_HMAC_MD5_96;
2712                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2713                 break;
2714         case RTE_CRYPTO_AUTH_SHA256_HMAC:
2715                 session->auth_key.alg = OP_PCL_IPSEC_HMAC_SHA2_256_128;
2716                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2717                 if (session->digest_length != 16)
2718                         DPAA_SEC_WARN(
2719                         "+++Using sha256-hmac truncated len is non-standard,"
2720                         "it will not work with lookaside proto");
2721                 break;
2722         case RTE_CRYPTO_AUTH_SHA384_HMAC:
2723                 session->auth_key.alg = OP_PCL_IPSEC_HMAC_SHA2_384_192;
2724                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2725                 break;
2726         case RTE_CRYPTO_AUTH_SHA512_HMAC:
2727                 session->auth_key.alg = OP_PCL_IPSEC_HMAC_SHA2_512_256;
2728                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2729                 break;
2730         case RTE_CRYPTO_AUTH_AES_CMAC:
2731                 session->auth_key.alg = OP_PCL_IPSEC_AES_CMAC_96;
2732                 session->auth_key.algmode = OP_ALG_AAI_CMAC;
2733                 break;
2734         case RTE_CRYPTO_AUTH_NULL:
2735                 session->auth_key.alg = OP_PCL_IPSEC_HMAC_NULL;
2736                 break;
2737         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2738                 session->auth_key.alg = OP_PCL_IPSEC_AES_XCBC_MAC_96;
2739                 session->auth_key.algmode = OP_ALG_AAI_XCBC_MAC;
2740                 break;
2741         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2742         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2743         case RTE_CRYPTO_AUTH_SHA1:
2744         case RTE_CRYPTO_AUTH_SHA256:
2745         case RTE_CRYPTO_AUTH_SHA512:
2746         case RTE_CRYPTO_AUTH_SHA224:
2747         case RTE_CRYPTO_AUTH_SHA384:
2748         case RTE_CRYPTO_AUTH_MD5:
2749         case RTE_CRYPTO_AUTH_AES_GMAC:
2750         case RTE_CRYPTO_AUTH_KASUMI_F9:
2751         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2752         case RTE_CRYPTO_AUTH_ZUC_EIA3:
2753                 DPAA_SEC_ERR("Crypto: Unsupported auth alg %u",
2754                               session->auth_alg);
2755                 return -ENOTSUP;
2756         default:
2757                 DPAA_SEC_ERR("Crypto: Undefined Auth specified %u",
2758                               session->auth_alg);
2759                 return -ENOTSUP;
2760         }
2761
2762         switch (session->cipher_alg) {
2763         case RTE_CRYPTO_CIPHER_AES_CBC:
2764                 session->cipher_key.alg = OP_PCL_IPSEC_AES_CBC;
2765                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2766                 break;
2767         case RTE_CRYPTO_CIPHER_DES_CBC:
2768                 session->cipher_key.alg = OP_PCL_IPSEC_DES;
2769                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2770                 break;
2771         case RTE_CRYPTO_CIPHER_3DES_CBC:
2772                 session->cipher_key.alg = OP_PCL_IPSEC_3DES;
2773                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2774                 break;
2775         case RTE_CRYPTO_CIPHER_AES_CTR:
2776                 session->cipher_key.alg = OP_PCL_IPSEC_AES_CTR;
2777                 session->cipher_key.algmode = OP_ALG_AAI_CTR;
2778                 if (session->dir == DIR_ENC) {
2779                         session->encap_pdb.ctr.ctr_initial = 0x00000001;
2780                         session->encap_pdb.ctr.ctr_nonce = ipsec_xform->salt;
2781                 } else {
2782                         session->decap_pdb.ctr.ctr_initial = 0x00000001;
2783                         session->decap_pdb.ctr.ctr_nonce = ipsec_xform->salt;
2784                 }
2785                 break;
2786         case RTE_CRYPTO_CIPHER_NULL:
2787                 session->cipher_key.alg = OP_PCL_IPSEC_NULL;
2788                 break;
2789         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2790         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2791         case RTE_CRYPTO_CIPHER_3DES_ECB:
2792         case RTE_CRYPTO_CIPHER_AES_ECB:
2793         case RTE_CRYPTO_CIPHER_KASUMI_F8:
2794                 DPAA_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2795                               session->cipher_alg);
2796                 return -ENOTSUP;
2797         default:
2798                 DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u",
2799                               session->cipher_alg);
2800                 return -ENOTSUP;
2801         }
2802
2803         return 0;
2804 }
2805
2806 static int
2807 dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
2808                            struct rte_security_session_conf *conf,
2809                            void *sess)
2810 {
2811         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
2812         struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
2813         struct rte_crypto_auth_xform *auth_xform = NULL;
2814         struct rte_crypto_cipher_xform *cipher_xform = NULL;
2815         struct rte_crypto_aead_xform *aead_xform = NULL;
2816         dpaa_sec_session *session = (dpaa_sec_session *)sess;
2817         uint32_t i;
2818         int ret;
2819
2820         PMD_INIT_FUNC_TRACE();
2821
2822         memset(session, 0, sizeof(dpaa_sec_session));
2823         session->proto_alg = conf->protocol;
2824         session->ctxt = DPAA_SEC_IPSEC;
2825
2826         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS)
2827                 session->dir = DIR_ENC;
2828         else
2829                 session->dir = DIR_DEC;
2830
2831         if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
2832                 cipher_xform = &conf->crypto_xform->cipher;
2833                 if (conf->crypto_xform->next)
2834                         auth_xform = &conf->crypto_xform->next->auth;
2835                 ret = dpaa_sec_ipsec_proto_init(cipher_xform, auth_xform,
2836                                         ipsec_xform, session);
2837         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
2838                 auth_xform = &conf->crypto_xform->auth;
2839                 if (conf->crypto_xform->next)
2840                         cipher_xform = &conf->crypto_xform->next->cipher;
2841                 ret = dpaa_sec_ipsec_proto_init(cipher_xform, auth_xform,
2842                                         ipsec_xform, session);
2843         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
2844                 aead_xform = &conf->crypto_xform->aead;
2845                 ret = dpaa_sec_ipsec_aead_init(aead_xform,
2846                                         ipsec_xform, session);
2847         } else {
2848                 DPAA_SEC_ERR("XFORM not specified");
2849                 ret = -EINVAL;
2850                 goto out;
2851         }
2852         if (ret) {
2853                 DPAA_SEC_ERR("Failed to process xform");
2854                 goto out;
2855         }
2856
2857         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
2858                 if (ipsec_xform->tunnel.type ==
2859                                 RTE_SECURITY_IPSEC_TUNNEL_IPV4) {
2860                         session->ip4_hdr.ip_v = IPVERSION;
2861                         session->ip4_hdr.ip_hl = 5;
2862                         session->ip4_hdr.ip_len = rte_cpu_to_be_16(
2863                                                 sizeof(session->ip4_hdr));
2864                         session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
2865                         session->ip4_hdr.ip_id = 0;
2866                         session->ip4_hdr.ip_off = 0;
2867                         session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
2868                         session->ip4_hdr.ip_p = (ipsec_xform->proto ==
2869                                         RTE_SECURITY_IPSEC_SA_PROTO_ESP) ?
2870                                         IPPROTO_ESP : IPPROTO_AH;
2871                         session->ip4_hdr.ip_sum = 0;
2872                         session->ip4_hdr.ip_src =
2873                                         ipsec_xform->tunnel.ipv4.src_ip;
2874                         session->ip4_hdr.ip_dst =
2875                                         ipsec_xform->tunnel.ipv4.dst_ip;
2876                         session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
2877                                                 (void *)&session->ip4_hdr,
2878                                                 sizeof(struct ip));
2879                         session->encap_pdb.ip_hdr_len = sizeof(struct ip);
2880                 } else if (ipsec_xform->tunnel.type ==
2881                                 RTE_SECURITY_IPSEC_TUNNEL_IPV6) {
2882                         session->ip6_hdr.vtc_flow = rte_cpu_to_be_32(
2883                                 DPAA_IPv6_DEFAULT_VTC_FLOW |
2884                                 ((ipsec_xform->tunnel.ipv6.dscp <<
2885                                         RTE_IPV6_HDR_TC_SHIFT) &
2886                                         RTE_IPV6_HDR_TC_MASK) |
2887                                 ((ipsec_xform->tunnel.ipv6.flabel <<
2888                                         RTE_IPV6_HDR_FL_SHIFT) &
2889                                         RTE_IPV6_HDR_FL_MASK));
2890                         /* Payload length will be updated by HW */
2891                         session->ip6_hdr.payload_len = 0;
2892                         session->ip6_hdr.hop_limits =
2893                                         ipsec_xform->tunnel.ipv6.hlimit;
2894                         session->ip6_hdr.proto = (ipsec_xform->proto ==
2895                                         RTE_SECURITY_IPSEC_SA_PROTO_ESP) ?
2896                                         IPPROTO_ESP : IPPROTO_AH;
2897                         memcpy(&session->ip6_hdr.src_addr,
2898                                         &ipsec_xform->tunnel.ipv6.src_addr, 16);
2899                         memcpy(&session->ip6_hdr.dst_addr,
2900                                         &ipsec_xform->tunnel.ipv6.dst_addr, 16);
2901                         session->encap_pdb.ip_hdr_len =
2902                                                 sizeof(struct rte_ipv6_hdr);
2903                 }
2904                 session->encap_pdb.options =
2905                         (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
2906                         PDBOPTS_ESP_OIHI_PDB_INL |
2907                         PDBOPTS_ESP_IVSRC |
2908                         PDBHMO_ESP_ENCAP_DTTL |
2909                         PDBHMO_ESP_SNR;
2910                 if (ipsec_xform->options.esn)
2911                         session->encap_pdb.options |= PDBOPTS_ESP_ESN;
2912                 session->encap_pdb.spi = ipsec_xform->spi;
2913
2914         } else if (ipsec_xform->direction ==
2915                         RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
2916                 if (ipsec_xform->tunnel.type == RTE_SECURITY_IPSEC_TUNNEL_IPV4)
2917                         session->decap_pdb.options = sizeof(struct ip) << 16;
2918                 else
2919                         session->decap_pdb.options =
2920                                         sizeof(struct rte_ipv6_hdr) << 16;
2921                 if (ipsec_xform->options.esn)
2922                         session->decap_pdb.options |= PDBOPTS_ESP_ESN;
2923                 if (ipsec_xform->replay_win_sz) {
2924                         uint32_t win_sz;
2925                         win_sz = rte_align32pow2(ipsec_xform->replay_win_sz);
2926
2927                         switch (win_sz) {
2928                         case 1:
2929                         case 2:
2930                         case 4:
2931                         case 8:
2932                         case 16:
2933                         case 32:
2934                                 session->decap_pdb.options |= PDBOPTS_ESP_ARS32;
2935                                 break;
2936                         case 64:
2937                                 session->decap_pdb.options |= PDBOPTS_ESP_ARS64;
2938                                 break;
2939                         default:
2940                                 session->decap_pdb.options |=
2941                                                         PDBOPTS_ESP_ARS128;
2942                         }
2943                 }
2944         } else
2945                 goto out;
2946         rte_spinlock_lock(&internals->lock);
2947         for (i = 0; i < MAX_DPAA_CORES; i++) {
2948                 session->inq[i] = dpaa_sec_attach_rxq(internals);
2949                 if (session->inq[i] == NULL) {
2950                         DPAA_SEC_ERR("unable to attach sec queue");
2951                         rte_spinlock_unlock(&internals->lock);
2952                         goto out;
2953                 }
2954         }
2955         rte_spinlock_unlock(&internals->lock);
2956
2957         return 0;
2958 out:
2959         free_session_data(session);
2960         return -1;
2961 }
2962
2963 static int
2964 dpaa_sec_set_pdcp_session(struct rte_cryptodev *dev,
2965                           struct rte_security_session_conf *conf,
2966                           void *sess)
2967 {
2968         struct rte_security_pdcp_xform *pdcp_xform = &conf->pdcp;
2969         struct rte_crypto_sym_xform *xform = conf->crypto_xform;
2970         struct rte_crypto_auth_xform *auth_xform = NULL;
2971         struct rte_crypto_cipher_xform *cipher_xform = NULL;
2972         dpaa_sec_session *session = (dpaa_sec_session *)sess;
2973         struct dpaa_sec_dev_private *dev_priv = dev->data->dev_private;
2974         uint32_t i;
2975         int ret;
2976
2977         PMD_INIT_FUNC_TRACE();
2978
2979         memset(session, 0, sizeof(dpaa_sec_session));
2980
2981         /* find xfrm types */
2982         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
2983                 cipher_xform = &xform->cipher;
2984                 if (xform->next != NULL)
2985                         auth_xform = &xform->next->auth;
2986         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
2987                 auth_xform = &xform->auth;
2988                 if (xform->next != NULL)
2989                         cipher_xform = &xform->next->cipher;
2990         } else {
2991                 DPAA_SEC_ERR("Invalid crypto type");
2992                 return -EINVAL;
2993         }
2994
2995         session->proto_alg = conf->protocol;
2996         session->ctxt = DPAA_SEC_PDCP;
2997
2998         if (cipher_xform) {
2999                 switch (cipher_xform->algo) {
3000                 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
3001                         session->cipher_key.alg = PDCP_CIPHER_TYPE_SNOW;
3002                         break;
3003                 case RTE_CRYPTO_CIPHER_ZUC_EEA3:
3004                         session->cipher_key.alg = PDCP_CIPHER_TYPE_ZUC;
3005                         break;
3006                 case RTE_CRYPTO_CIPHER_AES_CTR:
3007                         session->cipher_key.alg = PDCP_CIPHER_TYPE_AES;
3008                         break;
3009                 case RTE_CRYPTO_CIPHER_NULL:
3010                         session->cipher_key.alg = PDCP_CIPHER_TYPE_NULL;
3011                         break;
3012                 default:
3013                         DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u",
3014                                       session->cipher_alg);
3015                         return -EINVAL;
3016                 }
3017
3018                 session->cipher_key.data = rte_zmalloc(NULL,
3019                                                cipher_xform->key.length,
3020                                                RTE_CACHE_LINE_SIZE);
3021                 if (session->cipher_key.data == NULL &&
3022                                 cipher_xform->key.length > 0) {
3023                         DPAA_SEC_ERR("No Memory for cipher key");
3024                         return -ENOMEM;
3025                 }
3026                 session->cipher_key.length = cipher_xform->key.length;
3027                 memcpy(session->cipher_key.data, cipher_xform->key.data,
3028                         cipher_xform->key.length);
3029                 session->dir = (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
3030                                         DIR_ENC : DIR_DEC;
3031                 session->cipher_alg = cipher_xform->algo;
3032         } else {
3033                 session->cipher_key.data = NULL;
3034                 session->cipher_key.length = 0;
3035                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
3036                 session->dir = DIR_ENC;
3037         }
3038
3039         if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
3040                 if (pdcp_xform->sn_size != RTE_SECURITY_PDCP_SN_SIZE_5 &&
3041                     pdcp_xform->sn_size != RTE_SECURITY_PDCP_SN_SIZE_12) {
3042                         DPAA_SEC_ERR(
3043                                 "PDCP Seq Num size should be 5/12 bits for cmode");
3044                         ret = -EINVAL;
3045                         goto out;
3046                 }
3047         }
3048
3049         if (auth_xform) {
3050                 switch (auth_xform->algo) {
3051                 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
3052                         session->auth_key.alg = PDCP_AUTH_TYPE_SNOW;
3053                         break;
3054                 case RTE_CRYPTO_AUTH_ZUC_EIA3:
3055                         session->auth_key.alg = PDCP_AUTH_TYPE_ZUC;
3056                         break;
3057                 case RTE_CRYPTO_AUTH_AES_CMAC:
3058                         session->auth_key.alg = PDCP_AUTH_TYPE_AES;
3059                         break;
3060                 case RTE_CRYPTO_AUTH_NULL:
3061                         session->auth_key.alg = PDCP_AUTH_TYPE_NULL;
3062                         break;
3063                 default:
3064                         DPAA_SEC_ERR("Crypto: Unsupported auth alg %u",
3065                                       session->auth_alg);
3066                         rte_free(session->cipher_key.data);
3067                         return -EINVAL;
3068                 }
3069                 session->auth_key.data = rte_zmalloc(NULL,
3070                                                      auth_xform->key.length,
3071                                                      RTE_CACHE_LINE_SIZE);
3072                 if (!session->auth_key.data &&
3073                     auth_xform->key.length > 0) {
3074                         DPAA_SEC_ERR("No Memory for auth key");
3075                         rte_free(session->cipher_key.data);
3076                         return -ENOMEM;
3077                 }
3078                 session->auth_key.length = auth_xform->key.length;
3079                 memcpy(session->auth_key.data, auth_xform->key.data,
3080                        auth_xform->key.length);
3081                 session->auth_alg = auth_xform->algo;
3082         } else {
3083                 session->auth_key.data = NULL;
3084                 session->auth_key.length = 0;
3085                 session->auth_alg = 0;
3086         }
3087         session->pdcp.domain = pdcp_xform->domain;
3088         session->pdcp.bearer = pdcp_xform->bearer;
3089         session->pdcp.pkt_dir = pdcp_xform->pkt_dir;
3090         session->pdcp.sn_size = pdcp_xform->sn_size;
3091         session->pdcp.hfn = pdcp_xform->hfn;
3092         session->pdcp.hfn_threshold = pdcp_xform->hfn_threshold;
3093         session->pdcp.hfn_ovd = pdcp_xform->hfn_ovrd;
3094         session->pdcp.sdap_enabled = pdcp_xform->sdap_enabled;
3095         if (cipher_xform)
3096                 session->pdcp.hfn_ovd_offset = cipher_xform->iv.offset;
3097
3098         rte_spinlock_lock(&dev_priv->lock);
3099         for (i = 0; i < MAX_DPAA_CORES; i++) {
3100                 session->inq[i] = dpaa_sec_attach_rxq(dev_priv);
3101                 if (session->inq[i] == NULL) {
3102                         DPAA_SEC_ERR("unable to attach sec queue");
3103                         rte_spinlock_unlock(&dev_priv->lock);
3104                         ret = -EBUSY;
3105                         goto out;
3106                 }
3107         }
3108         rte_spinlock_unlock(&dev_priv->lock);
3109         return 0;
3110 out:
3111         rte_free(session->auth_key.data);
3112         rte_free(session->cipher_key.data);
3113         memset(session, 0, sizeof(dpaa_sec_session));
3114         return ret;
3115 }
3116
3117 static int
3118 dpaa_sec_security_session_create(void *dev,
3119                                  struct rte_security_session_conf *conf,
3120                                  struct rte_security_session *sess,
3121                                  struct rte_mempool *mempool)
3122 {
3123         void *sess_private_data;
3124         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
3125         int ret;
3126
3127         if (rte_mempool_get(mempool, &sess_private_data)) {
3128                 DPAA_SEC_ERR("Couldn't get object from session mempool");
3129                 return -ENOMEM;
3130         }
3131
3132         switch (conf->protocol) {
3133         case RTE_SECURITY_PROTOCOL_IPSEC:
3134                 ret = dpaa_sec_set_ipsec_session(cdev, conf,
3135                                 sess_private_data);
3136                 break;
3137         case RTE_SECURITY_PROTOCOL_PDCP:
3138                 ret = dpaa_sec_set_pdcp_session(cdev, conf,
3139                                 sess_private_data);
3140                 break;
3141         case RTE_SECURITY_PROTOCOL_MACSEC:
3142                 return -ENOTSUP;
3143         default:
3144                 return -EINVAL;
3145         }
3146         if (ret != 0) {
3147                 DPAA_SEC_ERR("failed to configure session parameters");
3148                 /* Return session to mempool */
3149                 rte_mempool_put(mempool, sess_private_data);
3150                 return ret;
3151         }
3152
3153         set_sec_session_private_data(sess, sess_private_data);
3154
3155         return ret;
3156 }
3157
3158 /** Clear the memory of session so it doesn't leave key material behind */
3159 static int
3160 dpaa_sec_security_session_destroy(void *dev __rte_unused,
3161                 struct rte_security_session *sess)
3162 {
3163         PMD_INIT_FUNC_TRACE();
3164         void *sess_priv = get_sec_session_private_data(sess);
3165         dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
3166
3167         if (sess_priv) {
3168                 free_session_memory((struct rte_cryptodev *)dev, s);
3169                 set_sec_session_private_data(sess, NULL);
3170         }
3171         return 0;
3172 }
3173 #endif
3174 static int
3175 dpaa_sec_dev_configure(struct rte_cryptodev *dev __rte_unused,
3176                        struct rte_cryptodev_config *config __rte_unused)
3177 {
3178         PMD_INIT_FUNC_TRACE();
3179
3180         return 0;
3181 }
3182
3183 static int
3184 dpaa_sec_dev_start(struct rte_cryptodev *dev __rte_unused)
3185 {
3186         PMD_INIT_FUNC_TRACE();
3187         return 0;
3188 }
3189
3190 static void
3191 dpaa_sec_dev_stop(struct rte_cryptodev *dev __rte_unused)
3192 {
3193         PMD_INIT_FUNC_TRACE();
3194 }
3195
3196 static int
3197 dpaa_sec_dev_close(struct rte_cryptodev *dev)
3198 {
3199         PMD_INIT_FUNC_TRACE();
3200
3201         if (dev == NULL)
3202                 return -ENOMEM;
3203
3204         return 0;
3205 }
3206
3207 static void
3208 dpaa_sec_dev_infos_get(struct rte_cryptodev *dev,
3209                        struct rte_cryptodev_info *info)
3210 {
3211         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
3212
3213         PMD_INIT_FUNC_TRACE();
3214         if (info != NULL) {
3215                 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
3216                 info->feature_flags = dev->feature_flags;
3217                 info->capabilities = dpaa_sec_capabilities;
3218                 info->sym.max_nb_sessions = internals->max_nb_sessions;
3219                 info->driver_id = cryptodev_driver_id;
3220         }
3221 }
3222
3223 static enum qman_cb_dqrr_result
3224 dpaa_sec_process_parallel_event(void *event,
3225                         struct qman_portal *qm __always_unused,
3226                         struct qman_fq *outq,
3227                         const struct qm_dqrr_entry *dqrr,
3228                         void **bufs)
3229 {
3230         const struct qm_fd *fd;
3231         struct dpaa_sec_job *job;
3232         struct dpaa_sec_op_ctx *ctx;
3233         struct rte_event *ev = (struct rte_event *)event;
3234
3235         fd = &dqrr->fd;
3236
3237         /* sg is embedded in an op ctx,
3238          * sg[0] is for output
3239          * sg[1] for input
3240          */
3241         job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd));
3242
3243         ctx = container_of(job, struct dpaa_sec_op_ctx, job);
3244         ctx->fd_status = fd->status;
3245         if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
3246                 struct qm_sg_entry *sg_out;
3247                 uint32_t len;
3248
3249                 sg_out = &job->sg[0];
3250                 hw_sg_to_cpu(sg_out);
3251                 len = sg_out->length;
3252                 ctx->op->sym->m_src->pkt_len = len;
3253                 ctx->op->sym->m_src->data_len = len;
3254         }
3255         if (!ctx->fd_status) {
3256                 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
3257         } else {
3258                 DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status);
3259                 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
3260         }
3261         ev->event_ptr = (void *)ctx->op;
3262
3263         ev->flow_id = outq->ev.flow_id;
3264         ev->sub_event_type = outq->ev.sub_event_type;
3265         ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
3266         ev->op = RTE_EVENT_OP_NEW;
3267         ev->sched_type = outq->ev.sched_type;
3268         ev->queue_id = outq->ev.queue_id;
3269         ev->priority = outq->ev.priority;
3270         *bufs = (void *)ctx->op;
3271
3272         rte_mempool_put(ctx->ctx_pool, (void *)ctx);
3273
3274         return qman_cb_dqrr_consume;
3275 }
3276
3277 static enum qman_cb_dqrr_result
3278 dpaa_sec_process_atomic_event(void *event,
3279                         struct qman_portal *qm __rte_unused,
3280                         struct qman_fq *outq,
3281                         const struct qm_dqrr_entry *dqrr,
3282                         void **bufs)
3283 {
3284         u8 index;
3285         const struct qm_fd *fd;
3286         struct dpaa_sec_job *job;
3287         struct dpaa_sec_op_ctx *ctx;
3288         struct rte_event *ev = (struct rte_event *)event;
3289
3290         fd = &dqrr->fd;
3291
3292         /* sg is embedded in an op ctx,
3293          * sg[0] is for output
3294          * sg[1] for input
3295          */
3296         job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd));
3297
3298         ctx = container_of(job, struct dpaa_sec_op_ctx, job);
3299         ctx->fd_status = fd->status;
3300         if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
3301                 struct qm_sg_entry *sg_out;
3302                 uint32_t len;
3303
3304                 sg_out = &job->sg[0];
3305                 hw_sg_to_cpu(sg_out);
3306                 len = sg_out->length;
3307                 ctx->op->sym->m_src->pkt_len = len;
3308                 ctx->op->sym->m_src->data_len = len;
3309         }
3310         if (!ctx->fd_status) {
3311                 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
3312         } else {
3313                 DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status);
3314                 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
3315         }
3316         ev->event_ptr = (void *)ctx->op;
3317         ev->flow_id = outq->ev.flow_id;
3318         ev->sub_event_type = outq->ev.sub_event_type;
3319         ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
3320         ev->op = RTE_EVENT_OP_NEW;
3321         ev->sched_type = outq->ev.sched_type;
3322         ev->queue_id = outq->ev.queue_id;
3323         ev->priority = outq->ev.priority;
3324
3325         /* Save active dqrr entries */
3326         index = ((uintptr_t)dqrr >> 6) & (16/*QM_DQRR_SIZE*/ - 1);
3327         DPAA_PER_LCORE_DQRR_SIZE++;
3328         DPAA_PER_LCORE_DQRR_HELD |= 1 << index;
3329         DPAA_PER_LCORE_DQRR_MBUF(index) = ctx->op->sym->m_src;
3330         ev->impl_opaque = index + 1;
3331         *dpaa_seqn(ctx->op->sym->m_src) = (uint32_t)index + 1;
3332         *bufs = (void *)ctx->op;
3333
3334         rte_mempool_put(ctx->ctx_pool, (void *)ctx);
3335
3336         return qman_cb_dqrr_defer;
3337 }
3338
3339 int
3340 dpaa_sec_eventq_attach(const struct rte_cryptodev *dev,
3341                 int qp_id,
3342                 uint16_t ch_id,
3343                 const struct rte_event *event)
3344 {
3345         struct dpaa_sec_qp *qp = dev->data->queue_pairs[qp_id];
3346         struct qm_mcc_initfq opts = {0};
3347
3348         int ret;
3349
3350         opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
3351                        QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB;
3352         opts.fqd.dest.channel = ch_id;
3353
3354         switch (event->sched_type) {
3355         case RTE_SCHED_TYPE_ATOMIC:
3356                 opts.fqd.fq_ctrl |= QM_FQCTRL_HOLDACTIVE;
3357                 /* Reset FQCTRL_AVOIDBLOCK bit as it is unnecessary
3358                  * configuration with HOLD_ACTIVE setting
3359                  */
3360                 opts.fqd.fq_ctrl &= (~QM_FQCTRL_AVOIDBLOCK);
3361                 qp->outq.cb.dqrr_dpdk_cb = dpaa_sec_process_atomic_event;
3362                 break;
3363         case RTE_SCHED_TYPE_ORDERED:
3364                 DPAA_SEC_ERR("Ordered queue schedule type is not supported\n");
3365                 return -ENOTSUP;
3366         default:
3367                 opts.fqd.fq_ctrl |= QM_FQCTRL_AVOIDBLOCK;
3368                 qp->outq.cb.dqrr_dpdk_cb = dpaa_sec_process_parallel_event;
3369                 break;
3370         }
3371
3372         ret = qman_init_fq(&qp->outq, QMAN_INITFQ_FLAG_SCHED, &opts);
3373         if (unlikely(ret)) {
3374                 DPAA_SEC_ERR("unable to init caam source fq!");
3375                 return ret;
3376         }
3377
3378         memcpy(&qp->outq.ev, event, sizeof(struct rte_event));
3379
3380         return 0;
3381 }
3382
3383 int
3384 dpaa_sec_eventq_detach(const struct rte_cryptodev *dev,
3385                         int qp_id)
3386 {
3387         struct qm_mcc_initfq opts = {0};
3388         int ret;
3389         struct dpaa_sec_qp *qp = dev->data->queue_pairs[qp_id];
3390
3391         opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
3392                        QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB;
3393         qp->outq.cb.dqrr = dqrr_out_fq_cb_rx;
3394         qp->outq.cb.ern  = ern_sec_fq_handler;
3395         qman_retire_fq(&qp->outq, NULL);
3396         qman_oos_fq(&qp->outq);
3397         ret = qman_init_fq(&qp->outq, 0, &opts);
3398         if (ret)
3399                 RTE_LOG(ERR, PMD, "Error in qman_init_fq: ret: %d\n", ret);
3400         qp->outq.cb.dqrr = NULL;
3401
3402         return ret;
3403 }
3404
3405 static struct rte_cryptodev_ops crypto_ops = {
3406         .dev_configure        = dpaa_sec_dev_configure,
3407         .dev_start            = dpaa_sec_dev_start,
3408         .dev_stop             = dpaa_sec_dev_stop,
3409         .dev_close            = dpaa_sec_dev_close,
3410         .dev_infos_get        = dpaa_sec_dev_infos_get,
3411         .queue_pair_setup     = dpaa_sec_queue_pair_setup,
3412         .queue_pair_release   = dpaa_sec_queue_pair_release,
3413         .sym_session_get_size     = dpaa_sec_sym_session_get_size,
3414         .sym_session_configure    = dpaa_sec_sym_session_configure,
3415         .sym_session_clear        = dpaa_sec_sym_session_clear
3416 };
3417
3418 #ifdef RTE_LIB_SECURITY
3419 static const struct rte_security_capability *
3420 dpaa_sec_capabilities_get(void *device __rte_unused)
3421 {
3422         return dpaa_sec_security_cap;
3423 }
3424
3425 static const struct rte_security_ops dpaa_sec_security_ops = {
3426         .session_create = dpaa_sec_security_session_create,
3427         .session_update = NULL,
3428         .session_stats_get = NULL,
3429         .session_destroy = dpaa_sec_security_session_destroy,
3430         .set_pkt_metadata = NULL,
3431         .capabilities_get = dpaa_sec_capabilities_get
3432 };
3433 #endif
3434 static int
3435 dpaa_sec_uninit(struct rte_cryptodev *dev)
3436 {
3437         struct dpaa_sec_dev_private *internals;
3438
3439         if (dev == NULL)
3440                 return -ENODEV;
3441
3442         internals = dev->data->dev_private;
3443         rte_free(dev->security_ctx);
3444
3445         rte_free(internals);
3446
3447         DPAA_SEC_INFO("Closing DPAA_SEC device %s on numa socket %u",
3448                       dev->data->name, rte_socket_id());
3449
3450         return 0;
3451 }
3452
3453 static int
3454 dpaa_sec_dev_init(struct rte_cryptodev *cryptodev)
3455 {
3456         struct dpaa_sec_dev_private *internals;
3457 #ifdef RTE_LIB_SECURITY
3458         struct rte_security_ctx *security_instance;
3459 #endif
3460         struct dpaa_sec_qp *qp;
3461         uint32_t i, flags;
3462         int ret;
3463
3464         PMD_INIT_FUNC_TRACE();
3465
3466         cryptodev->driver_id = cryptodev_driver_id;
3467         cryptodev->dev_ops = &crypto_ops;
3468
3469         cryptodev->enqueue_burst = dpaa_sec_enqueue_burst;
3470         cryptodev->dequeue_burst = dpaa_sec_dequeue_burst;
3471         cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
3472                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
3473                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
3474                         RTE_CRYPTODEV_FF_SECURITY |
3475                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
3476                         RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
3477                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
3478                         RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
3479                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
3480
3481         internals = cryptodev->data->dev_private;
3482         internals->max_nb_queue_pairs = RTE_DPAA_MAX_NB_SEC_QPS;
3483         internals->max_nb_sessions = RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS;
3484
3485         /*
3486          * For secondary processes, we don't initialise any further as primary
3487          * has already done this work. Only check we don't need a different
3488          * RX function
3489          */
3490         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3491                 DPAA_SEC_WARN("Device already init by primary process");
3492                 return 0;
3493         }
3494 #ifdef RTE_LIB_SECURITY
3495         /* Initialize security_ctx only for primary process*/
3496         security_instance = rte_malloc("rte_security_instances_ops",
3497                                 sizeof(struct rte_security_ctx), 0);
3498         if (security_instance == NULL)
3499                 return -ENOMEM;
3500         security_instance->device = (void *)cryptodev;
3501         security_instance->ops = &dpaa_sec_security_ops;
3502         security_instance->sess_cnt = 0;
3503         cryptodev->security_ctx = security_instance;
3504 #endif
3505         rte_spinlock_init(&internals->lock);
3506         for (i = 0; i < internals->max_nb_queue_pairs; i++) {
3507                 /* init qman fq for queue pair */
3508                 qp = &internals->qps[i];
3509                 ret = dpaa_sec_init_tx(&qp->outq);
3510                 if (ret) {
3511                         DPAA_SEC_ERR("config tx of queue pair  %d", i);
3512                         goto init_error;
3513                 }
3514         }
3515
3516         flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID |
3517                 QMAN_FQ_FLAG_TO_DCPORTAL;
3518         for (i = 0; i < RTE_DPAA_MAX_RX_QUEUE; i++) {
3519                 /* create rx qman fq for sessions*/
3520                 ret = qman_create_fq(0, flags, &internals->inq[i]);
3521                 if (unlikely(ret != 0)) {
3522                         DPAA_SEC_ERR("sec qman_create_fq failed");
3523                         goto init_error;
3524                 }
3525         }
3526
3527         RTE_LOG(INFO, PMD, "%s cryptodev init\n", cryptodev->data->name);
3528         return 0;
3529
3530 init_error:
3531         DPAA_SEC_ERR("driver %s: create failed\n", cryptodev->data->name);
3532
3533         rte_free(cryptodev->security_ctx);
3534         return -EFAULT;
3535 }
3536
3537 static int
3538 cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv __rte_unused,
3539                                 struct rte_dpaa_device *dpaa_dev)
3540 {
3541         struct rte_cryptodev *cryptodev;
3542         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
3543
3544         int retval;
3545
3546         snprintf(cryptodev_name, sizeof(cryptodev_name), "%s", dpaa_dev->name);
3547
3548         cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id());
3549         if (cryptodev == NULL)
3550                 return -ENOMEM;
3551
3552         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
3553                 cryptodev->data->dev_private = rte_zmalloc_socket(
3554                                         "cryptodev private structure",
3555                                         sizeof(struct dpaa_sec_dev_private),
3556                                         RTE_CACHE_LINE_SIZE,
3557                                         rte_socket_id());
3558
3559                 if (cryptodev->data->dev_private == NULL)
3560                         rte_panic("Cannot allocate memzone for private "
3561                                         "device data");
3562         }
3563
3564         dpaa_dev->crypto_dev = cryptodev;
3565         cryptodev->device = &dpaa_dev->device;
3566
3567         /* init user callbacks */
3568         TAILQ_INIT(&(cryptodev->link_intr_cbs));
3569
3570         /* if sec device version is not configured */
3571         if (!rta_get_sec_era()) {
3572                 const struct device_node *caam_node;
3573
3574                 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
3575                         const uint32_t *prop = of_get_property(caam_node,
3576                                         "fsl,sec-era",
3577                                         NULL);
3578                         if (prop) {
3579                                 rta_set_sec_era(
3580                                         INTL_SEC_ERA(rte_cpu_to_be_32(*prop)));
3581                                 break;
3582                         }
3583                 }
3584         }
3585
3586         if (unlikely(!DPAA_PER_LCORE_PORTAL)) {
3587                 retval = rte_dpaa_portal_init((void *)1);
3588                 if (retval) {
3589                         DPAA_SEC_ERR("Unable to initialize portal");
3590                         goto out;
3591                 }
3592         }
3593
3594         /* Invoke PMD device initialization function */
3595         retval = dpaa_sec_dev_init(cryptodev);
3596         if (retval == 0)
3597                 return 0;
3598
3599         retval = -ENXIO;
3600 out:
3601         /* In case of error, cleanup is done */
3602         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
3603                 rte_free(cryptodev->data->dev_private);
3604
3605         rte_cryptodev_pmd_release_device(cryptodev);
3606
3607         return retval;
3608 }
3609
3610 static int
3611 cryptodev_dpaa_sec_remove(struct rte_dpaa_device *dpaa_dev)
3612 {
3613         struct rte_cryptodev *cryptodev;
3614         int ret;
3615
3616         cryptodev = dpaa_dev->crypto_dev;
3617         if (cryptodev == NULL)
3618                 return -ENODEV;
3619
3620         ret = dpaa_sec_uninit(cryptodev);
3621         if (ret)
3622                 return ret;
3623
3624         return rte_cryptodev_pmd_destroy(cryptodev);
3625 }
3626
3627 static struct rte_dpaa_driver rte_dpaa_sec_driver = {
3628         .drv_type = FSL_DPAA_CRYPTO,
3629         .driver = {
3630                 .name = "DPAA SEC PMD"
3631         },
3632         .probe = cryptodev_dpaa_sec_probe,
3633         .remove = cryptodev_dpaa_sec_remove,
3634 };
3635
3636 static struct cryptodev_driver dpaa_sec_crypto_drv;
3637
3638 RTE_PMD_REGISTER_DPAA(CRYPTODEV_NAME_DPAA_SEC_PMD, rte_dpaa_sec_driver);
3639 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa_sec_crypto_drv, rte_dpaa_sec_driver.driver,
3640                 cryptodev_driver_id);
3641 RTE_LOG_REGISTER(dpaa_logtype_sec, pmd.crypto.dpaa, NOTICE);