drivers/crypto/dpaa_sec/dpaa_sec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2017-2022 NXP
   5  *
   6  */
   7
   8 #include <fcntl.h>
   9 #include <unistd.h>
  10 #include <sched.h>
  11 #include <net/if.h>
  12
  13 #include <rte_byteorder.h>
  14 #include <rte_common.h>
  15 #include <cryptodev_pmd.h>
  16 #include <rte_crypto.h>
  17 #include <rte_cryptodev.h>
  18 #ifdef RTE_LIB_SECURITY
  19 #include <rte_security_driver.h>
  20 #endif
  21 #include <rte_cycles.h>
  22 #include <rte_dev.h>
  23 #include <rte_io.h>
  24 #include <rte_ip.h>
  25 #include <rte_kvargs.h>
  26 #include <rte_malloc.h>
  27 #include <rte_mbuf.h>
  28 #include <rte_memcpy.h>
  29 #include <rte_string_fns.h>
  30 #include <rte_spinlock.h>
  31 #include <rte_hexdump.h>
  32
  33 #include <fsl_usd.h>
  34 #include <fsl_qman.h>
  35 #include <dpaa_of.h>
  36
  37 /* RTA header files */
  38 #include <desc/common.h>
  39 #include <desc/algo.h>
  40 #include <desc/ipsec.h>
  41 #include <desc/pdcp.h>
  42 #include <desc/sdap.h>
  43
  44 #include <rte_dpaa_bus.h>
  45 #include <dpaa_sec.h>
  46 #include <dpaa_sec_event.h>
  47 #include <dpaa_sec_log.h>
  48 #include <dpaax_iova_table.h>
  49
  50 #define DRIVER_DUMP_MODE "drv_dump_mode"
  51
  52 /* DPAA_SEC_DP_DUMP levels */
  53 enum dpaa_sec_dump_levels {
  54         DPAA_SEC_DP_NO_DUMP,
  55         DPAA_SEC_DP_ERR_DUMP,
  56         DPAA_SEC_DP_FULL_DUMP
  57 };
  58
  59 uint8_t dpaa_sec_dp_dump = DPAA_SEC_DP_ERR_DUMP;
  60
  61 uint8_t dpaa_cryptodev_driver_id;
  62
  63 static inline void
  64 dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx)
  65 {
  66         if (!ctx->fd_status) {
  67                 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
  68         } else {
  69                 DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status);
  70                 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
  71         }
  72 }
  73
  74 static inline struct dpaa_sec_op_ctx *
  75 dpaa_sec_alloc_ctx(dpaa_sec_session *ses, int sg_count)
  76 {
  77         struct dpaa_sec_op_ctx *ctx;
  78         int i, retval;
  79
  80         retval = rte_mempool_get(
  81                         ses->qp[rte_lcore_id() % MAX_DPAA_CORES]->ctx_pool,
  82                         (void **)(&ctx));
  83         if (!ctx || retval) {
  84                 DPAA_SEC_DP_WARN("Alloc sec descriptor failed!");
  85                 return NULL;
  86         }
  87         /*
  88          * Clear SG memory. There are 16 SG entries of 16 Bytes each.
  89          * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
  90          * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for
  91          * each packet, memset is costlier than dcbz_64().
  92          */
  93         for (i = 0; i < sg_count && i < MAX_JOB_SG_ENTRIES; i += 4)
  94                 dcbz_64(&ctx->job.sg[i]);
  95
  96         ctx->ctx_pool = ses->qp[rte_lcore_id() % MAX_DPAA_CORES]->ctx_pool;
  97         ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx);
  98
  99         return ctx;
 100 }
 101
 102 static void
 103 ern_sec_fq_handler(struct qman_portal *qm __rte_unused,
 104                    struct qman_fq *fq,
 105                    const struct qm_mr_entry *msg)
 106 {
 107         DPAA_SEC_DP_ERR("sec fq %d error, RC = %x, seqnum = %x\n",
 108                         fq->fqid, msg->ern.rc, msg->ern.seqnum);
 109 }
 110
 111 /* initialize the queue with dest chan as caam chan so that
 112  * all the packets in this queue could be dispatched into caam
 113  */
 114 static int
 115 dpaa_sec_init_rx(struct qman_fq *fq_in, rte_iova_t hwdesc,
 116                  uint32_t fqid_out)
 117 {
 118         struct qm_mcc_initfq fq_opts;
 119         uint32_t flags;
 120         int ret = -1;
 121
 122         /* Clear FQ options */
 123         memset(&fq_opts, 0x00, sizeof(struct qm_mcc_initfq));
 124
 125         flags = QMAN_INITFQ_FLAG_SCHED;
 126         fq_opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_CONTEXTA |
 127                           QM_INITFQ_WE_CONTEXTB;
 128
 129         qm_fqd_context_a_set64(&fq_opts.fqd, hwdesc);
 130         fq_opts.fqd.context_b = fqid_out;
 131         fq_opts.fqd.dest.channel = dpaa_get_qm_channel_caam();
 132         fq_opts.fqd.dest.wq = 0;
 133
 134         fq_in->cb.ern  = ern_sec_fq_handler;
 135
 136         DPAA_SEC_DEBUG("in-%x out-%x", fq_in->fqid, fqid_out);
 137
 138         ret = qman_init_fq(fq_in, flags, &fq_opts);
 139         if (unlikely(ret != 0))
 140                 DPAA_SEC_ERR("qman_init_fq failed %d", ret);
 141
 142         return ret;
 143 }
 144
 145 /* something is put into in_fq and caam put the crypto result into out_fq */
 146 static enum qman_cb_dqrr_result
 147 dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused,
 148                   struct qman_fq *fq __always_unused,
 149                   const struct qm_dqrr_entry *dqrr)
 150 {
 151         const struct qm_fd *fd;
 152         struct dpaa_sec_job *job;
 153         struct dpaa_sec_op_ctx *ctx;
 154
 155         if (!(dqrr->stat & QM_DQRR_STAT_FD_VALID))
 156                 return qman_cb_dqrr_consume;
 157
 158         fd = &dqrr->fd;
 159         /* sg is embedded in an op ctx,
 160          * sg[0] is for output
 161          * sg[1] for input
 162          */
 163         job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd));
 164
 165         ctx = container_of(job, struct dpaa_sec_op_ctx, job);
 166         ctx->fd_status = fd->status;
 167         if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 168                 struct qm_sg_entry *sg_out;
 169                 uint32_t len;
 170                 struct rte_mbuf *mbuf = (ctx->op->sym->m_dst == NULL) ?
 171                                 ctx->op->sym->m_src : ctx->op->sym->m_dst;
 172
 173                 sg_out = &job->sg[0];
 174                 hw_sg_to_cpu(sg_out);
 175                 len = sg_out->length;
 176                 mbuf->pkt_len = len;
 177                 while (mbuf->next != NULL) {
 178                         len -= mbuf->data_len;
 179                         mbuf = mbuf->next;
 180                 }
 181                 mbuf->data_len = len;
 182         }
 183         dpaa_sec_op_ending(ctx);
 184
 185         return qman_cb_dqrr_consume;
 186 }
 187
 188 /* caam result is put into this queue */
 189 static int
 190 dpaa_sec_init_tx(struct qman_fq *fq)
 191 {
 192         int ret;
 193         struct qm_mcc_initfq opts;
 194         uint32_t flags;
 195
 196         flags = QMAN_FQ_FLAG_NO_ENQUEUE | QMAN_FQ_FLAG_LOCKED |
 197                 QMAN_FQ_FLAG_DYNAMIC_FQID;
 198
 199         ret = qman_create_fq(0, flags, fq);
 200         if (unlikely(ret)) {
 201                 DPAA_SEC_ERR("qman_create_fq failed");
 202                 return ret;
 203         }
 204
 205         memset(&opts, 0, sizeof(opts));
 206         opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
 207                        QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB;
 208
 209         /* opts.fqd.dest.channel = dpaa_sec_pool_chan; */
 210
 211         fq->cb.dqrr = dqrr_out_fq_cb_rx;
 212         fq->cb.ern  = ern_sec_fq_handler;
 213
 214         ret = qman_init_fq(fq, 0, &opts);
 215         if (unlikely(ret)) {
 216                 DPAA_SEC_ERR("unable to init caam source fq!");
 217                 return ret;
 218         }
 219
 220         return ret;
 221 }
 222
 223 static inline int is_aead(dpaa_sec_session *ses)
 224 {
 225         return ((ses->cipher_alg == 0) &&
 226                 (ses->auth_alg == 0) &&
 227                 (ses->aead_alg != 0));
 228 }
 229
 230 static inline int is_encode(dpaa_sec_session *ses)
 231 {
 232         return ses->dir == DIR_ENC;
 233 }
 234
 235 static inline int is_decode(dpaa_sec_session *ses)
 236 {
 237         return ses->dir == DIR_DEC;
 238 }
 239
 240 #ifdef RTE_LIB_SECURITY
 241 static int
 242 dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses)
 243 {
 244         struct alginfo authdata = {0}, cipherdata = {0};
 245         struct sec_cdb *cdb = &ses->cdb;
 246         struct alginfo *p_authdata = NULL;
 247         int32_t shared_desc_len = 0;
 248 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 249         int swap = false;
 250 #else
 251         int swap = true;
 252 #endif
 253
 254         cipherdata.key = (size_t)ses->cipher_key.data;
 255         cipherdata.keylen = ses->cipher_key.length;
 256         cipherdata.key_enc_flags = 0;
 257         cipherdata.key_type = RTA_DATA_IMM;
 258         cipherdata.algtype = ses->cipher_key.alg;
 259         cipherdata.algmode = ses->cipher_key.algmode;
 260
 261         if (ses->auth_alg) {
 262                 authdata.key = (size_t)ses->auth_key.data;
 263                 authdata.keylen = ses->auth_key.length;
 264                 authdata.key_enc_flags = 0;
 265                 authdata.key_type = RTA_DATA_IMM;
 266                 authdata.algtype = ses->auth_key.alg;
 267                 authdata.algmode = ses->auth_key.algmode;
 268
 269                 p_authdata = &authdata;
 270         }
 271
 272         if (ses->pdcp.sdap_enabled) {
 273                 int nb_keys_to_inline =
 274                                 rta_inline_pdcp_sdap_query(authdata.algtype,
 275                                         cipherdata.algtype,
 276                                         ses->pdcp.sn_size,
 277                                         ses->pdcp.hfn_ovd);
 278                 if (nb_keys_to_inline >= 1) {
 279                         cipherdata.key = (size_t)rte_dpaa_mem_vtop((void *)
 280                                                 (size_t)cipherdata.key);
 281                         cipherdata.key_type = RTA_DATA_PTR;
 282                 }
 283                 if (nb_keys_to_inline >= 2) {
 284                         authdata.key = (size_t)rte_dpaa_mem_vtop((void *)
 285                                                 (size_t)authdata.key);
 286                         authdata.key_type = RTA_DATA_PTR;
 287                 }
 288         } else {
 289                 if (rta_inline_pdcp_query(authdata.algtype,
 290                                         cipherdata.algtype,
 291                                         ses->pdcp.sn_size,
 292                                         ses->pdcp.hfn_ovd)) {
 293                         cipherdata.key = (size_t)rte_dpaa_mem_vtop((void *)
 294                                                 (size_t)cipherdata.key);
 295                         cipherdata.key_type = RTA_DATA_PTR;
 296                 }
 297         }
 298
 299         if (ses->pdcp.domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
 300                 if (ses->dir == DIR_ENC)
 301                         shared_desc_len = cnstr_shdsc_pdcp_c_plane_encap(
 302                                         cdb->sh_desc, 1, swap,
 303                                         ses->pdcp.hfn,
 304                                         ses->pdcp.sn_size,
 305                                         ses->pdcp.bearer,
 306                                         ses->pdcp.pkt_dir,
 307                                         ses->pdcp.hfn_threshold,
 308                                         &cipherdata, &authdata);
 309                 else if (ses->dir == DIR_DEC)
 310                         shared_desc_len = cnstr_shdsc_pdcp_c_plane_decap(
 311                                         cdb->sh_desc, 1, swap,
 312                                         ses->pdcp.hfn,
 313                                         ses->pdcp.sn_size,
 314                                         ses->pdcp.bearer,
 315                                         ses->pdcp.pkt_dir,
 316                                         ses->pdcp.hfn_threshold,
 317                                         &cipherdata, &authdata);
 318         } else if (ses->pdcp.domain == RTE_SECURITY_PDCP_MODE_SHORT_MAC) {
 319                 shared_desc_len = cnstr_shdsc_pdcp_short_mac(cdb->sh_desc,
 320                                                      1, swap, &authdata);
 321         } else {
 322                 if (ses->dir == DIR_ENC) {
 323                         if (ses->pdcp.sdap_enabled)
 324                                 shared_desc_len =
 325                                         cnstr_shdsc_pdcp_sdap_u_plane_encap(
 326                                                 cdb->sh_desc, 1, swap,
 327                                                 ses->pdcp.sn_size,
 328                                                 ses->pdcp.hfn,
 329                                                 ses->pdcp.bearer,
 330                                                 ses->pdcp.pkt_dir,
 331                                                 ses->pdcp.hfn_threshold,
 332                                                 &cipherdata, p_authdata);
 333                         else
 334                                 shared_desc_len =
 335                                         cnstr_shdsc_pdcp_u_plane_encap(
 336                                                 cdb->sh_desc, 1, swap,
 337                                                 ses->pdcp.sn_size,
 338                                                 ses->pdcp.hfn,
 339                                                 ses->pdcp.bearer,
 340                                                 ses->pdcp.pkt_dir,
 341                                                 ses->pdcp.hfn_threshold,
 342                                                 &cipherdata, p_authdata);
 343                 } else if (ses->dir == DIR_DEC) {
 344                         if (ses->pdcp.sdap_enabled)
 345                                 shared_desc_len =
 346                                         cnstr_shdsc_pdcp_sdap_u_plane_decap(
 347                                                 cdb->sh_desc, 1, swap,
 348                                                 ses->pdcp.sn_size,
 349                                                 ses->pdcp.hfn,
 350                                                 ses->pdcp.bearer,
 351                                                 ses->pdcp.pkt_dir,
 352                                                 ses->pdcp.hfn_threshold,
 353                                                 &cipherdata, p_authdata);
 354                         else
 355                                 shared_desc_len =
 356                                         cnstr_shdsc_pdcp_u_plane_decap(
 357                                                 cdb->sh_desc, 1, swap,
 358                                                 ses->pdcp.sn_size,
 359                                                 ses->pdcp.hfn,
 360                                                 ses->pdcp.bearer,
 361                                                 ses->pdcp.pkt_dir,
 362                                                 ses->pdcp.hfn_threshold,
 363                                                 &cipherdata, p_authdata);
 364                 }
 365         }
 366         return shared_desc_len;
 367 }
 368
 369 /* prepare ipsec proto command block of the session */
 370 static int
 371 dpaa_sec_prep_ipsec_cdb(dpaa_sec_session *ses)
 372 {
 373         struct alginfo cipherdata = {0}, authdata = {0};
 374         struct sec_cdb *cdb = &ses->cdb;
 375         int32_t shared_desc_len = 0;
 376         int err;
 377 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 378         int swap = false;
 379 #else
 380         int swap = true;
 381 #endif
 382
 383         cipherdata.key = (size_t)ses->cipher_key.data;
 384         cipherdata.keylen = ses->cipher_key.length;
 385         cipherdata.key_enc_flags = 0;
 386         cipherdata.key_type = RTA_DATA_IMM;
 387         cipherdata.algtype = ses->cipher_key.alg;
 388         cipherdata.algmode = ses->cipher_key.algmode;
 389
 390         if (ses->auth_key.length) {
 391                 authdata.key = (size_t)ses->auth_key.data;
 392                 authdata.keylen = ses->auth_key.length;
 393                 authdata.key_enc_flags = 0;
 394                 authdata.key_type = RTA_DATA_IMM;
 395                 authdata.algtype = ses->auth_key.alg;
 396                 authdata.algmode = ses->auth_key.algmode;
 397         }
 398
 399         cdb->sh_desc[0] = cipherdata.keylen;
 400         cdb->sh_desc[1] = authdata.keylen;
 401         err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
 402                                DESC_JOB_IO_LEN,
 403                                (unsigned int *)cdb->sh_desc,
 404                                &cdb->sh_desc[2], 2);
 405
 406         if (err < 0) {
 407                 DPAA_SEC_ERR("Crypto: Incorrect key lengths");
 408                 return err;
 409         }
 410         if (cdb->sh_desc[2] & 1)
 411                 cipherdata.key_type = RTA_DATA_IMM;
 412         else {
 413                 cipherdata.key = (size_t)rte_dpaa_mem_vtop(
 414                                         (void *)(size_t)cipherdata.key);
 415                 cipherdata.key_type = RTA_DATA_PTR;
 416         }
 417         if (cdb->sh_desc[2] & (1<<1))
 418                 authdata.key_type = RTA_DATA_IMM;
 419         else {
 420                 authdata.key = (size_t)rte_dpaa_mem_vtop(
 421                                         (void *)(size_t)authdata.key);
 422                 authdata.key_type = RTA_DATA_PTR;
 423         }
 424
 425         cdb->sh_desc[0] = 0;
 426         cdb->sh_desc[1] = 0;
 427         cdb->sh_desc[2] = 0;
 428         if (ses->dir == DIR_ENC) {
 429                 shared_desc_len = cnstr_shdsc_ipsec_new_encap(
 430                                 cdb->sh_desc,
 431                                 true, swap, SHR_SERIAL,
 432                                 &ses->encap_pdb,
 433                                 (uint8_t *)&ses->ip4_hdr,
 434                                 &cipherdata, &authdata);
 435         } else if (ses->dir == DIR_DEC) {
 436                 shared_desc_len = cnstr_shdsc_ipsec_new_decap(
 437                                 cdb->sh_desc,
 438                                 true, swap, SHR_SERIAL,
 439                                 &ses->decap_pdb,
 440                                 &cipherdata, &authdata);
 441         }
 442         return shared_desc_len;
 443 }
 444 #endif
 445 /* prepare command block of the session */
 446 static int
 447 dpaa_sec_prep_cdb(dpaa_sec_session *ses)
 448 {
 449         struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0};
 450         int32_t shared_desc_len = 0;
 451         struct sec_cdb *cdb = &ses->cdb;
 452         int err;
 453 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 454         int swap = false;
 455 #else
 456         int swap = true;
 457 #endif
 458
 459         memset(cdb, 0, sizeof(struct sec_cdb));
 460
 461         switch (ses->ctxt) {
 462 #ifdef RTE_LIB_SECURITY
 463         case DPAA_SEC_IPSEC:
 464                 shared_desc_len = dpaa_sec_prep_ipsec_cdb(ses);
 465                 break;
 466         case DPAA_SEC_PDCP:
 467                 shared_desc_len = dpaa_sec_prep_pdcp_cdb(ses);
 468                 break;
 469 #endif
 470         case DPAA_SEC_CIPHER:
 471                 alginfo_c.key = (size_t)ses->cipher_key.data;
 472                 alginfo_c.keylen = ses->cipher_key.length;
 473                 alginfo_c.key_enc_flags = 0;
 474                 alginfo_c.key_type = RTA_DATA_IMM;
 475                 alginfo_c.algtype = ses->cipher_key.alg;
 476                 alginfo_c.algmode = ses->cipher_key.algmode;
 477
 478                 switch (ses->cipher_alg) {
 479                 case RTE_CRYPTO_CIPHER_AES_CBC:
 480                 case RTE_CRYPTO_CIPHER_3DES_CBC:
 481                 case RTE_CRYPTO_CIPHER_DES_CBC:
 482                 case RTE_CRYPTO_CIPHER_AES_CTR:
 483                 case RTE_CRYPTO_CIPHER_3DES_CTR:
 484                         shared_desc_len = cnstr_shdsc_blkcipher(
 485                                         cdb->sh_desc, true,
 486                                         swap, SHR_NEVER, &alginfo_c,
 487                                         ses->iv.length,
 488                                         ses->dir);
 489                         break;
 490                 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
 491                         shared_desc_len = cnstr_shdsc_snow_f8(
 492                                         cdb->sh_desc, true, swap,
 493                                         &alginfo_c,
 494                                         ses->dir);
 495                         break;
 496                 case RTE_CRYPTO_CIPHER_ZUC_EEA3:
 497                         shared_desc_len = cnstr_shdsc_zuce(
 498                                         cdb->sh_desc, true, swap,
 499                                         &alginfo_c,
 500                                         ses->dir);
 501                         break;
 502                 default:
 503                         DPAA_SEC_ERR("unsupported cipher alg %d",
 504                                      ses->cipher_alg);
 505                         return -ENOTSUP;
 506                 }
 507                 break;
 508         case DPAA_SEC_AUTH:
 509                 alginfo_a.key = (size_t)ses->auth_key.data;
 510                 alginfo_a.keylen = ses->auth_key.length;
 511                 alginfo_a.key_enc_flags = 0;
 512                 alginfo_a.key_type = RTA_DATA_IMM;
 513                 alginfo_a.algtype = ses->auth_key.alg;
 514                 alginfo_a.algmode = ses->auth_key.algmode;
 515                 switch (ses->auth_alg) {
 516                 case RTE_CRYPTO_AUTH_MD5:
 517                 case RTE_CRYPTO_AUTH_SHA1:
 518                 case RTE_CRYPTO_AUTH_SHA224:
 519                 case RTE_CRYPTO_AUTH_SHA256:
 520                 case RTE_CRYPTO_AUTH_SHA384:
 521                 case RTE_CRYPTO_AUTH_SHA512:
 522                         shared_desc_len = cnstr_shdsc_hash(
 523                                                 cdb->sh_desc, true,
 524                                                 swap, SHR_NEVER, &alginfo_a,
 525                                                 !ses->dir,
 526                                                 ses->digest_length);
 527                         break;
 528                 case RTE_CRYPTO_AUTH_MD5_HMAC:
 529                 case RTE_CRYPTO_AUTH_SHA1_HMAC:
 530                 case RTE_CRYPTO_AUTH_SHA224_HMAC:
 531                 case RTE_CRYPTO_AUTH_SHA256_HMAC:
 532                 case RTE_CRYPTO_AUTH_SHA384_HMAC:
 533                 case RTE_CRYPTO_AUTH_SHA512_HMAC:
 534                         shared_desc_len = cnstr_shdsc_hmac(
 535                                                 cdb->sh_desc, true,
 536                                                 swap, SHR_NEVER, &alginfo_a,
 537                                                 !ses->dir,
 538                                                 ses->digest_length);
 539                         break;
 540                 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
 541                         shared_desc_len = cnstr_shdsc_snow_f9(
 542                                                 cdb->sh_desc, true, swap,
 543                                                 &alginfo_a,
 544                                                 !ses->dir,
 545                                                 ses->digest_length);
 546                         break;
 547                 case RTE_CRYPTO_AUTH_ZUC_EIA3:
 548                         shared_desc_len = cnstr_shdsc_zuca(
 549                                                 cdb->sh_desc, true, swap,
 550                                                 &alginfo_a,
 551                                                 !ses->dir,
 552                                                 ses->digest_length);
 553                         break;
 554                 case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
 555                 case RTE_CRYPTO_AUTH_AES_CMAC:
 556                         shared_desc_len = cnstr_shdsc_aes_mac(
 557                                                 cdb->sh_desc,
 558                                                 true, swap, SHR_NEVER,
 559                                                 &alginfo_a,
 560                                                 !ses->dir,
 561                                                 ses->digest_length);
 562                         break;
 563                 default:
 564                         DPAA_SEC_ERR("unsupported auth alg %u", ses->auth_alg);
 565                 }
 566                 break;
 567         case DPAA_SEC_AEAD:
 568                 if (alginfo.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) {
 569                         DPAA_SEC_ERR("not supported aead alg");
 570                         return -ENOTSUP;
 571                 }
 572                 alginfo.key = (size_t)ses->aead_key.data;
 573                 alginfo.keylen = ses->aead_key.length;
 574                 alginfo.key_enc_flags = 0;
 575                 alginfo.key_type = RTA_DATA_IMM;
 576                 alginfo.algtype = ses->aead_key.alg;
 577                 alginfo.algmode = ses->aead_key.algmode;
 578
 579                 if (ses->dir == DIR_ENC)
 580                         shared_desc_len = cnstr_shdsc_gcm_encap(
 581                                         cdb->sh_desc, true, swap, SHR_NEVER,
 582                                         &alginfo,
 583                                         ses->iv.length,
 584                                         ses->digest_length);
 585                 else
 586                         shared_desc_len = cnstr_shdsc_gcm_decap(
 587                                         cdb->sh_desc, true, swap, SHR_NEVER,
 588                                         &alginfo,
 589                                         ses->iv.length,
 590                                         ses->digest_length);
 591                 break;
 592         case DPAA_SEC_CIPHER_HASH:
 593                 alginfo_c.key = (size_t)ses->cipher_key.data;
 594                 alginfo_c.keylen = ses->cipher_key.length;
 595                 alginfo_c.key_enc_flags = 0;
 596                 alginfo_c.key_type = RTA_DATA_IMM;
 597                 alginfo_c.algtype = ses->cipher_key.alg;
 598                 alginfo_c.algmode = ses->cipher_key.algmode;
 599
 600                 alginfo_a.key = (size_t)ses->auth_key.data;
 601                 alginfo_a.keylen = ses->auth_key.length;
 602                 alginfo_a.key_enc_flags = 0;
 603                 alginfo_a.key_type = RTA_DATA_IMM;
 604                 alginfo_a.algtype = ses->auth_key.alg;
 605                 alginfo_a.algmode = ses->auth_key.algmode;
 606
 607                 cdb->sh_desc[0] = alginfo_c.keylen;
 608                 cdb->sh_desc[1] = alginfo_a.keylen;
 609                 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
 610                                        DESC_JOB_IO_LEN,
 611                                        (unsigned int *)cdb->sh_desc,
 612                                        &cdb->sh_desc[2], 2);
 613
 614                 if (err < 0) {
 615                         DPAA_SEC_ERR("Crypto: Incorrect key lengths");
 616                         return err;
 617                 }
 618                 if (cdb->sh_desc[2] & 1)
 619                         alginfo_c.key_type = RTA_DATA_IMM;
 620                 else {
 621                         alginfo_c.key = (size_t)rte_dpaa_mem_vtop(
 622                                                 (void *)(size_t)alginfo_c.key);
 623                         alginfo_c.key_type = RTA_DATA_PTR;
 624                 }
 625                 if (cdb->sh_desc[2] & (1<<1))
 626                         alginfo_a.key_type = RTA_DATA_IMM;
 627                 else {
 628                         alginfo_a.key = (size_t)rte_dpaa_mem_vtop(
 629                                                 (void *)(size_t)alginfo_a.key);
 630                         alginfo_a.key_type = RTA_DATA_PTR;
 631                 }
 632                 cdb->sh_desc[0] = 0;
 633                 cdb->sh_desc[1] = 0;
 634                 cdb->sh_desc[2] = 0;
 635                 /* Auth_only_len is set as 0 here and it will be
 636                  * overwritten in fd for each packet.
 637                  */
 638                 shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc,
 639                                 true, swap, SHR_SERIAL, &alginfo_c, &alginfo_a,
 640                                 ses->iv.length,
 641                                 ses->digest_length, ses->dir);
 642                 break;
 643         case DPAA_SEC_HASH_CIPHER:
 644         default:
 645                 DPAA_SEC_ERR("error: Unsupported session");
 646                 return -ENOTSUP;
 647         }
 648
 649         if (shared_desc_len < 0) {
 650                 DPAA_SEC_ERR("error in preparing command block");
 651                 return shared_desc_len;
 652         }
 653
 654         cdb->sh_hdr.hi.field.idlen = shared_desc_len;
 655         cdb->sh_hdr.hi.word = rte_cpu_to_be_32(cdb->sh_hdr.hi.word);
 656         cdb->sh_hdr.lo.word = rte_cpu_to_be_32(cdb->sh_hdr.lo.word);
 657
 658         return 0;
 659 }
 660
 661 static void
 662 dpaa_sec_dump(struct dpaa_sec_op_ctx *ctx, struct dpaa_sec_qp *qp)
 663 {
 664         struct dpaa_sec_job *job = &ctx->job;
 665         struct rte_crypto_op *op = ctx->op;
 666         dpaa_sec_session *sess = NULL;
 667         struct sec_cdb c_cdb, *cdb;
 668         uint8_t bufsize;
 669         struct rte_crypto_sym_op *sym_op;
 670         struct qm_sg_entry sg[2];
 671
 672         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)
 673                 sess = (dpaa_sec_session *)
 674                         get_sym_session_private_data(
 675                                         op->sym->session,
 676                                         dpaa_cryptodev_driver_id);
 677 #ifdef RTE_LIBRTE_SECURITY
 678         else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
 679                 sess = (dpaa_sec_session *)
 680                         get_sec_session_private_data(
 681                                         op->sym->sec_session);
 682 #endif
 683         if (sess == NULL) {
 684                 printf("session is NULL\n");
 685                 goto mbuf_dump;
 686         }
 687
 688         cdb = &sess->cdb;
 689         rte_memcpy(&c_cdb, cdb, sizeof(struct sec_cdb));
 690 #ifdef RTE_LIBRTE_SECURITY
 691         printf("\nsession protocol type = %d\n", sess->proto_alg);
 692 #endif
 693         printf("\n****************************************\n"
 694                 "session params:\n\tContext type:\t%d\n\tDirection:\t%s\n"
 695                 "\tCipher alg:\t%d\n\tAuth alg:\t%d\n\tAead alg:\t%d\n"
 696                 "\tCipher key len:\t%"PRIu64"\n\tCipher alg:\t%d\n"
 697                 "\tCipher algmode:\t%d\n", sess->ctxt,
 698                 (sess->dir == DIR_ENC) ? "DIR_ENC" : "DIR_DEC",
 699                 sess->cipher_alg, sess->auth_alg, sess->aead_alg,
 700                 (uint64_t)sess->cipher_key.length, sess->cipher_key.alg,
 701                 sess->cipher_key.algmode);
 702                 rte_hexdump(stdout, "cipher key", sess->cipher_key.data,
 703                                 sess->cipher_key.length);
 704                 rte_hexdump(stdout, "auth key", sess->auth_key.data,
 705                                 sess->auth_key.length);
 706         printf("\tAuth key len:\t%"PRIu64"\n\tAuth alg:\t%d\n"
 707                 "\tAuth algmode:\t%d\n\tIV len:\t\t%d\n\tIV offset:\t%d\n"
 708                 "\tdigest length:\t%d\n\tauth only len:\t\t%d\n"
 709                 "\taead cipher text:\t%d\n",
 710                 (uint64_t)sess->auth_key.length, sess->auth_key.alg,
 711                 sess->auth_key.algmode,
 712                 sess->iv.length, sess->iv.offset,
 713                 sess->digest_length, sess->auth_only_len,
 714                 sess->auth_cipher_text);
 715 #ifdef RTE_LIBRTE_SECURITY
 716         printf("PDCP session params:\n"
 717                 "\tDomain:\t\t%d\n\tBearer:\t\t%d\n\tpkt_dir:\t%d\n\thfn_ovd:"
 718                 "\t%d\n\tsn_size:\t%d\n\tsdap_enabled:\t%d\n\thfn_ovd_offset:"
 719                 "\t%d\n\thfn:\t\t%d\n"
 720                 "\thfn_threshold:\t0x%x\n", sess->pdcp.domain,
 721                 sess->pdcp.bearer, sess->pdcp.pkt_dir, sess->pdcp.hfn_ovd,
 722                 sess->pdcp.sn_size, sess->pdcp.sdap_enabled,
 723                 sess->pdcp.hfn_ovd_offset, sess->pdcp.hfn,
 724                 sess->pdcp.hfn_threshold);
 725 #endif
 726         c_cdb.sh_hdr.hi.word = rte_be_to_cpu_32(c_cdb.sh_hdr.hi.word);
 727         c_cdb.sh_hdr.lo.word = rte_be_to_cpu_32(c_cdb.sh_hdr.lo.word);
 728         bufsize = c_cdb.sh_hdr.hi.field.idlen;
 729
 730         printf("cdb = %p\n\n", cdb);
 731         printf("Descriptor size = %d\n", bufsize);
 732         int m;
 733         for (m = 0; m < bufsize; m++)
 734                 printf("0x%x\n", rte_be_to_cpu_32(c_cdb.sh_desc[m]));
 735
 736         printf("\n");
 737 mbuf_dump:
 738         sym_op = op->sym;
 739         if (sym_op->m_src) {
 740                 printf("Source mbuf:\n");
 741                 rte_pktmbuf_dump(stdout, sym_op->m_src,
 742                                  sym_op->m_src->data_len);
 743         }
 744         if (sym_op->m_dst) {
 745                 printf("Destination mbuf:\n");
 746                 rte_pktmbuf_dump(stdout, sym_op->m_dst,
 747                                  sym_op->m_dst->data_len);
 748         }
 749
 750         printf("Session address = %p\ncipher offset: %d, length: %d\n"
 751                 "auth offset: %d, length:  %d\n aead offset: %d, length: %d\n",
 752                 sym_op->session, sym_op->cipher.data.offset,
 753                 sym_op->cipher.data.length,
 754                 sym_op->auth.data.offset, sym_op->auth.data.length,
 755                 sym_op->aead.data.offset, sym_op->aead.data.length);
 756         printf("\n");
 757
 758         printf("******************************************************\n");
 759         printf("ctx info:\n");
 760         printf("job->sg[0] output info:\n");
 761         memcpy(&sg[0], &job->sg[0], sizeof(sg[0]));
 762         printf("\taddr = %"PRIx64",\n\tlen = %d,\n\tfinal = %d,\n\textention = %d"
 763                 "\n\tbpid = %d\n\toffset = %d\n",
 764                 (uint64_t)sg[0].addr, sg[0].length, sg[0].final,
 765                 sg[0].extension, sg[0].bpid, sg[0].offset);
 766         printf("\njob->sg[1] input info:\n");
 767         memcpy(&sg[1], &job->sg[1], sizeof(sg[1]));
 768         hw_sg_to_cpu(&sg[1]);
 769         printf("\taddr = %"PRIx64",\n\tlen = %d,\n\tfinal = %d,\n\textention = %d"
 770                 "\n\tbpid = %d\n\toffset = %d\n",
 771                 (uint64_t)sg[1].addr, sg[1].length, sg[1].final,
 772                 sg[1].extension, sg[1].bpid, sg[1].offset);
 773
 774         printf("\nctx pool addr = %p\n", ctx->ctx_pool);
 775         if (ctx->ctx_pool)
 776                 printf("ctx pool available counts = %d\n",
 777                         rte_mempool_avail_count(ctx->ctx_pool));
 778
 779         printf("\nop pool addr = %p\n", op->mempool);
 780         if (op->mempool)
 781                 printf("op pool available counts = %d\n",
 782                         rte_mempool_avail_count(op->mempool));
 783
 784         printf("********************************************************\n");
 785         printf("Queue data:\n");
 786         printf("\tFQID = 0x%x\n\tstate = %d\n\tnb_desc = %d\n"
 787                 "\tctx_pool = %p\n\trx_pkts = %d\n\ttx_pkts"
 788                "= %d\n\trx_errs = %d\n\ttx_errs = %d\n\n",
 789                 qp->outq.fqid, qp->outq.state, qp->outq.nb_desc,
 790                 qp->ctx_pool, qp->rx_pkts, qp->tx_pkts,
 791                 qp->rx_errs, qp->tx_errs);
 792 }
 793
 794 /* qp is lockless, should be accessed by only one thread */
 795 static int
 796 dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops)
 797 {
 798         struct qman_fq *fq;
 799         unsigned int pkts = 0;
 800         int num_rx_bufs, ret;
 801         struct qm_dqrr_entry *dq;
 802         uint32_t vdqcr_flags = 0;
 803
 804         fq = &qp->outq;
 805         /*
 806          * Until request for four buffers, we provide exact number of buffers.
 807          * Otherwise we do not set the QM_VDQCR_EXACT flag.
 808          * Not setting QM_VDQCR_EXACT flag can provide two more buffers than
 809          * requested, so we request two less in this case.
 810          */
 811         if (nb_ops < 4) {
 812                 vdqcr_flags = QM_VDQCR_EXACT;
 813                 num_rx_bufs = nb_ops;
 814         } else {
 815                 num_rx_bufs = nb_ops > DPAA_MAX_DEQUEUE_NUM_FRAMES ?
 816                         (DPAA_MAX_DEQUEUE_NUM_FRAMES - 2) : (nb_ops - 2);
 817         }
 818         ret = qman_set_vdq(fq, num_rx_bufs, vdqcr_flags);
 819         if (ret)
 820                 return 0;
 821
 822         do {
 823                 const struct qm_fd *fd;
 824                 struct dpaa_sec_job *job;
 825                 struct dpaa_sec_op_ctx *ctx;
 826                 struct rte_crypto_op *op;
 827
 828                 dq = qman_dequeue(fq);
 829                 if (!dq)
 830                         continue;
 831
 832                 fd = &dq->fd;
 833                 /* sg is embedded in an op ctx,
 834                  * sg[0] is for output
 835                  * sg[1] for input
 836                  */
 837                 job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd));
 838
 839                 ctx = container_of(job, struct dpaa_sec_op_ctx, job);
 840                 ctx->fd_status = fd->status;
 841                 op = ctx->op;
 842                 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
 843                         struct qm_sg_entry *sg_out;
 844                         uint32_t len;
 845                         struct rte_mbuf *mbuf = (op->sym->m_dst == NULL) ?
 846                                                 op->sym->m_src : op->sym->m_dst;
 847
 848                         sg_out = &job->sg[0];
 849                         hw_sg_to_cpu(sg_out);
 850                         len = sg_out->length;
 851                         mbuf->pkt_len = len;
 852                         while (mbuf->next != NULL) {
 853                                 len -= mbuf->data_len;
 854                                 mbuf = mbuf->next;
 855                         }
 856                         mbuf->data_len = len;
 857                 }
 858                 if (!ctx->fd_status) {
 859                         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 860                 } else {
 861                         if (dpaa_sec_dp_dump > DPAA_SEC_DP_NO_DUMP) {
 862                                 DPAA_SEC_DP_WARN("SEC return err:0x%x\n",
 863                                                   ctx->fd_status);
 864                                 if (dpaa_sec_dp_dump > DPAA_SEC_DP_ERR_DUMP)
 865                                         dpaa_sec_dump(ctx, qp);
 866                         }
 867                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 868                 }
 869                 ops[pkts++] = op;
 870
 871                 /* report op status to sym->op and then free the ctx memory */
 872                 rte_mempool_put(ctx->ctx_pool, (void *)ctx);
 873
 874                 qman_dqrr_consume(fq, dq);
 875         } while (fq->flags & QMAN_FQ_STATE_VDQCR);
 876
 877         return pkts;
 878 }
 879
 880 static inline struct dpaa_sec_job *
 881 build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
 882 {
 883         struct rte_crypto_sym_op *sym = op->sym;
 884         struct rte_mbuf *mbuf = sym->m_src;
 885         struct dpaa_sec_job *cf;
 886         struct dpaa_sec_op_ctx *ctx;
 887         struct qm_sg_entry *sg, *out_sg, *in_sg;
 888         phys_addr_t start_addr;
 889         uint8_t *old_digest, extra_segs;
 890         int data_len, data_offset;
 891
 892         data_len = sym->auth.data.length;
 893         data_offset = sym->auth.data.offset;
 894
 895         if (ses->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
 896             ses->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
 897                 if ((data_len & 7) || (data_offset & 7)) {
 898                         DPAA_SEC_ERR("AUTH: len/offset must be full bytes");
 899                         return NULL;
 900                 }
 901
 902                 data_len = data_len >> 3;
 903                 data_offset = data_offset >> 3;
 904         }
 905
 906         if (is_decode(ses))
 907                 extra_segs = 3;
 908         else
 909                 extra_segs = 2;
 910
 911         if (mbuf->nb_segs > MAX_SG_ENTRIES) {
 912                 DPAA_SEC_DP_ERR("Auth: Max sec segs supported is %d",
 913                                 MAX_SG_ENTRIES);
 914                 return NULL;
 915         }
 916         ctx = dpaa_sec_alloc_ctx(ses, mbuf->nb_segs + extra_segs);
 917         if (!ctx)
 918                 return NULL;
 919
 920         cf = &ctx->job;
 921         ctx->op = op;
 922         old_digest = ctx->digest;
 923
 924         /* output */
 925         out_sg = &cf->sg[0];
 926         qm_sg_entry_set64(out_sg, sym->auth.digest.phys_addr);
 927         out_sg->length = ses->digest_length;
 928         cpu_to_hw_sg(out_sg);
 929
 930         /* input */
 931         in_sg = &cf->sg[1];
 932         /* need to extend the input to a compound frame */
 933         in_sg->extension = 1;
 934         in_sg->final = 1;
 935         in_sg->length = data_len;
 936         qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
 937
 938         /* 1st seg */
 939         sg = in_sg + 1;
 940
 941         if (ses->iv.length) {
 942                 uint8_t *iv_ptr;
 943
 944                 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 945                                                    ses->iv.offset);
 946
 947                 if (ses->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
 948                         iv_ptr = conv_to_snow_f9_iv(iv_ptr);
 949                         sg->length = 12;
 950                 } else if (ses->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
 951                         iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
 952                         sg->length = 8;
 953                 } else {
 954                         sg->length = ses->iv.length;
 955                 }
 956                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(iv_ptr));
 957                 in_sg->length += sg->length;
 958                 cpu_to_hw_sg(sg);
 959                 sg++;
 960         }
 961
 962         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
 963         sg->offset = data_offset;
 964
 965         if (data_len <= (mbuf->data_len - data_offset)) {
 966                 sg->length = data_len;
 967         } else {
 968                 sg->length = mbuf->data_len - data_offset;
 969
 970                 /* remaining i/p segs */
 971                 while ((data_len = data_len - sg->length) &&
 972                        (mbuf = mbuf->next)) {
 973                         cpu_to_hw_sg(sg);
 974                         sg++;
 975                         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
 976                         if (data_len > mbuf->data_len)
 977                                 sg->length = mbuf->data_len;
 978                         else
 979                                 sg->length = data_len;
 980                 }
 981         }
 982
 983         if (is_decode(ses)) {
 984                 /* Digest verification case */
 985                 cpu_to_hw_sg(sg);
 986                 sg++;
 987                 rte_memcpy(old_digest, sym->auth.digest.data,
 988                                 ses->digest_length);
 989                 start_addr = rte_dpaa_mem_vtop(old_digest);
 990                 qm_sg_entry_set64(sg, start_addr);
 991                 sg->length = ses->digest_length;
 992                 in_sg->length += ses->digest_length;
 993         }
 994         sg->final = 1;
 995         cpu_to_hw_sg(sg);
 996         cpu_to_hw_sg(in_sg);
 997
 998         return cf;
 999 }
1000
1001 /**
1002  * packet looks like:
1003  *              |<----data_len------->|
1004  *    |ip_header|ah_header|icv|payload|
1005  *              ^
1006  *              |
1007  *         mbuf->pkt.data
1008  */
1009 static inline struct dpaa_sec_job *
1010 build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
1011 {
1012         struct rte_crypto_sym_op *sym = op->sym;
1013         struct rte_mbuf *mbuf = sym->m_src;
1014         struct dpaa_sec_job *cf;
1015         struct dpaa_sec_op_ctx *ctx;
1016         struct qm_sg_entry *sg, *in_sg;
1017         rte_iova_t start_addr;
1018         uint8_t *old_digest;
1019         int data_len, data_offset;
1020
1021         data_len = sym->auth.data.length;
1022         data_offset = sym->auth.data.offset;
1023
1024         if (ses->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
1025             ses->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
1026                 if ((data_len & 7) || (data_offset & 7)) {
1027                         DPAA_SEC_ERR("AUTH: len/offset must be full bytes");
1028                         return NULL;
1029                 }
1030
1031                 data_len = data_len >> 3;
1032                 data_offset = data_offset >> 3;
1033         }
1034
1035         ctx = dpaa_sec_alloc_ctx(ses, 4);
1036         if (!ctx)
1037                 return NULL;
1038
1039         cf = &ctx->job;
1040         ctx->op = op;
1041         old_digest = ctx->digest;
1042
1043         start_addr = rte_pktmbuf_iova(mbuf);
1044         /* output */
1045         sg = &cf->sg[0];
1046         qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
1047         sg->length = ses->digest_length;
1048         cpu_to_hw_sg(sg);
1049
1050         /* input */
1051         in_sg = &cf->sg[1];
1052         /* need to extend the input to a compound frame */
1053         in_sg->extension = 1;
1054         in_sg->final = 1;
1055         in_sg->length = data_len;
1056         qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
1057         sg = &cf->sg[2];
1058
1059         if (ses->iv.length) {
1060                 uint8_t *iv_ptr;
1061
1062                 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1063                                                    ses->iv.offset);
1064
1065                 if (ses->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
1066                         iv_ptr = conv_to_snow_f9_iv(iv_ptr);
1067                         sg->length = 12;
1068                 } else if (ses->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
1069                         iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
1070                         sg->length = 8;
1071                 } else {
1072                         sg->length = ses->iv.length;
1073                 }
1074                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(iv_ptr));
1075                 in_sg->length += sg->length;
1076                 cpu_to_hw_sg(sg);
1077                 sg++;
1078         }
1079
1080         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1081         sg->offset = data_offset;
1082         sg->length = data_len;
1083
1084         if (is_decode(ses)) {
1085                 /* Digest verification case */
1086                 cpu_to_hw_sg(sg);
1087                 /* hash result or digest, save digest first */
1088                 rte_memcpy(old_digest, sym->auth.digest.data,
1089                                 ses->digest_length);
1090                 /* let's check digest by hw */
1091                 start_addr = rte_dpaa_mem_vtop(old_digest);
1092                 sg++;
1093                 qm_sg_entry_set64(sg, start_addr);
1094                 sg->length = ses->digest_length;
1095                 in_sg->length += ses->digest_length;
1096         }
1097         sg->final = 1;
1098         cpu_to_hw_sg(sg);
1099         cpu_to_hw_sg(in_sg);
1100
1101         return cf;
1102 }
1103
1104 static inline struct dpaa_sec_job *
1105 build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
1106 {
1107         struct rte_crypto_sym_op *sym = op->sym;
1108         struct dpaa_sec_job *cf;
1109         struct dpaa_sec_op_ctx *ctx;
1110         struct qm_sg_entry *sg, *out_sg, *in_sg;
1111         struct rte_mbuf *mbuf;
1112         uint8_t req_segs;
1113         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1114                         ses->iv.offset);
1115         int data_len, data_offset;
1116
1117         data_len = sym->cipher.data.length;
1118         data_offset = sym->cipher.data.offset;
1119
1120         if (ses->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
1121                 ses->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
1122                 if ((data_len & 7) || (data_offset & 7)) {
1123                         DPAA_SEC_ERR("CIPHER: len/offset must be full bytes");
1124                         return NULL;
1125                 }
1126
1127                 data_len = data_len >> 3;
1128                 data_offset = data_offset >> 3;
1129         }
1130
1131         if (sym->m_dst) {
1132                 mbuf = sym->m_dst;
1133                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3;
1134         } else {
1135                 mbuf = sym->m_src;
1136                 req_segs = mbuf->nb_segs * 2 + 3;
1137         }
1138         if (mbuf->nb_segs > MAX_SG_ENTRIES) {
1139                 DPAA_SEC_DP_ERR("Cipher: Max sec segs supported is %d",
1140                                 MAX_SG_ENTRIES);
1141                 return NULL;
1142         }
1143
1144         ctx = dpaa_sec_alloc_ctx(ses, req_segs);
1145         if (!ctx)
1146                 return NULL;
1147
1148         cf = &ctx->job;
1149         ctx->op = op;
1150
1151         /* output */
1152         out_sg = &cf->sg[0];
1153         out_sg->extension = 1;
1154         out_sg->length = data_len;
1155         qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
1156         cpu_to_hw_sg(out_sg);
1157
1158         /* 1st seg */
1159         sg = &cf->sg[2];
1160         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1161         sg->length = mbuf->data_len - data_offset;
1162         sg->offset = data_offset;
1163
1164         /* Successive segs */
1165         mbuf = mbuf->next;
1166         while (mbuf) {
1167                 cpu_to_hw_sg(sg);
1168                 sg++;
1169                 qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1170                 sg->length = mbuf->data_len;
1171                 mbuf = mbuf->next;
1172         }
1173         sg->final = 1;
1174         cpu_to_hw_sg(sg);
1175
1176         /* input */
1177         mbuf = sym->m_src;
1178         in_sg = &cf->sg[1];
1179         in_sg->extension = 1;
1180         in_sg->final = 1;
1181         in_sg->length = data_len + ses->iv.length;
1182
1183         sg++;
1184         qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
1185         cpu_to_hw_sg(in_sg);
1186
1187         /* IV */
1188         qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
1189         sg->length = ses->iv.length;
1190         cpu_to_hw_sg(sg);
1191
1192         /* 1st seg */
1193         sg++;
1194         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1195         sg->length = mbuf->data_len - data_offset;
1196         sg->offset = data_offset;
1197
1198         /* Successive segs */
1199         mbuf = mbuf->next;
1200         while (mbuf) {
1201                 cpu_to_hw_sg(sg);
1202                 sg++;
1203                 qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1204                 sg->length = mbuf->data_len;
1205                 mbuf = mbuf->next;
1206         }
1207         sg->final = 1;
1208         cpu_to_hw_sg(sg);
1209
1210         return cf;
1211 }
1212
1213 static inline struct dpaa_sec_job *
1214 build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses)
1215 {
1216         struct rte_crypto_sym_op *sym = op->sym;
1217         struct dpaa_sec_job *cf;
1218         struct dpaa_sec_op_ctx *ctx;
1219         struct qm_sg_entry *sg;
1220         rte_iova_t src_start_addr, dst_start_addr;
1221         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1222                         ses->iv.offset);
1223         int data_len, data_offset;
1224
1225         data_len = sym->cipher.data.length;
1226         data_offset = sym->cipher.data.offset;
1227
1228         if (ses->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
1229                 ses->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
1230                 if ((data_len & 7) || (data_offset & 7)) {
1231                         DPAA_SEC_ERR("CIPHER: len/offset must be full bytes");
1232                         return NULL;
1233                 }
1234
1235                 data_len = data_len >> 3;
1236                 data_offset = data_offset >> 3;
1237         }
1238
1239         ctx = dpaa_sec_alloc_ctx(ses, 4);
1240         if (!ctx)
1241                 return NULL;
1242
1243         cf = &ctx->job;
1244         ctx->op = op;
1245
1246         src_start_addr = rte_pktmbuf_iova(sym->m_src);
1247
1248         if (sym->m_dst)
1249                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
1250         else
1251                 dst_start_addr = src_start_addr;
1252
1253         /* output */
1254         sg = &cf->sg[0];
1255         qm_sg_entry_set64(sg, dst_start_addr + data_offset);
1256         sg->length = data_len + ses->iv.length;
1257         cpu_to_hw_sg(sg);
1258
1259         /* input */
1260         sg = &cf->sg[1];
1261
1262         /* need to extend the input to a compound frame */
1263         sg->extension = 1;
1264         sg->final = 1;
1265         sg->length = data_len + ses->iv.length;
1266         qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(&cf->sg[2]));
1267         cpu_to_hw_sg(sg);
1268
1269         sg = &cf->sg[2];
1270         qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
1271         sg->length = ses->iv.length;
1272         cpu_to_hw_sg(sg);
1273
1274         sg++;
1275         qm_sg_entry_set64(sg, src_start_addr + data_offset);
1276         sg->length = data_len;
1277         sg->final = 1;
1278         cpu_to_hw_sg(sg);
1279
1280         return cf;
1281 }
1282
1283 static inline struct dpaa_sec_job *
1284 build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
1285 {
1286         struct rte_crypto_sym_op *sym = op->sym;
1287         struct dpaa_sec_job *cf;
1288         struct dpaa_sec_op_ctx *ctx;
1289         struct qm_sg_entry *sg, *out_sg, *in_sg;
1290         struct rte_mbuf *mbuf;
1291         uint8_t req_segs;
1292         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1293                         ses->iv.offset);
1294
1295         if (sym->m_dst) {
1296                 mbuf = sym->m_dst;
1297                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4;
1298         } else {
1299                 mbuf = sym->m_src;
1300                 req_segs = mbuf->nb_segs * 2 + 4;
1301         }
1302
1303         if (ses->auth_only_len)
1304                 req_segs++;
1305
1306         if (mbuf->nb_segs > MAX_SG_ENTRIES) {
1307                 DPAA_SEC_DP_ERR("AEAD: Max sec segs supported is %d",
1308                                 MAX_SG_ENTRIES);
1309                 return NULL;
1310         }
1311
1312         ctx = dpaa_sec_alloc_ctx(ses, req_segs);
1313         if (!ctx)
1314                 return NULL;
1315
1316         cf = &ctx->job;
1317         ctx->op = op;
1318
1319         rte_prefetch0(cf->sg);
1320
1321         /* output */
1322         out_sg = &cf->sg[0];
1323         out_sg->extension = 1;
1324         if (is_encode(ses))
1325                 out_sg->length = sym->aead.data.length + ses->digest_length;
1326         else
1327                 out_sg->length = sym->aead.data.length;
1328
1329         /* output sg entries */
1330         sg = &cf->sg[2];
1331         qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(sg));
1332         cpu_to_hw_sg(out_sg);
1333
1334         /* 1st seg */
1335         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1336         sg->length = mbuf->data_len - sym->aead.data.offset;
1337         sg->offset = sym->aead.data.offset;
1338
1339         /* Successive segs */
1340         mbuf = mbuf->next;
1341         while (mbuf) {
1342                 cpu_to_hw_sg(sg);
1343                 sg++;
1344                 qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1345                 sg->length = mbuf->data_len;
1346                 mbuf = mbuf->next;
1347         }
1348         sg->length -= ses->digest_length;
1349
1350         if (is_encode(ses)) {
1351                 cpu_to_hw_sg(sg);
1352                 /* set auth output */
1353                 sg++;
1354                 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr);
1355                 sg->length = ses->digest_length;
1356         }
1357         sg->final = 1;
1358         cpu_to_hw_sg(sg);
1359
1360         /* input */
1361         mbuf = sym->m_src;
1362         in_sg = &cf->sg[1];
1363         in_sg->extension = 1;
1364         in_sg->final = 1;
1365         if (is_encode(ses))
1366                 in_sg->length = ses->iv.length + sym->aead.data.length
1367                                                         + ses->auth_only_len;
1368         else
1369                 in_sg->length = ses->iv.length + sym->aead.data.length
1370                                 + ses->auth_only_len + ses->digest_length;
1371
1372         /* input sg entries */
1373         sg++;
1374         qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
1375         cpu_to_hw_sg(in_sg);
1376
1377         /* 1st seg IV */
1378         qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
1379         sg->length = ses->iv.length;
1380         cpu_to_hw_sg(sg);
1381
1382         /* 2nd seg auth only */
1383         if (ses->auth_only_len) {
1384                 sg++;
1385                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(sym->aead.aad.data));
1386                 sg->length = ses->auth_only_len;
1387                 cpu_to_hw_sg(sg);
1388         }
1389
1390         /* 3rd seg */
1391         sg++;
1392         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1393         sg->length = mbuf->data_len - sym->aead.data.offset;
1394         sg->offset = sym->aead.data.offset;
1395
1396         /* Successive segs */
1397         mbuf = mbuf->next;
1398         while (mbuf) {
1399                 cpu_to_hw_sg(sg);
1400                 sg++;
1401                 qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1402                 sg->length = mbuf->data_len;
1403                 mbuf = mbuf->next;
1404         }
1405
1406         if (is_decode(ses)) {
1407                 cpu_to_hw_sg(sg);
1408                 sg++;
1409                 memcpy(ctx->digest, sym->aead.digest.data,
1410                         ses->digest_length);
1411                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest));
1412                 sg->length = ses->digest_length;
1413         }
1414         sg->final = 1;
1415         cpu_to_hw_sg(sg);
1416
1417         return cf;
1418 }
1419
1420 static inline struct dpaa_sec_job *
1421 build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses)
1422 {
1423         struct rte_crypto_sym_op *sym = op->sym;
1424         struct dpaa_sec_job *cf;
1425         struct dpaa_sec_op_ctx *ctx;
1426         struct qm_sg_entry *sg;
1427         uint32_t length = 0;
1428         rte_iova_t src_start_addr, dst_start_addr;
1429         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1430                         ses->iv.offset);
1431
1432         src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
1433
1434         if (sym->m_dst)
1435                 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
1436         else
1437                 dst_start_addr = src_start_addr;
1438
1439         ctx = dpaa_sec_alloc_ctx(ses, 7);
1440         if (!ctx)
1441                 return NULL;
1442
1443         cf = &ctx->job;
1444         ctx->op = op;
1445
1446         /* input */
1447         rte_prefetch0(cf->sg);
1448         sg = &cf->sg[2];
1449         qm_sg_entry_set64(&cf->sg[1], rte_dpaa_mem_vtop(sg));
1450         if (is_encode(ses)) {
1451                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
1452                 sg->length = ses->iv.length;
1453                 length += sg->length;
1454                 cpu_to_hw_sg(sg);
1455
1456                 sg++;
1457                 if (ses->auth_only_len) {
1458                         qm_sg_entry_set64(sg,
1459                                           rte_dpaa_mem_vtop(sym->aead.aad.data));
1460                         sg->length = ses->auth_only_len;
1461                         length += sg->length;
1462                         cpu_to_hw_sg(sg);
1463                         sg++;
1464                 }
1465                 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
1466                 sg->length = sym->aead.data.length;
1467                 length += sg->length;
1468                 sg->final = 1;
1469                 cpu_to_hw_sg(sg);
1470         } else {
1471                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
1472                 sg->length = ses->iv.length;
1473                 length += sg->length;
1474                 cpu_to_hw_sg(sg);
1475
1476                 sg++;
1477                 if (ses->auth_only_len) {
1478                         qm_sg_entry_set64(sg,
1479                                           rte_dpaa_mem_vtop(sym->aead.aad.data));
1480                         sg->length = ses->auth_only_len;
1481                         length += sg->length;
1482                         cpu_to_hw_sg(sg);
1483                         sg++;
1484                 }
1485                 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset);
1486                 sg->length = sym->aead.data.length;
1487                 length += sg->length;
1488                 cpu_to_hw_sg(sg);
1489
1490                 memcpy(ctx->digest, sym->aead.digest.data,
1491                        ses->digest_length);
1492                 sg++;
1493
1494                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest));
1495                 sg->length = ses->digest_length;
1496                 length += sg->length;
1497                 sg->final = 1;
1498                 cpu_to_hw_sg(sg);
1499         }
1500         /* input compound frame */
1501         cf->sg[1].length = length;
1502         cf->sg[1].extension = 1;
1503         cf->sg[1].final = 1;
1504         cpu_to_hw_sg(&cf->sg[1]);
1505
1506         /* output */
1507         sg++;
1508         qm_sg_entry_set64(&cf->sg[0], rte_dpaa_mem_vtop(sg));
1509         qm_sg_entry_set64(sg,
1510                 dst_start_addr + sym->aead.data.offset);
1511         sg->length = sym->aead.data.length;
1512         length = sg->length;
1513         if (is_encode(ses)) {
1514                 cpu_to_hw_sg(sg);
1515                 /* set auth output */
1516                 sg++;
1517                 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr);
1518                 sg->length = ses->digest_length;
1519                 length += sg->length;
1520         }
1521         sg->final = 1;
1522         cpu_to_hw_sg(sg);
1523
1524         /* output compound frame */
1525         cf->sg[0].length = length;
1526         cf->sg[0].extension = 1;
1527         cpu_to_hw_sg(&cf->sg[0]);
1528
1529         return cf;
1530 }
1531
1532 static inline struct dpaa_sec_job *
1533 build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
1534 {
1535         struct rte_crypto_sym_op *sym = op->sym;
1536         struct dpaa_sec_job *cf;
1537         struct dpaa_sec_op_ctx *ctx;
1538         struct qm_sg_entry *sg, *out_sg, *in_sg;
1539         struct rte_mbuf *mbuf;
1540         uint8_t req_segs;
1541         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1542                         ses->iv.offset);
1543
1544         if (sym->m_dst) {
1545                 mbuf = sym->m_dst;
1546                 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4;
1547         } else {
1548                 mbuf = sym->m_src;
1549                 req_segs = mbuf->nb_segs * 2 + 4;
1550         }
1551
1552         if (mbuf->nb_segs > MAX_SG_ENTRIES) {
1553                 DPAA_SEC_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
1554                                 MAX_SG_ENTRIES);
1555                 return NULL;
1556         }
1557
1558         ctx = dpaa_sec_alloc_ctx(ses, req_segs);
1559         if (!ctx)
1560                 return NULL;
1561
1562         cf = &ctx->job;
1563         ctx->op = op;
1564
1565         rte_prefetch0(cf->sg);
1566
1567         /* output */
1568         out_sg = &cf->sg[0];
1569         out_sg->extension = 1;
1570         if (is_encode(ses))
1571                 out_sg->length = sym->auth.data.length + ses->digest_length;
1572         else
1573                 out_sg->length = sym->auth.data.length;
1574
1575         /* output sg entries */
1576         sg = &cf->sg[2];
1577         qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(sg));
1578         cpu_to_hw_sg(out_sg);
1579
1580         /* 1st seg */
1581         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1582         sg->length = mbuf->data_len - sym->auth.data.offset;
1583         sg->offset = sym->auth.data.offset;
1584
1585         /* Successive segs */
1586         mbuf = mbuf->next;
1587         while (mbuf) {
1588                 cpu_to_hw_sg(sg);
1589                 sg++;
1590                 qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1591                 sg->length = mbuf->data_len;
1592                 mbuf = mbuf->next;
1593         }
1594         sg->length -= ses->digest_length;
1595
1596         if (is_encode(ses)) {
1597                 cpu_to_hw_sg(sg);
1598                 /* set auth output */
1599                 sg++;
1600                 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
1601                 sg->length = ses->digest_length;
1602         }
1603         sg->final = 1;
1604         cpu_to_hw_sg(sg);
1605
1606         /* input */
1607         mbuf = sym->m_src;
1608         in_sg = &cf->sg[1];
1609         in_sg->extension = 1;
1610         in_sg->final = 1;
1611         if (is_encode(ses))
1612                 in_sg->length = ses->iv.length + sym->auth.data.length;
1613         else
1614                 in_sg->length = ses->iv.length + sym->auth.data.length
1615                                                 + ses->digest_length;
1616
1617         /* input sg entries */
1618         sg++;
1619         qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
1620         cpu_to_hw_sg(in_sg);
1621
1622         /* 1st seg IV */
1623         qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
1624         sg->length = ses->iv.length;
1625         cpu_to_hw_sg(sg);
1626
1627         /* 2nd seg */
1628         sg++;
1629         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1630         sg->length = mbuf->data_len - sym->auth.data.offset;
1631         sg->offset = sym->auth.data.offset;
1632
1633         /* Successive segs */
1634         mbuf = mbuf->next;
1635         while (mbuf) {
1636                 cpu_to_hw_sg(sg);
1637                 sg++;
1638                 qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1639                 sg->length = mbuf->data_len;
1640                 mbuf = mbuf->next;
1641         }
1642
1643         sg->length -= ses->digest_length;
1644         if (is_decode(ses)) {
1645                 cpu_to_hw_sg(sg);
1646                 sg++;
1647                 memcpy(ctx->digest, sym->auth.digest.data,
1648                         ses->digest_length);
1649                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest));
1650                 sg->length = ses->digest_length;
1651         }
1652         sg->final = 1;
1653         cpu_to_hw_sg(sg);
1654
1655         return cf;
1656 }
1657
1658 static inline struct dpaa_sec_job *
1659 build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses)
1660 {
1661         struct rte_crypto_sym_op *sym = op->sym;
1662         struct dpaa_sec_job *cf;
1663         struct dpaa_sec_op_ctx *ctx;
1664         struct qm_sg_entry *sg;
1665         rte_iova_t src_start_addr, dst_start_addr;
1666         uint32_t length = 0;
1667         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1668                         ses->iv.offset);
1669
1670         src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off;
1671         if (sym->m_dst)
1672                 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off;
1673         else
1674                 dst_start_addr = src_start_addr;
1675
1676         ctx = dpaa_sec_alloc_ctx(ses, 7);
1677         if (!ctx)
1678                 return NULL;
1679
1680         cf = &ctx->job;
1681         ctx->op = op;
1682
1683         /* input */
1684         rte_prefetch0(cf->sg);
1685         sg = &cf->sg[2];
1686         qm_sg_entry_set64(&cf->sg[1], rte_dpaa_mem_vtop(sg));
1687         if (is_encode(ses)) {
1688                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
1689                 sg->length = ses->iv.length;
1690                 length += sg->length;
1691                 cpu_to_hw_sg(sg);
1692
1693                 sg++;
1694                 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
1695                 sg->length = sym->auth.data.length;
1696                 length += sg->length;
1697                 sg->final = 1;
1698                 cpu_to_hw_sg(sg);
1699         } else {
1700                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
1701                 sg->length = ses->iv.length;
1702                 length += sg->length;
1703                 cpu_to_hw_sg(sg);
1704
1705                 sg++;
1706
1707                 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset);
1708                 sg->length = sym->auth.data.length;
1709                 length += sg->length;
1710                 cpu_to_hw_sg(sg);
1711
1712                 memcpy(ctx->digest, sym->auth.digest.data,
1713                        ses->digest_length);
1714                 sg++;
1715
1716                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest));
1717                 sg->length = ses->digest_length;
1718                 length += sg->length;
1719                 sg->final = 1;
1720                 cpu_to_hw_sg(sg);
1721         }
1722         /* input compound frame */
1723         cf->sg[1].length = length;
1724         cf->sg[1].extension = 1;
1725         cf->sg[1].final = 1;
1726         cpu_to_hw_sg(&cf->sg[1]);
1727
1728         /* output */
1729         sg++;
1730         qm_sg_entry_set64(&cf->sg[0], rte_dpaa_mem_vtop(sg));
1731         qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset);
1732         sg->length = sym->cipher.data.length;
1733         length = sg->length;
1734         if (is_encode(ses)) {
1735                 cpu_to_hw_sg(sg);
1736                 /* set auth output */
1737                 sg++;
1738                 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr);
1739                 sg->length = ses->digest_length;
1740                 length += sg->length;
1741         }
1742         sg->final = 1;
1743         cpu_to_hw_sg(sg);
1744
1745         /* output compound frame */
1746         cf->sg[0].length = length;
1747         cf->sg[0].extension = 1;
1748         cpu_to_hw_sg(&cf->sg[0]);
1749
1750         return cf;
1751 }
1752
1753 #ifdef RTE_LIB_SECURITY
1754 static inline struct dpaa_sec_job *
1755 build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses)
1756 {
1757         struct rte_crypto_sym_op *sym = op->sym;
1758         struct dpaa_sec_job *cf;
1759         struct dpaa_sec_op_ctx *ctx;
1760         struct qm_sg_entry *sg;
1761         phys_addr_t src_start_addr, dst_start_addr;
1762
1763         ctx = dpaa_sec_alloc_ctx(ses, 2);
1764         if (!ctx)
1765                 return NULL;
1766         cf = &ctx->job;
1767         ctx->op = op;
1768
1769         src_start_addr = rte_pktmbuf_iova(sym->m_src);
1770
1771         if (sym->m_dst)
1772                 dst_start_addr = rte_pktmbuf_iova(sym->m_dst);
1773         else
1774                 dst_start_addr = src_start_addr;
1775
1776         /* input */
1777         sg = &cf->sg[1];
1778         qm_sg_entry_set64(sg, src_start_addr);
1779         sg->length = sym->m_src->pkt_len;
1780         sg->final = 1;
1781         cpu_to_hw_sg(sg);
1782
1783         sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
1784         /* output */
1785         sg = &cf->sg[0];
1786         qm_sg_entry_set64(sg, dst_start_addr);
1787         sg->length = sym->m_src->buf_len - sym->m_src->data_off;
1788         cpu_to_hw_sg(sg);
1789
1790         return cf;
1791 }
1792
1793 static inline struct dpaa_sec_job *
1794 build_proto_sg(struct rte_crypto_op *op, dpaa_sec_session *ses)
1795 {
1796         struct rte_crypto_sym_op *sym = op->sym;
1797         struct dpaa_sec_job *cf;
1798         struct dpaa_sec_op_ctx *ctx;
1799         struct qm_sg_entry *sg, *out_sg, *in_sg;
1800         struct rte_mbuf *mbuf;
1801         uint8_t req_segs;
1802         uint32_t in_len = 0, out_len = 0;
1803
1804         if (sym->m_dst)
1805                 mbuf = sym->m_dst;
1806         else
1807                 mbuf = sym->m_src;
1808
1809         req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 2;
1810         if (mbuf->nb_segs > MAX_SG_ENTRIES) {
1811                 DPAA_SEC_DP_ERR("Proto: Max sec segs supported is %d",
1812                                 MAX_SG_ENTRIES);
1813                 return NULL;
1814         }
1815
1816         ctx = dpaa_sec_alloc_ctx(ses, req_segs);
1817         if (!ctx)
1818                 return NULL;
1819         cf = &ctx->job;
1820         ctx->op = op;
1821         /* output */
1822         out_sg = &cf->sg[0];
1823         out_sg->extension = 1;
1824         qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
1825
1826         /* 1st seg */
1827         sg = &cf->sg[2];
1828         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1829         sg->offset = 0;
1830
1831         /* Successive segs */
1832         while (mbuf->next) {
1833                 sg->length = mbuf->data_len;
1834                 out_len += sg->length;
1835                 mbuf = mbuf->next;
1836                 cpu_to_hw_sg(sg);
1837                 sg++;
1838                 qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1839                 sg->offset = 0;
1840         }
1841         sg->length = mbuf->buf_len - mbuf->data_off;
1842         out_len += sg->length;
1843         sg->final = 1;
1844         cpu_to_hw_sg(sg);
1845
1846         out_sg->length = out_len;
1847         cpu_to_hw_sg(out_sg);
1848
1849         /* input */
1850         mbuf = sym->m_src;
1851         in_sg = &cf->sg[1];
1852         in_sg->extension = 1;
1853         in_sg->final = 1;
1854         in_len = mbuf->data_len;
1855
1856         sg++;
1857         qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
1858
1859         /* 1st seg */
1860         qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1861         sg->length = mbuf->data_len;
1862         sg->offset = 0;
1863
1864         /* Successive segs */
1865         mbuf = mbuf->next;
1866         while (mbuf) {
1867                 cpu_to_hw_sg(sg);
1868                 sg++;
1869                 qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf));
1870                 sg->length = mbuf->data_len;
1871                 sg->offset = 0;
1872                 in_len += sg->length;
1873                 mbuf = mbuf->next;
1874         }
1875         sg->final = 1;
1876         cpu_to_hw_sg(sg);
1877
1878         in_sg->length = in_len;
1879         cpu_to_hw_sg(in_sg);
1880
1881         sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK;
1882
1883         return cf;
1884 }
1885 #endif
1886
1887 static uint16_t
1888 dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1889                        uint16_t nb_ops)
1890 {
1891         /* Function to transmit the frames to given device and queuepair */
1892         uint32_t loop;
1893         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp;
1894         uint16_t num_tx = 0;
1895         struct qm_fd fds[DPAA_SEC_BURST], *fd;
1896         uint32_t frames_to_send;
1897         struct rte_crypto_op *op;
1898         struct dpaa_sec_job *cf;
1899         dpaa_sec_session *ses;
1900         uint16_t auth_hdr_len, auth_tail_len;
1901         uint32_t index, flags[DPAA_SEC_BURST] = {0};
1902         struct qman_fq *inq[DPAA_SEC_BURST];
1903
1904         if (unlikely(!DPAA_PER_LCORE_PORTAL)) {
1905                 if (rte_dpaa_portal_init((void *)0)) {
1906                         DPAA_SEC_ERR("Failure in affining portal");
1907                         return 0;
1908                 }
1909         }
1910
1911         while (nb_ops) {
1912                 frames_to_send = (nb_ops > DPAA_SEC_BURST) ?
1913                                 DPAA_SEC_BURST : nb_ops;
1914                 for (loop = 0; loop < frames_to_send; loop++) {
1915                         op = *(ops++);
1916                         if (*dpaa_seqn(op->sym->m_src) != 0) {
1917                                 index = *dpaa_seqn(op->sym->m_src) - 1;
1918                                 if (DPAA_PER_LCORE_DQRR_HELD & (1 << index)) {
1919                                         /* QM_EQCR_DCA_IDXMASK = 0x0f */
1920                                         flags[loop] = ((index & 0x0f) << 8);
1921                                         flags[loop] |= QMAN_ENQUEUE_FLAG_DCA;
1922                                         DPAA_PER_LCORE_DQRR_SIZE--;
1923                                         DPAA_PER_LCORE_DQRR_HELD &=
1924                                                                 ~(1 << index);
1925                                 }
1926                         }
1927
1928                         switch (op->sess_type) {
1929                         case RTE_CRYPTO_OP_WITH_SESSION:
1930                                 ses = (dpaa_sec_session *)
1931                                         get_sym_session_private_data(
1932                                                 op->sym->session,
1933                                                 dpaa_cryptodev_driver_id);
1934                                 break;
1935 #ifdef RTE_LIB_SECURITY
1936                         case RTE_CRYPTO_OP_SECURITY_SESSION:
1937                                 ses = (dpaa_sec_session *)
1938                                         get_sec_session_private_data(
1939                                                         op->sym->sec_session);
1940                                 break;
1941 #endif
1942                         default:
1943                                 DPAA_SEC_DP_ERR(
1944                                         "sessionless crypto op not supported");
1945                                 frames_to_send = loop;
1946                                 nb_ops = loop;
1947                                 goto send_pkts;
1948                         }
1949
1950                         if (!ses) {
1951                                 DPAA_SEC_DP_ERR("session not available");
1952                                 frames_to_send = loop;
1953                                 nb_ops = loop;
1954                                 goto send_pkts;
1955                         }
1956
1957                         if (unlikely(!ses->qp[rte_lcore_id() % MAX_DPAA_CORES])) {
1958                                 if (dpaa_sec_attach_sess_q(qp, ses)) {
1959                                         frames_to_send = loop;
1960                                         nb_ops = loop;
1961                                         goto send_pkts;
1962                                 }
1963                         } else if (unlikely(ses->qp[rte_lcore_id() %
1964                                                 MAX_DPAA_CORES] != qp)) {
1965                                 DPAA_SEC_DP_ERR("Old:sess->qp = %p"
1966                                         " New qp = %p\n",
1967                                         ses->qp[rte_lcore_id() %
1968                                         MAX_DPAA_CORES], qp);
1969                                 frames_to_send = loop;
1970                                 nb_ops = loop;
1971                                 goto send_pkts;
1972                         }
1973
1974                         auth_hdr_len = op->sym->auth.data.length -
1975                                                 op->sym->cipher.data.length;
1976                         auth_tail_len = 0;
1977
1978                         if (rte_pktmbuf_is_contiguous(op->sym->m_src) &&
1979                                   ((op->sym->m_dst == NULL) ||
1980                                    rte_pktmbuf_is_contiguous(op->sym->m_dst))) {
1981                                 switch (ses->ctxt) {
1982 #ifdef RTE_LIB_SECURITY
1983                                 case DPAA_SEC_PDCP:
1984                                 case DPAA_SEC_IPSEC:
1985                                         cf = build_proto(op, ses);
1986                                         break;
1987 #endif
1988                                 case DPAA_SEC_AUTH:
1989                                         cf = build_auth_only(op, ses);
1990                                         break;
1991                                 case DPAA_SEC_CIPHER:
1992                                         cf = build_cipher_only(op, ses);
1993                                         break;
1994                                 case DPAA_SEC_AEAD:
1995                                         cf = build_cipher_auth_gcm(op, ses);
1996                                         auth_hdr_len = ses->auth_only_len;
1997                                         break;
1998                                 case DPAA_SEC_CIPHER_HASH:
1999                                         auth_hdr_len =
2000                                                 op->sym->cipher.data.offset
2001                                                 - op->sym->auth.data.offset;
2002                                         auth_tail_len =
2003                                                 op->sym->auth.data.length
2004                                                 - op->sym->cipher.data.length
2005                                                 - auth_hdr_len;
2006                                         cf = build_cipher_auth(op, ses);
2007                                         break;
2008                                 default:
2009                                         DPAA_SEC_DP_ERR("not supported ops");
2010                                         frames_to_send = loop;
2011                                         nb_ops = loop;
2012                                         goto send_pkts;
2013                                 }
2014                         } else {
2015                                 switch (ses->ctxt) {
2016 #ifdef RTE_LIB_SECURITY
2017                                 case DPAA_SEC_PDCP:
2018                                 case DPAA_SEC_IPSEC:
2019                                         cf = build_proto_sg(op, ses);
2020                                         break;
2021 #endif
2022                                 case DPAA_SEC_AUTH:
2023                                         cf = build_auth_only_sg(op, ses);
2024                                         break;
2025                                 case DPAA_SEC_CIPHER:
2026                                         cf = build_cipher_only_sg(op, ses);
2027                                         break;
2028                                 case DPAA_SEC_AEAD:
2029                                         cf = build_cipher_auth_gcm_sg(op, ses);
2030                                         auth_hdr_len = ses->auth_only_len;
2031                                         break;
2032                                 case DPAA_SEC_CIPHER_HASH:
2033                                         auth_hdr_len =
2034                                                 op->sym->cipher.data.offset
2035                                                 - op->sym->auth.data.offset;
2036                                         auth_tail_len =
2037                                                 op->sym->auth.data.length
2038                                                 - op->sym->cipher.data.length
2039                                                 - auth_hdr_len;
2040                                         cf = build_cipher_auth_sg(op, ses);
2041                                         break;
2042                                 default:
2043                                         DPAA_SEC_DP_ERR("not supported ops");
2044                                         frames_to_send = loop;
2045                                         nb_ops = loop;
2046                                         goto send_pkts;
2047                                 }
2048                         }
2049                         if (unlikely(!cf)) {
2050                                 frames_to_send = loop;
2051                                 nb_ops = loop;
2052                                 goto send_pkts;
2053                         }
2054
2055                         fd = &fds[loop];
2056                         inq[loop] = ses->inq[rte_lcore_id() % MAX_DPAA_CORES];
2057                         fd->opaque_addr = 0;
2058                         fd->cmd = 0;
2059                         qm_fd_addr_set64(fd, rte_dpaa_mem_vtop(cf->sg));
2060                         fd->_format1 = qm_fd_compound;
2061                         fd->length29 = 2 * sizeof(struct qm_sg_entry);
2062
2063                         /* Auth_only_len is set as 0 in descriptor and it is
2064                          * overwritten here in the fd.cmd which will update
2065                          * the DPOVRD reg.
2066                          */
2067                         if (auth_hdr_len || auth_tail_len) {
2068                                 fd->cmd = 0x80000000;
2069                                 fd->cmd |=
2070                                         ((auth_tail_len << 16) | auth_hdr_len);
2071                         }
2072
2073 #ifdef RTE_LIB_SECURITY
2074                         /* In case of PDCP, per packet HFN is stored in
2075                          * mbuf priv after sym_op.
2076                          */
2077                         if ((ses->ctxt == DPAA_SEC_PDCP) && ses->pdcp.hfn_ovd) {
2078                                 fd->cmd = 0x80000000 |
2079                                         *((uint32_t *)((uint8_t *)op +
2080                                         ses->pdcp.hfn_ovd_offset));
2081                                 DPAA_SEC_DP_DEBUG("Per packet HFN: %x, ovd:%u\n",
2082                                         *((uint32_t *)((uint8_t *)op +
2083                                         ses->pdcp.hfn_ovd_offset)),
2084                                         ses->pdcp.hfn_ovd);
2085                         }
2086 #endif
2087                 }
2088 send_pkts:
2089                 loop = 0;
2090                 while (loop < frames_to_send) {
2091                         loop += qman_enqueue_multi_fq(&inq[loop], &fds[loop],
2092                                         &flags[loop], frames_to_send - loop);
2093                 }
2094                 nb_ops -= frames_to_send;
2095                 num_tx += frames_to_send;
2096         }
2097
2098         dpaa_qp->tx_pkts += num_tx;
2099         dpaa_qp->tx_errs += nb_ops - num_tx;
2100
2101         return num_tx;
2102 }
2103
2104 static uint16_t
2105 dpaa_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops,
2106                        uint16_t nb_ops)
2107 {
2108         uint16_t num_rx;
2109         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp;
2110
2111         if (unlikely(!DPAA_PER_LCORE_PORTAL)) {
2112                 if (rte_dpaa_portal_init((void *)0)) {
2113                         DPAA_SEC_ERR("Failure in affining portal");
2114                         return 0;
2115                 }
2116         }
2117
2118         num_rx = dpaa_sec_deq(dpaa_qp, ops, nb_ops);
2119
2120         dpaa_qp->rx_pkts += num_rx;
2121         dpaa_qp->rx_errs += nb_ops - num_rx;
2122
2123         DPAA_SEC_DP_DEBUG("SEC Received %d Packets\n", num_rx);
2124
2125         return num_rx;
2126 }
2127
2128 /** Release queue pair */
2129 static int
2130 dpaa_sec_queue_pair_release(struct rte_cryptodev *dev,
2131                             uint16_t qp_id)
2132 {
2133         struct dpaa_sec_dev_private *internals;
2134         struct dpaa_sec_qp *qp = NULL;
2135
2136         PMD_INIT_FUNC_TRACE();
2137
2138         DPAA_SEC_DEBUG("dev =%p, queue =%d", dev, qp_id);
2139
2140         internals = dev->data->dev_private;
2141         if (qp_id >= internals->max_nb_queue_pairs) {
2142                 DPAA_SEC_ERR("Max supported qpid %d",
2143                              internals->max_nb_queue_pairs);
2144                 return -EINVAL;
2145         }
2146
2147         qp = &internals->qps[qp_id];
2148         rte_mempool_free(qp->ctx_pool);
2149         qp->internals = NULL;
2150         dev->data->queue_pairs[qp_id] = NULL;
2151
2152         return 0;
2153 }
2154
2155 /** Setup a queue pair */
2156 static int
2157 dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
2158                 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
2159                 __rte_unused int socket_id)
2160 {
2161         struct dpaa_sec_dev_private *internals;
2162         struct dpaa_sec_qp *qp = NULL;
2163         char str[20];
2164
2165         DPAA_SEC_DEBUG("dev =%p, queue =%d, conf =%p", dev, qp_id, qp_conf);
2166
2167         internals = dev->data->dev_private;
2168         if (qp_id >= internals->max_nb_queue_pairs) {
2169                 DPAA_SEC_ERR("Max supported qpid %d",
2170                              internals->max_nb_queue_pairs);
2171                 return -EINVAL;
2172         }
2173
2174         qp = &internals->qps[qp_id];
2175         qp->internals = internals;
2176         snprintf(str, sizeof(str), "ctx_pool_d%d_qp%d",
2177                         dev->data->dev_id, qp_id);
2178         if (!qp->ctx_pool) {
2179                 qp->ctx_pool = rte_mempool_create((const char *)str,
2180                                                         CTX_POOL_NUM_BUFS,
2181                                                         CTX_POOL_BUF_SIZE,
2182                                                         CTX_POOL_CACHE_SIZE, 0,
2183                                                         NULL, NULL, NULL, NULL,
2184                                                         SOCKET_ID_ANY, 0);
2185                 if (!qp->ctx_pool) {
2186                         DPAA_SEC_ERR("%s create failed\n", str);
2187                         return -ENOMEM;
2188                 }
2189         } else
2190                 DPAA_SEC_INFO("mempool already created for dev_id : %d, qp: %d",
2191                                 dev->data->dev_id, qp_id);
2192         dev->data->queue_pairs[qp_id] = qp;
2193
2194         return 0;
2195 }
2196
2197 /** Returns the size of session structure */
2198 static unsigned int
2199 dpaa_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
2200 {
2201         PMD_INIT_FUNC_TRACE();
2202
2203         return sizeof(dpaa_sec_session);
2204 }
2205
2206 static int
2207 dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused,
2208                      struct rte_crypto_sym_xform *xform,
2209                      dpaa_sec_session *session)
2210 {
2211         session->ctxt = DPAA_SEC_CIPHER;
2212         session->cipher_alg = xform->cipher.algo;
2213         session->iv.length = xform->cipher.iv.length;
2214         session->iv.offset = xform->cipher.iv.offset;
2215         session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
2216                                                RTE_CACHE_LINE_SIZE);
2217         if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
2218                 DPAA_SEC_ERR("No Memory for cipher key");
2219                 return -ENOMEM;
2220         }
2221         session->cipher_key.length = xform->cipher.key.length;
2222
2223         memcpy(session->cipher_key.data, xform->cipher.key.data,
2224                xform->cipher.key.length);
2225         switch (xform->cipher.algo) {
2226         case RTE_CRYPTO_CIPHER_AES_CBC:
2227                 session->cipher_key.alg = OP_ALG_ALGSEL_AES;
2228                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2229                 break;
2230         case RTE_CRYPTO_CIPHER_DES_CBC:
2231                 session->cipher_key.alg = OP_ALG_ALGSEL_DES;
2232                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2233                 break;
2234         case RTE_CRYPTO_CIPHER_3DES_CBC:
2235                 session->cipher_key.alg = OP_ALG_ALGSEL_3DES;
2236                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2237                 break;
2238         case RTE_CRYPTO_CIPHER_AES_CTR:
2239                 session->cipher_key.alg = OP_ALG_ALGSEL_AES;
2240                 session->cipher_key.algmode = OP_ALG_AAI_CTR;
2241                 break;
2242         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2243                 session->cipher_key.alg = OP_ALG_ALGSEL_SNOW_F8;
2244                 break;
2245         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2246                 session->cipher_key.alg = OP_ALG_ALGSEL_ZUCE;
2247                 break;
2248         default:
2249                 DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u",
2250                               xform->cipher.algo);
2251                 return -ENOTSUP;
2252         }
2253         session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2254                         DIR_ENC : DIR_DEC;
2255
2256         return 0;
2257 }
2258
2259 static int
2260 dpaa_sec_auth_init(struct rte_cryptodev *dev __rte_unused,
2261                    struct rte_crypto_sym_xform *xform,
2262                    dpaa_sec_session *session)
2263 {
2264         session->ctxt = DPAA_SEC_AUTH;
2265         session->auth_alg = xform->auth.algo;
2266         session->auth_key.length = xform->auth.key.length;
2267         if (xform->auth.key.length) {
2268                 session->auth_key.data =
2269                                 rte_zmalloc(NULL, xform->auth.key.length,
2270                                              RTE_CACHE_LINE_SIZE);
2271                 if (session->auth_key.data == NULL) {
2272                         DPAA_SEC_ERR("No Memory for auth key");
2273                         return -ENOMEM;
2274                 }
2275                 memcpy(session->auth_key.data, xform->auth.key.data,
2276                                 xform->auth.key.length);
2277
2278         }
2279         session->digest_length = xform->auth.digest_length;
2280         if (session->cipher_alg == RTE_CRYPTO_CIPHER_NULL) {
2281                 session->iv.offset = xform->auth.iv.offset;
2282                 session->iv.length = xform->auth.iv.length;
2283         }
2284
2285         switch (xform->auth.algo) {
2286         case RTE_CRYPTO_AUTH_SHA1:
2287                 session->auth_key.alg = OP_ALG_ALGSEL_SHA1;
2288                 session->auth_key.algmode = OP_ALG_AAI_HASH;
2289                 break;
2290         case RTE_CRYPTO_AUTH_SHA1_HMAC:
2291                 session->auth_key.alg = OP_ALG_ALGSEL_SHA1;
2292                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2293                 break;
2294         case RTE_CRYPTO_AUTH_MD5:
2295                 session->auth_key.alg = OP_ALG_ALGSEL_MD5;
2296                 session->auth_key.algmode = OP_ALG_AAI_HASH;
2297                 break;
2298         case RTE_CRYPTO_AUTH_MD5_HMAC:
2299                 session->auth_key.alg = OP_ALG_ALGSEL_MD5;
2300                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2301                 break;
2302         case RTE_CRYPTO_AUTH_SHA224:
2303                 session->auth_key.alg = OP_ALG_ALGSEL_SHA224;
2304                 session->auth_key.algmode = OP_ALG_AAI_HASH;
2305                 break;
2306         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2307                 session->auth_key.alg = OP_ALG_ALGSEL_SHA224;
2308                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2309                 break;
2310         case RTE_CRYPTO_AUTH_SHA256:
2311                 session->auth_key.alg = OP_ALG_ALGSEL_SHA256;
2312                 session->auth_key.algmode = OP_ALG_AAI_HASH;
2313                 break;
2314         case RTE_CRYPTO_AUTH_SHA256_HMAC:
2315                 session->auth_key.alg = OP_ALG_ALGSEL_SHA256;
2316                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2317                 break;
2318         case RTE_CRYPTO_AUTH_SHA384:
2319                 session->auth_key.alg = OP_ALG_ALGSEL_SHA384;
2320                 session->auth_key.algmode = OP_ALG_AAI_HASH;
2321                 break;
2322         case RTE_CRYPTO_AUTH_SHA384_HMAC:
2323                 session->auth_key.alg = OP_ALG_ALGSEL_SHA384;
2324                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2325                 break;
2326         case RTE_CRYPTO_AUTH_SHA512:
2327                 session->auth_key.alg = OP_ALG_ALGSEL_SHA512;
2328                 session->auth_key.algmode = OP_ALG_AAI_HASH;
2329                 break;
2330         case RTE_CRYPTO_AUTH_SHA512_HMAC:
2331                 session->auth_key.alg = OP_ALG_ALGSEL_SHA512;
2332                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2333                 break;
2334         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2335                 session->auth_key.alg = OP_ALG_ALGSEL_SNOW_F9;
2336                 session->auth_key.algmode = OP_ALG_AAI_F9;
2337                 break;
2338         case RTE_CRYPTO_AUTH_ZUC_EIA3:
2339                 session->auth_key.alg = OP_ALG_ALGSEL_ZUCA;
2340                 session->auth_key.algmode = OP_ALG_AAI_F9;
2341                 break;
2342         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2343                 session->auth_key.alg = OP_ALG_ALGSEL_AES;
2344                 session->auth_key.algmode = OP_ALG_AAI_XCBC_MAC;
2345                 break;
2346         case RTE_CRYPTO_AUTH_AES_CMAC:
2347                 session->auth_key.alg = OP_ALG_ALGSEL_AES;
2348                 session->auth_key.algmode = OP_ALG_AAI_CMAC;
2349                 break;
2350         default:
2351                 DPAA_SEC_ERR("Crypto: Unsupported Auth specified %u",
2352                               xform->auth.algo);
2353                 return -ENOTSUP;
2354         }
2355
2356         session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
2357                         DIR_ENC : DIR_DEC;
2358
2359         return 0;
2360 }
2361
2362 static int
2363 dpaa_sec_chain_init(struct rte_cryptodev *dev __rte_unused,
2364                    struct rte_crypto_sym_xform *xform,
2365                    dpaa_sec_session *session)
2366 {
2367
2368         struct rte_crypto_cipher_xform *cipher_xform;
2369         struct rte_crypto_auth_xform *auth_xform;
2370
2371         session->ctxt = DPAA_SEC_CIPHER_HASH;
2372         if (session->auth_cipher_text) {
2373                 cipher_xform = &xform->cipher;
2374                 auth_xform = &xform->next->auth;
2375         } else {
2376                 cipher_xform = &xform->next->cipher;
2377                 auth_xform = &xform->auth;
2378         }
2379
2380         /* Set IV parameters */
2381         session->iv.offset = cipher_xform->iv.offset;
2382         session->iv.length = cipher_xform->iv.length;
2383
2384         session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length,
2385                                                RTE_CACHE_LINE_SIZE);
2386         if (session->cipher_key.data == NULL && cipher_xform->key.length > 0) {
2387                 DPAA_SEC_ERR("No Memory for cipher key");
2388                 return -ENOMEM;
2389         }
2390         session->cipher_key.length = cipher_xform->key.length;
2391         session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length,
2392                                              RTE_CACHE_LINE_SIZE);
2393         if (session->auth_key.data == NULL && auth_xform->key.length > 0) {
2394                 DPAA_SEC_ERR("No Memory for auth key");
2395                 return -ENOMEM;
2396         }
2397         session->auth_key.length = auth_xform->key.length;
2398         memcpy(session->cipher_key.data, cipher_xform->key.data,
2399                cipher_xform->key.length);
2400         memcpy(session->auth_key.data, auth_xform->key.data,
2401                auth_xform->key.length);
2402
2403         session->digest_length = auth_xform->digest_length;
2404         session->auth_alg = auth_xform->algo;
2405
2406         switch (auth_xform->algo) {
2407         case RTE_CRYPTO_AUTH_SHA1_HMAC:
2408                 session->auth_key.alg = OP_ALG_ALGSEL_SHA1;
2409                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2410                 break;
2411         case RTE_CRYPTO_AUTH_MD5_HMAC:
2412                 session->auth_key.alg = OP_ALG_ALGSEL_MD5;
2413                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2414                 break;
2415         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2416                 session->auth_key.alg = OP_ALG_ALGSEL_SHA224;
2417                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2418                 break;
2419         case RTE_CRYPTO_AUTH_SHA256_HMAC:
2420                 session->auth_key.alg = OP_ALG_ALGSEL_SHA256;
2421                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2422                 break;
2423         case RTE_CRYPTO_AUTH_SHA384_HMAC:
2424                 session->auth_key.alg = OP_ALG_ALGSEL_SHA384;
2425                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2426                 break;
2427         case RTE_CRYPTO_AUTH_SHA512_HMAC:
2428                 session->auth_key.alg = OP_ALG_ALGSEL_SHA512;
2429                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2430                 break;
2431         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2432                 session->auth_key.alg = OP_ALG_ALGSEL_AES;
2433                 session->auth_key.algmode = OP_ALG_AAI_XCBC_MAC;
2434                 break;
2435         case RTE_CRYPTO_AUTH_AES_CMAC:
2436                 session->auth_key.alg = OP_ALG_ALGSEL_AES;
2437                 session->auth_key.algmode = OP_ALG_AAI_CMAC;
2438                 break;
2439         default:
2440                 DPAA_SEC_ERR("Crypto: Unsupported Auth specified %u",
2441                               auth_xform->algo);
2442                 return -ENOTSUP;
2443         }
2444
2445         session->cipher_alg = cipher_xform->algo;
2446
2447         switch (cipher_xform->algo) {
2448         case RTE_CRYPTO_CIPHER_AES_CBC:
2449                 session->cipher_key.alg = OP_ALG_ALGSEL_AES;
2450                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2451                 break;
2452         case RTE_CRYPTO_CIPHER_DES_CBC:
2453                 session->cipher_key.alg = OP_ALG_ALGSEL_DES;
2454                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2455                 break;
2456         case RTE_CRYPTO_CIPHER_3DES_CBC:
2457                 session->cipher_key.alg = OP_ALG_ALGSEL_3DES;
2458                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2459                 break;
2460         case RTE_CRYPTO_CIPHER_AES_CTR:
2461                 session->cipher_key.alg = OP_ALG_ALGSEL_AES;
2462                 session->cipher_key.algmode = OP_ALG_AAI_CTR;
2463                 break;
2464         default:
2465                 DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u",
2466                               cipher_xform->algo);
2467                 return -ENOTSUP;
2468         }
2469         session->dir = (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2470                                 DIR_ENC : DIR_DEC;
2471         return 0;
2472 }
2473
2474 static int
2475 dpaa_sec_aead_init(struct rte_cryptodev *dev __rte_unused,
2476                    struct rte_crypto_sym_xform *xform,
2477                    dpaa_sec_session *session)
2478 {
2479         session->aead_alg = xform->aead.algo;
2480         session->ctxt = DPAA_SEC_AEAD;
2481         session->iv.length = xform->aead.iv.length;
2482         session->iv.offset = xform->aead.iv.offset;
2483         session->auth_only_len = xform->aead.aad_length;
2484         session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length,
2485                                              RTE_CACHE_LINE_SIZE);
2486         if (session->aead_key.data == NULL && xform->aead.key.length > 0) {
2487                 DPAA_SEC_ERR("No Memory for aead key\n");
2488                 return -ENOMEM;
2489         }
2490         session->aead_key.length = xform->aead.key.length;
2491         session->digest_length = xform->aead.digest_length;
2492
2493         memcpy(session->aead_key.data, xform->aead.key.data,
2494                xform->aead.key.length);
2495
2496         switch (session->aead_alg) {
2497         case RTE_CRYPTO_AEAD_AES_GCM:
2498                 session->aead_key.alg = OP_ALG_ALGSEL_AES;
2499                 session->aead_key.algmode = OP_ALG_AAI_GCM;
2500                 break;
2501         default:
2502                 DPAA_SEC_ERR("unsupported AEAD alg %d", session->aead_alg);
2503                 return -ENOTSUP;
2504         }
2505
2506         session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
2507                         DIR_ENC : DIR_DEC;
2508
2509         return 0;
2510 }
2511
2512 static struct qman_fq *
2513 dpaa_sec_attach_rxq(struct dpaa_sec_dev_private *qi)
2514 {
2515         unsigned int i;
2516
2517         for (i = 0; i < RTE_DPAA_MAX_RX_QUEUE; i++) {
2518                 if (qi->inq_attach[i] == 0) {
2519                         qi->inq_attach[i] = 1;
2520                         return &qi->inq[i];
2521                 }
2522         }
2523         DPAA_SEC_WARN("All session in use %u", qi->max_nb_sessions);
2524
2525         return NULL;
2526 }
2527
2528 static int
2529 dpaa_sec_detach_rxq(struct dpaa_sec_dev_private *qi, struct qman_fq *fq)
2530 {
2531         unsigned int i;
2532
2533         for (i = 0; i < RTE_DPAA_MAX_RX_QUEUE; i++) {
2534                 if (&qi->inq[i] == fq) {
2535                         if (qman_retire_fq(fq, NULL) != 0)
2536                                 DPAA_SEC_DEBUG("Queue is not retired\n");
2537                         qman_oos_fq(fq);
2538                         qi->inq_attach[i] = 0;
2539                         return 0;
2540                 }
2541         }
2542         return -1;
2543 }
2544
2545 int
2546 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess)
2547 {
2548         int ret;
2549
2550         sess->qp[rte_lcore_id() % MAX_DPAA_CORES] = qp;
2551         if (unlikely(!DPAA_PER_LCORE_PORTAL)) {
2552                 ret = rte_dpaa_portal_init((void *)0);
2553                 if (ret) {
2554                         DPAA_SEC_ERR("Failure in affining portal");
2555                         return ret;
2556                 }
2557         }
2558         ret = dpaa_sec_init_rx(sess->inq[rte_lcore_id() % MAX_DPAA_CORES],
2559                                rte_dpaa_mem_vtop(&sess->cdb),
2560                                qman_fq_fqid(&qp->outq));
2561         if (ret)
2562                 DPAA_SEC_ERR("Unable to init sec queue");
2563
2564         return ret;
2565 }
2566
2567 static inline void
2568 free_session_data(dpaa_sec_session *s)
2569 {
2570         if (is_aead(s))
2571                 rte_free(s->aead_key.data);
2572         else {
2573                 rte_free(s->auth_key.data);
2574                 rte_free(s->cipher_key.data);
2575         }
2576         memset(s, 0, sizeof(dpaa_sec_session));
2577 }
2578
2579 static int
2580 dpaa_sec_set_session_parameters(struct rte_cryptodev *dev,
2581                             struct rte_crypto_sym_xform *xform, void *sess)
2582 {
2583         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
2584         dpaa_sec_session *session = sess;
2585         uint32_t i;
2586         int ret;
2587
2588         PMD_INIT_FUNC_TRACE();
2589
2590         if (unlikely(sess == NULL)) {
2591                 DPAA_SEC_ERR("invalid session struct");
2592                 return -EINVAL;
2593         }
2594         memset(session, 0, sizeof(dpaa_sec_session));
2595
2596         /* Default IV length = 0 */
2597         session->iv.length = 0;
2598
2599         /* Cipher Only */
2600         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
2601                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
2602                 ret = dpaa_sec_cipher_init(dev, xform, session);
2603
2604         /* Authentication Only */
2605         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
2606                    xform->next == NULL) {
2607                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
2608                 session->ctxt = DPAA_SEC_AUTH;
2609                 ret = dpaa_sec_auth_init(dev, xform, session);
2610
2611         /* Cipher then Authenticate */
2612         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
2613                    xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
2614                 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
2615                         session->auth_cipher_text = 1;
2616                         if (xform->cipher.algo == RTE_CRYPTO_CIPHER_NULL)
2617                                 ret = dpaa_sec_auth_init(dev, xform, session);
2618                         else if (xform->next->auth.algo == RTE_CRYPTO_AUTH_NULL)
2619                                 ret = dpaa_sec_cipher_init(dev, xform, session);
2620                         else
2621                                 ret = dpaa_sec_chain_init(dev, xform, session);
2622                 } else {
2623                         DPAA_SEC_ERR("Not supported: Auth then Cipher");
2624                         return -ENOTSUP;
2625                 }
2626         /* Authenticate then Cipher */
2627         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
2628                    xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
2629                 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
2630                         session->auth_cipher_text = 0;
2631                         if (xform->auth.algo == RTE_CRYPTO_AUTH_NULL)
2632                                 ret = dpaa_sec_cipher_init(dev, xform, session);
2633                         else if (xform->next->cipher.algo
2634                                         == RTE_CRYPTO_CIPHER_NULL)
2635                                 ret = dpaa_sec_auth_init(dev, xform, session);
2636                         else
2637                                 ret = dpaa_sec_chain_init(dev, xform, session);
2638                 } else {
2639                         DPAA_SEC_ERR("Not supported: Auth then Cipher");
2640                         return -ENOTSUP;
2641                 }
2642
2643         /* AEAD operation for AES-GCM kind of Algorithms */
2644         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
2645                    xform->next == NULL) {
2646                 ret = dpaa_sec_aead_init(dev, xform, session);
2647
2648         } else {
2649                 DPAA_SEC_ERR("Invalid crypto type");
2650                 return -EINVAL;
2651         }
2652         if (ret) {
2653                 DPAA_SEC_ERR("unable to init session");
2654                 goto err1;
2655         }
2656
2657         rte_spinlock_lock(&internals->lock);
2658         for (i = 0; i < MAX_DPAA_CORES; i++) {
2659                 session->inq[i] = dpaa_sec_attach_rxq(internals);
2660                 if (session->inq[i] == NULL) {
2661                         DPAA_SEC_ERR("unable to attach sec queue");
2662                         rte_spinlock_unlock(&internals->lock);
2663                         ret = -EBUSY;
2664                         goto err1;
2665                 }
2666         }
2667         rte_spinlock_unlock(&internals->lock);
2668
2669         return 0;
2670
2671 err1:
2672         free_session_data(session);
2673         return ret;
2674 }
2675
2676 static int
2677 dpaa_sec_sym_session_configure(struct rte_cryptodev *dev,
2678                 struct rte_crypto_sym_xform *xform,
2679                 struct rte_cryptodev_sym_session *sess,
2680                 struct rte_mempool *mempool)
2681 {
2682         void *sess_private_data;
2683         int ret;
2684
2685         PMD_INIT_FUNC_TRACE();
2686
2687         if (rte_mempool_get(mempool, &sess_private_data)) {
2688                 DPAA_SEC_ERR("Couldn't get object from session mempool");
2689                 return -ENOMEM;
2690         }
2691
2692         ret = dpaa_sec_set_session_parameters(dev, xform, sess_private_data);
2693         if (ret != 0) {
2694                 DPAA_SEC_ERR("failed to configure session parameters");
2695
2696                 /* Return session to mempool */
2697                 rte_mempool_put(mempool, sess_private_data);
2698                 return ret;
2699         }
2700
2701         set_sym_session_private_data(sess, dev->driver_id,
2702                         sess_private_data);
2703
2704         ret = dpaa_sec_prep_cdb(sess_private_data);
2705         if (ret) {
2706                 DPAA_SEC_ERR("Unable to prepare sec cdb");
2707                 return ret;
2708         }
2709
2710         return 0;
2711 }
2712
2713 static inline void
2714 free_session_memory(struct rte_cryptodev *dev, dpaa_sec_session *s)
2715 {
2716         struct dpaa_sec_dev_private *qi = dev->data->dev_private;
2717         struct rte_mempool *sess_mp = rte_mempool_from_obj((void *)s);
2718         uint8_t i;
2719
2720         for (i = 0; i < MAX_DPAA_CORES; i++) {
2721                 if (s->inq[i])
2722                         dpaa_sec_detach_rxq(qi, s->inq[i]);
2723                 s->inq[i] = NULL;
2724                 s->qp[i] = NULL;
2725         }
2726         free_session_data(s);
2727         rte_mempool_put(sess_mp, (void *)s);
2728 }
2729
2730 /** Clear the memory of session so it doesn't leave key material behind */
2731 static void
2732 dpaa_sec_sym_session_clear(struct rte_cryptodev *dev,
2733                 struct rte_cryptodev_sym_session *sess)
2734 {
2735         PMD_INIT_FUNC_TRACE();
2736         uint8_t index = dev->driver_id;
2737         void *sess_priv = get_sym_session_private_data(sess, index);
2738         dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
2739
2740         if (sess_priv) {
2741                 free_session_memory(dev, s);
2742                 set_sym_session_private_data(sess, index, NULL);
2743         }
2744 }
2745
2746 #ifdef RTE_LIB_SECURITY
2747 static int
2748 dpaa_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform,
2749                         struct rte_security_ipsec_xform *ipsec_xform,
2750                         dpaa_sec_session *session)
2751 {
2752         PMD_INIT_FUNC_TRACE();
2753
2754         session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length,
2755                                                RTE_CACHE_LINE_SIZE);
2756         if (session->aead_key.data == NULL && aead_xform->key.length > 0) {
2757                 DPAA_SEC_ERR("No Memory for aead key");
2758                 return -ENOMEM;
2759         }
2760         memcpy(session->aead_key.data, aead_xform->key.data,
2761                aead_xform->key.length);
2762
2763         session->digest_length = aead_xform->digest_length;
2764         session->aead_key.length = aead_xform->key.length;
2765
2766         switch (aead_xform->algo) {
2767         case RTE_CRYPTO_AEAD_AES_GCM:
2768                 switch (session->digest_length) {
2769                 case 8:
2770                         session->aead_key.alg = OP_PCL_IPSEC_AES_GCM8;
2771                         break;
2772                 case 12:
2773                         session->aead_key.alg = OP_PCL_IPSEC_AES_GCM12;
2774                         break;
2775                 case 16:
2776                         session->aead_key.alg = OP_PCL_IPSEC_AES_GCM16;
2777                         break;
2778                 default:
2779                         DPAA_SEC_ERR("Crypto: Undefined GCM digest %d",
2780                                      session->digest_length);
2781                         return -EINVAL;
2782                 }
2783                 if (session->dir == DIR_ENC) {
2784                         memcpy(session->encap_pdb.gcm.salt,
2785                                 (uint8_t *)&(ipsec_xform->salt), 4);
2786                 } else {
2787                         memcpy(session->decap_pdb.gcm.salt,
2788                                 (uint8_t *)&(ipsec_xform->salt), 4);
2789                 }
2790                 session->aead_key.algmode = OP_ALG_AAI_GCM;
2791                 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM;
2792                 break;
2793         default:
2794                 DPAA_SEC_ERR("Crypto: Undefined AEAD specified %u",
2795                               aead_xform->algo);
2796                 return -ENOTSUP;
2797         }
2798         return 0;
2799 }
2800
2801 static int
2802 dpaa_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform,
2803         struct rte_crypto_auth_xform *auth_xform,
2804         struct rte_security_ipsec_xform *ipsec_xform,
2805         dpaa_sec_session *session)
2806 {
2807         if (cipher_xform) {
2808                 session->cipher_key.data = rte_zmalloc(NULL,
2809                                                        cipher_xform->key.length,
2810                                                        RTE_CACHE_LINE_SIZE);
2811                 if (session->cipher_key.data == NULL &&
2812                                 cipher_xform->key.length > 0) {
2813                         DPAA_SEC_ERR("No Memory for cipher key");
2814                         return -ENOMEM;
2815                 }
2816
2817                 session->cipher_key.length = cipher_xform->key.length;
2818                 memcpy(session->cipher_key.data, cipher_xform->key.data,
2819                                 cipher_xform->key.length);
2820                 session->cipher_alg = cipher_xform->algo;
2821         } else {
2822                 session->cipher_key.data = NULL;
2823                 session->cipher_key.length = 0;
2824                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
2825         }
2826
2827         if (auth_xform) {
2828                 session->auth_key.data = rte_zmalloc(NULL,
2829                                                 auth_xform->key.length,
2830                                                 RTE_CACHE_LINE_SIZE);
2831                 if (session->auth_key.data == NULL &&
2832                                 auth_xform->key.length > 0) {
2833                         DPAA_SEC_ERR("No Memory for auth key");
2834                         return -ENOMEM;
2835                 }
2836                 session->auth_key.length = auth_xform->key.length;
2837                 memcpy(session->auth_key.data, auth_xform->key.data,
2838                                 auth_xform->key.length);
2839                 session->auth_alg = auth_xform->algo;
2840                 session->digest_length = auth_xform->digest_length;
2841         } else {
2842                 session->auth_key.data = NULL;
2843                 session->auth_key.length = 0;
2844                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
2845         }
2846
2847         switch (session->auth_alg) {
2848         case RTE_CRYPTO_AUTH_SHA1_HMAC:
2849                 session->auth_key.alg = OP_PCL_IPSEC_HMAC_SHA1_96;
2850                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2851                 break;
2852         case RTE_CRYPTO_AUTH_MD5_HMAC:
2853                 session->auth_key.alg = OP_PCL_IPSEC_HMAC_MD5_96;
2854                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2855                 break;
2856         case RTE_CRYPTO_AUTH_SHA256_HMAC:
2857                 session->auth_key.alg = OP_PCL_IPSEC_HMAC_SHA2_256_128;
2858                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2859                 if (session->digest_length != 16)
2860                         DPAA_SEC_WARN(
2861                         "+++Using sha256-hmac truncated len is non-standard,"
2862                         "it will not work with lookaside proto");
2863                 break;
2864         case RTE_CRYPTO_AUTH_SHA384_HMAC:
2865                 session->auth_key.alg = OP_PCL_IPSEC_HMAC_SHA2_384_192;
2866                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2867                 break;
2868         case RTE_CRYPTO_AUTH_SHA512_HMAC:
2869                 session->auth_key.alg = OP_PCL_IPSEC_HMAC_SHA2_512_256;
2870                 session->auth_key.algmode = OP_ALG_AAI_HMAC;
2871                 break;
2872         case RTE_CRYPTO_AUTH_AES_CMAC:
2873                 session->auth_key.alg = OP_PCL_IPSEC_AES_CMAC_96;
2874                 session->auth_key.algmode = OP_ALG_AAI_CMAC;
2875                 break;
2876         case RTE_CRYPTO_AUTH_NULL:
2877                 session->auth_key.alg = OP_PCL_IPSEC_HMAC_NULL;
2878                 break;
2879         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2880                 session->auth_key.alg = OP_PCL_IPSEC_AES_XCBC_MAC_96;
2881                 session->auth_key.algmode = OP_ALG_AAI_XCBC_MAC;
2882                 break;
2883         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2884         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2885         case RTE_CRYPTO_AUTH_SHA1:
2886         case RTE_CRYPTO_AUTH_SHA256:
2887         case RTE_CRYPTO_AUTH_SHA512:
2888         case RTE_CRYPTO_AUTH_SHA224:
2889         case RTE_CRYPTO_AUTH_SHA384:
2890         case RTE_CRYPTO_AUTH_MD5:
2891         case RTE_CRYPTO_AUTH_AES_GMAC:
2892         case RTE_CRYPTO_AUTH_KASUMI_F9:
2893         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2894         case RTE_CRYPTO_AUTH_ZUC_EIA3:
2895                 DPAA_SEC_ERR("Crypto: Unsupported auth alg %u",
2896                               session->auth_alg);
2897                 return -ENOTSUP;
2898         default:
2899                 DPAA_SEC_ERR("Crypto: Undefined Auth specified %u",
2900                               session->auth_alg);
2901                 return -ENOTSUP;
2902         }
2903
2904         switch (session->cipher_alg) {
2905         case RTE_CRYPTO_CIPHER_AES_CBC:
2906                 session->cipher_key.alg = OP_PCL_IPSEC_AES_CBC;
2907                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2908                 break;
2909         case RTE_CRYPTO_CIPHER_DES_CBC:
2910                 session->cipher_key.alg = OP_PCL_IPSEC_DES;
2911                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2912                 break;
2913         case RTE_CRYPTO_CIPHER_3DES_CBC:
2914                 session->cipher_key.alg = OP_PCL_IPSEC_3DES;
2915                 session->cipher_key.algmode = OP_ALG_AAI_CBC;
2916                 break;
2917         case RTE_CRYPTO_CIPHER_AES_CTR:
2918                 session->cipher_key.alg = OP_PCL_IPSEC_AES_CTR;
2919                 session->cipher_key.algmode = OP_ALG_AAI_CTR;
2920                 if (session->dir == DIR_ENC) {
2921                         session->encap_pdb.ctr.ctr_initial = 0x00000001;
2922                         session->encap_pdb.ctr.ctr_nonce = ipsec_xform->salt;
2923                 } else {
2924                         session->decap_pdb.ctr.ctr_initial = 0x00000001;
2925                         session->decap_pdb.ctr.ctr_nonce = ipsec_xform->salt;
2926                 }
2927                 break;
2928         case RTE_CRYPTO_CIPHER_NULL:
2929                 session->cipher_key.alg = OP_PCL_IPSEC_NULL;
2930                 break;
2931         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2932         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2933         case RTE_CRYPTO_CIPHER_3DES_ECB:
2934         case RTE_CRYPTO_CIPHER_AES_ECB:
2935         case RTE_CRYPTO_CIPHER_KASUMI_F8:
2936                 DPAA_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2937                               session->cipher_alg);
2938                 return -ENOTSUP;
2939         default:
2940                 DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u",
2941                               session->cipher_alg);
2942                 return -ENOTSUP;
2943         }
2944
2945         return 0;
2946 }
2947
2948 static int
2949 dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev,
2950                            struct rte_security_session_conf *conf,
2951                            void *sess)
2952 {
2953         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
2954         struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
2955         struct rte_crypto_auth_xform *auth_xform = NULL;
2956         struct rte_crypto_cipher_xform *cipher_xform = NULL;
2957         struct rte_crypto_aead_xform *aead_xform = NULL;
2958         dpaa_sec_session *session = (dpaa_sec_session *)sess;
2959         uint32_t i;
2960         int ret;
2961
2962         PMD_INIT_FUNC_TRACE();
2963
2964         memset(session, 0, sizeof(dpaa_sec_session));
2965         session->proto_alg = conf->protocol;
2966         session->ctxt = DPAA_SEC_IPSEC;
2967
2968         if (ipsec_xform->life.bytes_hard_limit != 0 ||
2969             ipsec_xform->life.bytes_soft_limit != 0 ||
2970             ipsec_xform->life.packets_hard_limit != 0 ||
2971             ipsec_xform->life.packets_soft_limit != 0)
2972                 return -ENOTSUP;
2973
2974         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS)
2975                 session->dir = DIR_ENC;
2976         else
2977                 session->dir = DIR_DEC;
2978
2979         if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
2980                 cipher_xform = &conf->crypto_xform->cipher;
2981                 if (conf->crypto_xform->next)
2982                         auth_xform = &conf->crypto_xform->next->auth;
2983                 ret = dpaa_sec_ipsec_proto_init(cipher_xform, auth_xform,
2984                                         ipsec_xform, session);
2985         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
2986                 auth_xform = &conf->crypto_xform->auth;
2987                 if (conf->crypto_xform->next)
2988                         cipher_xform = &conf->crypto_xform->next->cipher;
2989                 ret = dpaa_sec_ipsec_proto_init(cipher_xform, auth_xform,
2990                                         ipsec_xform, session);
2991         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
2992                 aead_xform = &conf->crypto_xform->aead;
2993                 ret = dpaa_sec_ipsec_aead_init(aead_xform,
2994                                         ipsec_xform, session);
2995         } else {
2996                 DPAA_SEC_ERR("XFORM not specified");
2997                 ret = -EINVAL;
2998                 goto out;
2999         }
3000         if (ret) {
3001                 DPAA_SEC_ERR("Failed to process xform");
3002                 goto out;
3003         }
3004
3005         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
3006                 if (ipsec_xform->tunnel.type ==
3007                                 RTE_SECURITY_IPSEC_TUNNEL_IPV4) {
3008                         session->ip4_hdr.ip_v = IPVERSION;
3009                         session->ip4_hdr.ip_hl = 5;
3010                         session->ip4_hdr.ip_len = rte_cpu_to_be_16(
3011                                                 sizeof(session->ip4_hdr));
3012                         session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
3013                         session->ip4_hdr.ip_id = 0;
3014                         session->ip4_hdr.ip_off = 0;
3015                         session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
3016                         session->ip4_hdr.ip_p = (ipsec_xform->proto ==
3017                                         RTE_SECURITY_IPSEC_SA_PROTO_ESP) ?
3018                                         IPPROTO_ESP : IPPROTO_AH;
3019                         session->ip4_hdr.ip_sum = 0;
3020                         session->ip4_hdr.ip_src =
3021                                         ipsec_xform->tunnel.ipv4.src_ip;
3022                         session->ip4_hdr.ip_dst =
3023                                         ipsec_xform->tunnel.ipv4.dst_ip;
3024                         session->ip4_hdr.ip_sum = calc_chksum((uint16_t *)
3025                                                 (void *)&session->ip4_hdr,
3026                                                 sizeof(struct ip));
3027                         session->encap_pdb.ip_hdr_len = sizeof(struct ip);
3028                 } else if (ipsec_xform->tunnel.type ==
3029                                 RTE_SECURITY_IPSEC_TUNNEL_IPV6) {
3030                         session->ip6_hdr.vtc_flow = rte_cpu_to_be_32(
3031                                 DPAA_IPv6_DEFAULT_VTC_FLOW |
3032                                 ((ipsec_xform->tunnel.ipv6.dscp <<
3033                                         RTE_IPV6_HDR_TC_SHIFT) &
3034                                         RTE_IPV6_HDR_TC_MASK) |
3035                                 ((ipsec_xform->tunnel.ipv6.flabel <<
3036                                         RTE_IPV6_HDR_FL_SHIFT) &
3037                                         RTE_IPV6_HDR_FL_MASK));
3038                         /* Payload length will be updated by HW */
3039                         session->ip6_hdr.payload_len = 0;
3040                         session->ip6_hdr.hop_limits =
3041                                         ipsec_xform->tunnel.ipv6.hlimit;
3042                         session->ip6_hdr.proto = (ipsec_xform->proto ==
3043                                         RTE_SECURITY_IPSEC_SA_PROTO_ESP) ?
3044                                         IPPROTO_ESP : IPPROTO_AH;
3045                         memcpy(&session->ip6_hdr.src_addr,
3046                                         &ipsec_xform->tunnel.ipv6.src_addr, 16);
3047                         memcpy(&session->ip6_hdr.dst_addr,
3048                                         &ipsec_xform->tunnel.ipv6.dst_addr, 16);
3049                         session->encap_pdb.ip_hdr_len =
3050                                                 sizeof(struct rte_ipv6_hdr);
3051                 }
3052
3053                 session->encap_pdb.options =
3054                         (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
3055                         PDBOPTS_ESP_OIHI_PDB_INL |
3056                         PDBOPTS_ESP_IVSRC |
3057                         PDBHMO_ESP_SNR;
3058                 if (ipsec_xform->options.dec_ttl)
3059                         session->encap_pdb.options |= PDBHMO_ESP_ENCAP_DTTL;
3060                 if (ipsec_xform->options.esn)
3061                         session->encap_pdb.options |= PDBOPTS_ESP_ESN;
3062                 session->encap_pdb.spi = ipsec_xform->spi;
3063
3064         } else if (ipsec_xform->direction ==
3065                         RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
3066                 if (ipsec_xform->tunnel.type == RTE_SECURITY_IPSEC_TUNNEL_IPV4)
3067                         session->decap_pdb.options = sizeof(struct ip) << 16;
3068                 else
3069                         session->decap_pdb.options =
3070                                         sizeof(struct rte_ipv6_hdr) << 16;
3071                 if (ipsec_xform->options.esn)
3072                         session->decap_pdb.options |= PDBOPTS_ESP_ESN;
3073                 if (ipsec_xform->replay_win_sz) {
3074                         uint32_t win_sz;
3075                         win_sz = rte_align32pow2(ipsec_xform->replay_win_sz);
3076
3077                         switch (win_sz) {
3078                         case 1:
3079                         case 2:
3080                         case 4:
3081                         case 8:
3082                         case 16:
3083                         case 32:
3084                                 session->decap_pdb.options |= PDBOPTS_ESP_ARS32;
3085                                 break;
3086                         case 64:
3087                                 session->decap_pdb.options |= PDBOPTS_ESP_ARS64;
3088                                 break;
3089                         default:
3090                                 session->decap_pdb.options |=
3091                                                         PDBOPTS_ESP_ARS128;
3092                         }
3093                 }
3094         } else
3095                 goto out;
3096         rte_spinlock_lock(&internals->lock);
3097         for (i = 0; i < MAX_DPAA_CORES; i++) {
3098                 session->inq[i] = dpaa_sec_attach_rxq(internals);
3099                 if (session->inq[i] == NULL) {
3100                         DPAA_SEC_ERR("unable to attach sec queue");
3101                         rte_spinlock_unlock(&internals->lock);
3102                         goto out;
3103                 }
3104         }
3105         rte_spinlock_unlock(&internals->lock);
3106
3107         return 0;
3108 out:
3109         free_session_data(session);
3110         return -1;
3111 }
3112
3113 static int
3114 dpaa_sec_set_pdcp_session(struct rte_cryptodev *dev,
3115                           struct rte_security_session_conf *conf,
3116                           void *sess)
3117 {
3118         struct rte_security_pdcp_xform *pdcp_xform = &conf->pdcp;
3119         struct rte_crypto_sym_xform *xform = conf->crypto_xform;
3120         struct rte_crypto_auth_xform *auth_xform = NULL;
3121         struct rte_crypto_cipher_xform *cipher_xform = NULL;
3122         dpaa_sec_session *session = (dpaa_sec_session *)sess;
3123         struct dpaa_sec_dev_private *dev_priv = dev->data->dev_private;
3124         uint32_t i;
3125         int ret;
3126
3127         PMD_INIT_FUNC_TRACE();
3128
3129         memset(session, 0, sizeof(dpaa_sec_session));
3130
3131         /* find xfrm types */
3132         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
3133                 cipher_xform = &xform->cipher;
3134                 if (xform->next != NULL &&
3135                         xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
3136                         auth_xform = &xform->next->auth;
3137         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
3138                 auth_xform = &xform->auth;
3139                 if (xform->next != NULL &&
3140                         xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
3141                         cipher_xform = &xform->next->cipher;
3142         } else {
3143                 DPAA_SEC_ERR("Invalid crypto type");
3144                 return -EINVAL;
3145         }
3146
3147         session->proto_alg = conf->protocol;
3148         session->ctxt = DPAA_SEC_PDCP;
3149
3150         if (cipher_xform) {
3151                 switch (cipher_xform->algo) {
3152                 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
3153                         session->cipher_key.alg = PDCP_CIPHER_TYPE_SNOW;
3154                         break;
3155                 case RTE_CRYPTO_CIPHER_ZUC_EEA3:
3156                         session->cipher_key.alg = PDCP_CIPHER_TYPE_ZUC;
3157                         break;
3158                 case RTE_CRYPTO_CIPHER_AES_CTR:
3159                         session->cipher_key.alg = PDCP_CIPHER_TYPE_AES;
3160                         break;
3161                 case RTE_CRYPTO_CIPHER_NULL:
3162                         session->cipher_key.alg = PDCP_CIPHER_TYPE_NULL;
3163                         break;
3164                 default:
3165                         DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u",
3166                                       session->cipher_alg);
3167                         return -EINVAL;
3168                 }
3169
3170                 session->cipher_key.data = rte_zmalloc(NULL,
3171                                                cipher_xform->key.length,
3172                                                RTE_CACHE_LINE_SIZE);
3173                 if (session->cipher_key.data == NULL &&
3174                                 cipher_xform->key.length > 0) {
3175                         DPAA_SEC_ERR("No Memory for cipher key");
3176                         return -ENOMEM;
3177                 }
3178                 session->cipher_key.length = cipher_xform->key.length;
3179                 memcpy(session->cipher_key.data, cipher_xform->key.data,
3180                         cipher_xform->key.length);
3181                 session->dir = (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
3182                                         DIR_ENC : DIR_DEC;
3183                 session->cipher_alg = cipher_xform->algo;
3184         } else {
3185                 session->cipher_key.data = NULL;
3186                 session->cipher_key.length = 0;
3187                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
3188                 session->dir = DIR_ENC;
3189         }
3190
3191         if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
3192                 if (pdcp_xform->sn_size != RTE_SECURITY_PDCP_SN_SIZE_5 &&
3193                     pdcp_xform->sn_size != RTE_SECURITY_PDCP_SN_SIZE_12) {
3194                         DPAA_SEC_ERR(
3195                                 "PDCP Seq Num size should be 5/12 bits for cmode");
3196                         ret = -EINVAL;
3197                         goto out;
3198                 }
3199         }
3200
3201         if (auth_xform) {
3202                 switch (auth_xform->algo) {
3203                 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
3204                         session->auth_key.alg = PDCP_AUTH_TYPE_SNOW;
3205                         break;
3206                 case RTE_CRYPTO_AUTH_ZUC_EIA3:
3207                         session->auth_key.alg = PDCP_AUTH_TYPE_ZUC;
3208                         break;
3209                 case RTE_CRYPTO_AUTH_AES_CMAC:
3210                         session->auth_key.alg = PDCP_AUTH_TYPE_AES;
3211                         break;
3212                 case RTE_CRYPTO_AUTH_NULL:
3213                         session->auth_key.alg = PDCP_AUTH_TYPE_NULL;
3214                         break;
3215                 default:
3216                         DPAA_SEC_ERR("Crypto: Unsupported auth alg %u",
3217                                       session->auth_alg);
3218                         rte_free(session->cipher_key.data);
3219                         return -EINVAL;
3220                 }
3221                 session->auth_key.data = rte_zmalloc(NULL,
3222                                                      auth_xform->key.length,
3223                                                      RTE_CACHE_LINE_SIZE);
3224                 if (!session->auth_key.data &&
3225                     auth_xform->key.length > 0) {
3226                         DPAA_SEC_ERR("No Memory for auth key");
3227                         rte_free(session->cipher_key.data);
3228                         return -ENOMEM;
3229                 }
3230                 session->auth_key.length = auth_xform->key.length;
3231                 memcpy(session->auth_key.data, auth_xform->key.data,
3232                        auth_xform->key.length);
3233                 session->auth_alg = auth_xform->algo;
3234         } else {
3235                 session->auth_key.data = NULL;
3236                 session->auth_key.length = 0;
3237                 session->auth_alg = 0;
3238         }
3239         session->pdcp.domain = pdcp_xform->domain;
3240         session->pdcp.bearer = pdcp_xform->bearer;
3241         session->pdcp.pkt_dir = pdcp_xform->pkt_dir;
3242         session->pdcp.sn_size = pdcp_xform->sn_size;
3243         session->pdcp.hfn = pdcp_xform->hfn;
3244         session->pdcp.hfn_threshold = pdcp_xform->hfn_threshold;
3245         session->pdcp.hfn_ovd = pdcp_xform->hfn_ovrd;
3246         session->pdcp.sdap_enabled = pdcp_xform->sdap_enabled;
3247         if (cipher_xform)
3248                 session->pdcp.hfn_ovd_offset = cipher_xform->iv.offset;
3249
3250         rte_spinlock_lock(&dev_priv->lock);
3251         for (i = 0; i < MAX_DPAA_CORES; i++) {
3252                 session->inq[i] = dpaa_sec_attach_rxq(dev_priv);
3253                 if (session->inq[i] == NULL) {
3254                         DPAA_SEC_ERR("unable to attach sec queue");
3255                         rte_spinlock_unlock(&dev_priv->lock);
3256                         ret = -EBUSY;
3257                         goto out;
3258                 }
3259         }
3260         rte_spinlock_unlock(&dev_priv->lock);
3261         return 0;
3262 out:
3263         rte_free(session->auth_key.data);
3264         rte_free(session->cipher_key.data);
3265         memset(session, 0, sizeof(dpaa_sec_session));
3266         return ret;
3267 }
3268
3269 static int
3270 dpaa_sec_security_session_create(void *dev,
3271                                  struct rte_security_session_conf *conf,
3272                                  struct rte_security_session *sess,
3273                                  struct rte_mempool *mempool)
3274 {
3275         void *sess_private_data;
3276         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
3277         int ret;
3278
3279         if (rte_mempool_get(mempool, &sess_private_data)) {
3280                 DPAA_SEC_ERR("Couldn't get object from session mempool");
3281                 return -ENOMEM;
3282         }
3283
3284         switch (conf->protocol) {
3285         case RTE_SECURITY_PROTOCOL_IPSEC:
3286                 ret = dpaa_sec_set_ipsec_session(cdev, conf,
3287                                 sess_private_data);
3288                 break;
3289         case RTE_SECURITY_PROTOCOL_PDCP:
3290                 ret = dpaa_sec_set_pdcp_session(cdev, conf,
3291                                 sess_private_data);
3292                 break;
3293         case RTE_SECURITY_PROTOCOL_MACSEC:
3294                 return -ENOTSUP;
3295         default:
3296                 return -EINVAL;
3297         }
3298         if (ret != 0) {
3299                 DPAA_SEC_ERR("failed to configure session parameters");
3300                 /* Return session to mempool */
3301                 rte_mempool_put(mempool, sess_private_data);
3302                 return ret;
3303         }
3304
3305         set_sec_session_private_data(sess, sess_private_data);
3306
3307         ret = dpaa_sec_prep_cdb(sess_private_data);
3308         if (ret) {
3309                 DPAA_SEC_ERR("Unable to prepare sec cdb");
3310                 return ret;
3311         }
3312
3313         return ret;
3314 }
3315
3316 /** Clear the memory of session so it doesn't leave key material behind */
3317 static int
3318 dpaa_sec_security_session_destroy(void *dev __rte_unused,
3319                 struct rte_security_session *sess)
3320 {
3321         PMD_INIT_FUNC_TRACE();
3322         void *sess_priv = get_sec_session_private_data(sess);
3323         dpaa_sec_session *s = (dpaa_sec_session *)sess_priv;
3324
3325         if (sess_priv) {
3326                 free_session_memory((struct rte_cryptodev *)dev, s);
3327                 set_sec_session_private_data(sess, NULL);
3328         }
3329         return 0;
3330 }
3331 #endif
3332 static int
3333 dpaa_sec_dev_configure(struct rte_cryptodev *dev __rte_unused,
3334                        struct rte_cryptodev_config *config __rte_unused)
3335 {
3336         PMD_INIT_FUNC_TRACE();
3337
3338         return 0;
3339 }
3340
3341 static int
3342 dpaa_sec_dev_start(struct rte_cryptodev *dev __rte_unused)
3343 {
3344         PMD_INIT_FUNC_TRACE();
3345         return 0;
3346 }
3347
3348 static void
3349 dpaa_sec_dev_stop(struct rte_cryptodev *dev __rte_unused)
3350 {
3351         PMD_INIT_FUNC_TRACE();
3352 }
3353
3354 static int
3355 dpaa_sec_dev_close(struct rte_cryptodev *dev)
3356 {
3357         PMD_INIT_FUNC_TRACE();
3358
3359         if (dev == NULL)
3360                 return -ENOMEM;
3361
3362         return 0;
3363 }
3364
3365 static void
3366 dpaa_sec_dev_infos_get(struct rte_cryptodev *dev,
3367                        struct rte_cryptodev_info *info)
3368 {
3369         struct dpaa_sec_dev_private *internals = dev->data->dev_private;
3370
3371         PMD_INIT_FUNC_TRACE();
3372         if (info != NULL) {
3373                 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
3374                 info->feature_flags = dev->feature_flags;
3375                 info->capabilities = dpaa_sec_capabilities;
3376                 info->sym.max_nb_sessions = internals->max_nb_sessions;
3377                 info->driver_id = dpaa_cryptodev_driver_id;
3378         }
3379 }
3380
3381 static enum qman_cb_dqrr_result
3382 dpaa_sec_process_parallel_event(void *event,
3383                         struct qman_portal *qm __always_unused,
3384                         struct qman_fq *outq,
3385                         const struct qm_dqrr_entry *dqrr,
3386                         void **bufs)
3387 {
3388         const struct qm_fd *fd;
3389         struct dpaa_sec_job *job;
3390         struct dpaa_sec_op_ctx *ctx;
3391         struct rte_event *ev = (struct rte_event *)event;
3392
3393         fd = &dqrr->fd;
3394
3395         /* sg is embedded in an op ctx,
3396          * sg[0] is for output
3397          * sg[1] for input
3398          */
3399         job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd));
3400
3401         ctx = container_of(job, struct dpaa_sec_op_ctx, job);
3402         ctx->fd_status = fd->status;
3403         if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
3404                 struct qm_sg_entry *sg_out;
3405                 uint32_t len;
3406
3407                 sg_out = &job->sg[0];
3408                 hw_sg_to_cpu(sg_out);
3409                 len = sg_out->length;
3410                 ctx->op->sym->m_src->pkt_len = len;
3411                 ctx->op->sym->m_src->data_len = len;
3412         }
3413         if (!ctx->fd_status) {
3414                 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
3415         } else {
3416                 DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status);
3417                 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
3418         }
3419         ev->event_ptr = (void *)ctx->op;
3420
3421         ev->flow_id = outq->ev.flow_id;
3422         ev->sub_event_type = outq->ev.sub_event_type;
3423         ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
3424         ev->op = RTE_EVENT_OP_NEW;
3425         ev->sched_type = outq->ev.sched_type;
3426         ev->queue_id = outq->ev.queue_id;
3427         ev->priority = outq->ev.priority;
3428         *bufs = (void *)ctx->op;
3429
3430         rte_mempool_put(ctx->ctx_pool, (void *)ctx);
3431
3432         return qman_cb_dqrr_consume;
3433 }
3434
3435 static enum qman_cb_dqrr_result
3436 dpaa_sec_process_atomic_event(void *event,
3437                         struct qman_portal *qm __rte_unused,
3438                         struct qman_fq *outq,
3439                         const struct qm_dqrr_entry *dqrr,
3440                         void **bufs)
3441 {
3442         u8 index;
3443         const struct qm_fd *fd;
3444         struct dpaa_sec_job *job;
3445         struct dpaa_sec_op_ctx *ctx;
3446         struct rte_event *ev = (struct rte_event *)event;
3447
3448         fd = &dqrr->fd;
3449
3450         /* sg is embedded in an op ctx,
3451          * sg[0] is for output
3452          * sg[1] for input
3453          */
3454         job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd));
3455
3456         ctx = container_of(job, struct dpaa_sec_op_ctx, job);
3457         ctx->fd_status = fd->status;
3458         if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
3459                 struct qm_sg_entry *sg_out;
3460                 uint32_t len;
3461
3462                 sg_out = &job->sg[0];
3463                 hw_sg_to_cpu(sg_out);
3464                 len = sg_out->length;
3465                 ctx->op->sym->m_src->pkt_len = len;
3466                 ctx->op->sym->m_src->data_len = len;
3467         }
3468         if (!ctx->fd_status) {
3469                 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
3470         } else {
3471                 DPAA_SEC_DP_WARN("SEC return err: 0x%x", ctx->fd_status);
3472                 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR;
3473         }
3474         ev->event_ptr = (void *)ctx->op;
3475         ev->flow_id = outq->ev.flow_id;
3476         ev->sub_event_type = outq->ev.sub_event_type;
3477         ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
3478         ev->op = RTE_EVENT_OP_NEW;
3479         ev->sched_type = outq->ev.sched_type;
3480         ev->queue_id = outq->ev.queue_id;
3481         ev->priority = outq->ev.priority;
3482
3483         /* Save active dqrr entries */
3484         index = ((uintptr_t)dqrr >> 6) & (16/*QM_DQRR_SIZE*/ - 1);
3485         DPAA_PER_LCORE_DQRR_SIZE++;
3486         DPAA_PER_LCORE_DQRR_HELD |= 1 << index;
3487         DPAA_PER_LCORE_DQRR_MBUF(index) = ctx->op->sym->m_src;
3488         ev->impl_opaque = index + 1;
3489         *dpaa_seqn(ctx->op->sym->m_src) = (uint32_t)index + 1;
3490         *bufs = (void *)ctx->op;
3491
3492         rte_mempool_put(ctx->ctx_pool, (void *)ctx);
3493
3494         return qman_cb_dqrr_defer;
3495 }
3496
3497 int
3498 dpaa_sec_eventq_attach(const struct rte_cryptodev *dev,
3499                 int qp_id,
3500                 uint16_t ch_id,
3501                 const struct rte_event *event)
3502 {
3503         struct dpaa_sec_qp *qp = dev->data->queue_pairs[qp_id];
3504         struct qm_mcc_initfq opts = {0};
3505
3506         int ret;
3507
3508         opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
3509                        QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB;
3510         opts.fqd.dest.channel = ch_id;
3511
3512         switch (event->sched_type) {
3513         case RTE_SCHED_TYPE_ATOMIC:
3514                 opts.fqd.fq_ctrl |= QM_FQCTRL_HOLDACTIVE;
3515                 /* Reset FQCTRL_AVOIDBLOCK bit as it is unnecessary
3516                  * configuration with HOLD_ACTIVE setting
3517                  */
3518                 opts.fqd.fq_ctrl &= (~QM_FQCTRL_AVOIDBLOCK);
3519                 qp->outq.cb.dqrr_dpdk_cb = dpaa_sec_process_atomic_event;
3520                 break;
3521         case RTE_SCHED_TYPE_ORDERED:
3522                 DPAA_SEC_ERR("Ordered queue schedule type is not supported\n");
3523                 return -ENOTSUP;
3524         default:
3525                 opts.fqd.fq_ctrl |= QM_FQCTRL_AVOIDBLOCK;
3526                 qp->outq.cb.dqrr_dpdk_cb = dpaa_sec_process_parallel_event;
3527                 break;
3528         }
3529
3530         ret = qman_init_fq(&qp->outq, QMAN_INITFQ_FLAG_SCHED, &opts);
3531         if (unlikely(ret)) {
3532                 DPAA_SEC_ERR("unable to init caam source fq!");
3533                 return ret;
3534         }
3535
3536         memcpy(&qp->outq.ev, event, sizeof(struct rte_event));
3537
3538         return 0;
3539 }
3540
3541 int
3542 dpaa_sec_eventq_detach(const struct rte_cryptodev *dev,
3543                         int qp_id)
3544 {
3545         struct qm_mcc_initfq opts = {0};
3546         int ret;
3547         struct dpaa_sec_qp *qp = dev->data->queue_pairs[qp_id];
3548
3549         opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
3550                        QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB;
3551         qp->outq.cb.dqrr = dqrr_out_fq_cb_rx;
3552         qp->outq.cb.ern  = ern_sec_fq_handler;
3553         qman_retire_fq(&qp->outq, NULL);
3554         qman_oos_fq(&qp->outq);
3555         ret = qman_init_fq(&qp->outq, 0, &opts);
3556         if (ret)
3557                 RTE_LOG(ERR, PMD, "Error in qman_init_fq: ret: %d\n", ret);
3558         qp->outq.cb.dqrr = NULL;
3559
3560         return ret;
3561 }
3562
3563 static struct rte_cryptodev_ops crypto_ops = {
3564         .dev_configure        = dpaa_sec_dev_configure,
3565         .dev_start            = dpaa_sec_dev_start,
3566         .dev_stop             = dpaa_sec_dev_stop,
3567         .dev_close            = dpaa_sec_dev_close,
3568         .dev_infos_get        = dpaa_sec_dev_infos_get,
3569         .queue_pair_setup     = dpaa_sec_queue_pair_setup,
3570         .queue_pair_release   = dpaa_sec_queue_pair_release,
3571         .sym_session_get_size     = dpaa_sec_sym_session_get_size,
3572         .sym_session_configure    = dpaa_sec_sym_session_configure,
3573         .sym_session_clear        = dpaa_sec_sym_session_clear,
3574         /* Raw data-path API related operations */
3575         .sym_get_raw_dp_ctx_size = dpaa_sec_get_dp_ctx_size,
3576         .sym_configure_raw_dp_ctx = dpaa_sec_configure_raw_dp_ctx,
3577 };
3578
3579 #ifdef RTE_LIB_SECURITY
3580 static const struct rte_security_capability *
3581 dpaa_sec_capabilities_get(void *device __rte_unused)
3582 {
3583         return dpaa_sec_security_cap;
3584 }
3585
3586 static const struct rte_security_ops dpaa_sec_security_ops = {
3587         .session_create = dpaa_sec_security_session_create,
3588         .session_update = NULL,
3589         .session_stats_get = NULL,
3590         .session_destroy = dpaa_sec_security_session_destroy,
3591         .set_pkt_metadata = NULL,
3592         .capabilities_get = dpaa_sec_capabilities_get
3593 };
3594 #endif
3595 static int
3596 dpaa_sec_uninit(struct rte_cryptodev *dev)
3597 {
3598         struct dpaa_sec_dev_private *internals;
3599
3600         if (dev == NULL)
3601                 return -ENODEV;
3602
3603         internals = dev->data->dev_private;
3604         rte_free(dev->security_ctx);
3605
3606         rte_free(internals);
3607
3608         DPAA_SEC_INFO("Closing DPAA_SEC device %s on numa socket %u",
3609                       dev->data->name, rte_socket_id());
3610
3611         return 0;
3612 }
3613
3614 static int
3615 check_devargs_handler(__rte_unused const char *key, const char *value,
3616                       __rte_unused void *opaque)
3617 {
3618         dpaa_sec_dp_dump = atoi(value);
3619         if (dpaa_sec_dp_dump > DPAA_SEC_DP_FULL_DUMP) {
3620                 DPAA_SEC_WARN("WARN: DPAA_SEC_DP_DUMP_LEVEL is not "
3621                               "supported, changing to FULL error prints\n");
3622                 dpaa_sec_dp_dump = DPAA_SEC_DP_FULL_DUMP;
3623         }
3624
3625         return 0;
3626 }
3627
3628 static void
3629 dpaa_sec_get_devargs(struct rte_devargs *devargs, const char *key)
3630 {
3631         struct rte_kvargs *kvlist;
3632
3633         if (!devargs)
3634                 return;
3635
3636         kvlist = rte_kvargs_parse(devargs->args, NULL);
3637         if (!kvlist)
3638                 return;
3639
3640         if (!rte_kvargs_count(kvlist, key)) {
3641                 rte_kvargs_free(kvlist);
3642                 return;
3643         }
3644
3645         rte_kvargs_process(kvlist, key,
3646                                 check_devargs_handler, NULL);
3647         rte_kvargs_free(kvlist);
3648 }
3649
3650 static int
3651 dpaa_sec_dev_init(struct rte_cryptodev *cryptodev)
3652 {
3653         struct dpaa_sec_dev_private *internals;
3654 #ifdef RTE_LIB_SECURITY
3655         struct rte_security_ctx *security_instance;
3656 #endif
3657         struct dpaa_sec_qp *qp;
3658         uint32_t i, flags;
3659         int ret;
3660         void *cmd_map;
3661         int map_fd = -1;
3662
3663         PMD_INIT_FUNC_TRACE();
3664
3665         internals = cryptodev->data->dev_private;
3666         map_fd = open("/dev/mem", O_RDWR);
3667         if (unlikely(map_fd < 0)) {
3668                 DPAA_SEC_ERR("Unable to open (/dev/mem)");
3669                 return map_fd;
3670         }
3671         internals->sec_hw = mmap(NULL, MAP_SIZE, PROT_READ | PROT_WRITE,
3672                             MAP_SHARED, map_fd, SEC_BASE_ADDR);
3673         if (internals->sec_hw == MAP_FAILED) {
3674                 DPAA_SEC_ERR("Memory map failed");
3675                 close(map_fd);
3676                 return -EINVAL;
3677         }
3678         cmd_map = (uint8_t *)internals->sec_hw +
3679                   (BLOCK_OFFSET * QI_BLOCK_NUMBER) + CMD_REG;
3680         if (!(be32_to_cpu(rte_read32(cmd_map)) & QICTL_DQEN))
3681                 /* enable QI interface */
3682                 rte_write32(cpu_to_be32(QICTL_DQEN), cmd_map);
3683
3684         ret = munmap(internals->sec_hw, MAP_SIZE);
3685         if (ret)
3686                 DPAA_SEC_WARN("munmap failed\n");
3687
3688         close(map_fd);
3689         cryptodev->driver_id = dpaa_cryptodev_driver_id;
3690         cryptodev->dev_ops = &crypto_ops;
3691
3692         cryptodev->enqueue_burst = dpaa_sec_enqueue_burst;
3693         cryptodev->dequeue_burst = dpaa_sec_dequeue_burst;
3694         cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
3695                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
3696                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
3697                         RTE_CRYPTODEV_FF_SECURITY |
3698                         RTE_CRYPTODEV_FF_SYM_RAW_DP |
3699                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
3700                         RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
3701                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
3702                         RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
3703                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
3704
3705         internals->max_nb_queue_pairs = RTE_DPAA_MAX_NB_SEC_QPS;
3706         internals->max_nb_sessions = RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS;
3707
3708         /*
3709          * For secondary processes, we don't initialise any further as primary
3710          * has already done this work. Only check we don't need a different
3711          * RX function
3712          */
3713         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3714                 DPAA_SEC_WARN("Device already init by primary process");
3715                 return 0;
3716         }
3717 #ifdef RTE_LIB_SECURITY
3718         /* Initialize security_ctx only for primary process*/
3719         security_instance = rte_malloc("rte_security_instances_ops",
3720                                 sizeof(struct rte_security_ctx), 0);
3721         if (security_instance == NULL)
3722                 return -ENOMEM;
3723         security_instance->device = (void *)cryptodev;
3724         security_instance->ops = &dpaa_sec_security_ops;
3725         security_instance->sess_cnt = 0;
3726         cryptodev->security_ctx = security_instance;
3727 #endif
3728         rte_spinlock_init(&internals->lock);
3729         for (i = 0; i < internals->max_nb_queue_pairs; i++) {
3730                 /* init qman fq for queue pair */
3731                 qp = &internals->qps[i];
3732                 ret = dpaa_sec_init_tx(&qp->outq);
3733                 if (ret) {
3734                         DPAA_SEC_ERR("config tx of queue pair  %d", i);
3735                         goto init_error;
3736                 }
3737         }
3738
3739         flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID |
3740                 QMAN_FQ_FLAG_TO_DCPORTAL;
3741         for (i = 0; i < RTE_DPAA_MAX_RX_QUEUE; i++) {
3742                 /* create rx qman fq for sessions*/
3743                 ret = qman_create_fq(0, flags, &internals->inq[i]);
3744                 if (unlikely(ret != 0)) {
3745                         DPAA_SEC_ERR("sec qman_create_fq failed");
3746                         goto init_error;
3747                 }
3748         }
3749
3750         dpaa_sec_get_devargs(cryptodev->device->devargs, DRIVER_DUMP_MODE);
3751
3752         RTE_LOG(INFO, PMD, "%s cryptodev init\n", cryptodev->data->name);
3753         return 0;
3754
3755 init_error:
3756         DPAA_SEC_ERR("driver %s: create failed\n", cryptodev->data->name);
3757
3758         rte_free(cryptodev->security_ctx);
3759         return -EFAULT;
3760 }
3761
3762 static int
3763 cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv __rte_unused,
3764                                 struct rte_dpaa_device *dpaa_dev)
3765 {
3766         struct rte_cryptodev *cryptodev;
3767         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
3768
3769         int retval;
3770
3771         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3772                 return 0;
3773
3774         snprintf(cryptodev_name, sizeof(cryptodev_name), "%s", dpaa_dev->name);
3775
3776         cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id());
3777         if (cryptodev == NULL)
3778                 return -ENOMEM;
3779
3780         cryptodev->data->dev_private = rte_zmalloc_socket(
3781                                 "cryptodev private structure",
3782                                 sizeof(struct dpaa_sec_dev_private),
3783                                 RTE_CACHE_LINE_SIZE,
3784                                 rte_socket_id());
3785
3786         if (cryptodev->data->dev_private == NULL)
3787                 rte_panic("Cannot allocate memzone for private "
3788                                 "device data");
3789
3790         dpaa_dev->crypto_dev = cryptodev;
3791         cryptodev->device = &dpaa_dev->device;
3792
3793         /* init user callbacks */
3794         TAILQ_INIT(&(cryptodev->link_intr_cbs));
3795
3796         /* if sec device version is not configured */
3797         if (!rta_get_sec_era()) {
3798                 const struct device_node *caam_node;
3799
3800                 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
3801                         const uint32_t *prop = of_get_property(caam_node,
3802                                         "fsl,sec-era",
3803                                         NULL);
3804                         if (prop) {
3805                                 rta_set_sec_era(
3806                                         INTL_SEC_ERA(rte_cpu_to_be_32(*prop)));
3807                                 break;
3808                         }
3809                 }
3810         }
3811
3812         if (unlikely(!DPAA_PER_LCORE_PORTAL)) {
3813                 retval = rte_dpaa_portal_init((void *)1);
3814                 if (retval) {
3815                         DPAA_SEC_ERR("Unable to initialize portal");
3816                         goto out;
3817                 }
3818         }
3819
3820         /* Invoke PMD device initialization function */
3821         retval = dpaa_sec_dev_init(cryptodev);
3822         if (retval == 0) {
3823                 rte_cryptodev_pmd_probing_finish(cryptodev);
3824                 return 0;
3825         }
3826
3827         retval = -ENXIO;
3828 out:
3829         /* In case of error, cleanup is done */
3830         rte_free(cryptodev->data->dev_private);
3831
3832         rte_cryptodev_pmd_release_device(cryptodev);
3833
3834         return retval;
3835 }
3836
3837 static int
3838 cryptodev_dpaa_sec_remove(struct rte_dpaa_device *dpaa_dev)
3839 {
3840         struct rte_cryptodev *cryptodev;
3841         int ret;
3842
3843         cryptodev = dpaa_dev->crypto_dev;
3844         if (cryptodev == NULL)
3845                 return -ENODEV;
3846
3847         ret = dpaa_sec_uninit(cryptodev);
3848         if (ret)
3849                 return ret;
3850
3851         return rte_cryptodev_pmd_destroy(cryptodev);
3852 }
3853
3854 static struct rte_dpaa_driver rte_dpaa_sec_driver = {
3855         .drv_type = FSL_DPAA_CRYPTO,
3856         .driver = {
3857                 .name = "DPAA SEC PMD"
3858         },
3859         .probe = cryptodev_dpaa_sec_probe,
3860         .remove = cryptodev_dpaa_sec_remove,
3861 };
3862
3863 static struct cryptodev_driver dpaa_sec_crypto_drv;
3864
3865 RTE_PMD_REGISTER_DPAA(CRYPTODEV_NAME_DPAA_SEC_PMD, rte_dpaa_sec_driver);
3866 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa_sec_crypto_drv, rte_dpaa_sec_driver.driver,
3867                 dpaa_cryptodev_driver_id);
3868 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_DPAA_SEC_PMD,
3869                 DRIVER_DUMP_MODE "=<int>");
3870 RTE_LOG_REGISTER(dpaa_logtype_sec, pmd.crypto.dpaa, NOTICE);