drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2016-2021 NXP
   5  *
   6  */
   7
   8 #include <time.h>
   9 #include <net/if.h>
  10 #include <unistd.h>
  11
  12 #include <rte_ip.h>
  13 #include <rte_mbuf.h>
  14 #include <rte_cryptodev.h>
  15 #include <rte_malloc.h>
  16 #include <rte_memcpy.h>
  17 #include <rte_string_fns.h>
  18 #include <rte_cycles.h>
  19 #include <rte_kvargs.h>
  20 #include <rte_dev.h>
  21 #include <cryptodev_pmd.h>
  22 #include <rte_common.h>
  23 #include <rte_fslmc.h>
  24 #include <fslmc_vfio.h>
  25 #include <dpaa2_hw_pvt.h>
  26 #include <dpaa2_hw_dpio.h>
  27 #include <dpaa2_hw_mempool.h>
  28 #include <fsl_dpopr.h>
  29 #include <fsl_dpseci.h>
  30 #include <fsl_mc_sys.h>
  31 #include <rte_hexdump.h>
  32
  33 #include "dpaa2_sec_priv.h"
  34 #include "dpaa2_sec_event.h"
  35 #include "dpaa2_sec_logs.h"
  36
  37 /* RTA header files */
  38 #include <desc/ipsec.h>
  39 #include <desc/pdcp.h>
  40 #include <desc/sdap.h>
  41 #include <desc/algo.h>
  42
  43 /* Minimum job descriptor consists of a oneword job descriptor HEADER and
  44  * a pointer to the shared descriptor
  45  */
  46 #define MIN_JOB_DESC_SIZE       (CAAM_CMD_SZ + CAAM_PTR_SZ)
  47 #define FSL_VENDOR_ID           0x1957
  48 #define FSL_DEVICE_ID           0x410
  49 #define FSL_SUBSYSTEM_SEC       1
  50 #define FSL_MC_DPSECI_DEVID     3
  51
  52 #define NO_PREFETCH 0
  53
  54 #define DRIVER_DUMP_MODE "drv_dump_mode"
  55
  56 /* DPAA2_SEC_DP_DUMP levels */
  57 enum dpaa2_sec_dump_levels {
  58         DPAA2_SEC_DP_NO_DUMP,
  59         DPAA2_SEC_DP_ERR_DUMP,
  60         DPAA2_SEC_DP_FULL_DUMP
  61 };
  62
  63 uint8_t cryptodev_driver_id;
  64 uint8_t dpaa2_sec_dp_dump = DPAA2_SEC_DP_ERR_DUMP;
  65
  66 #ifdef RTE_LIB_SECURITY
  67 static inline int
  68 build_proto_compound_sg_fd(dpaa2_sec_session *sess,
  69                            struct rte_crypto_op *op,
  70                            struct qbman_fd *fd, uint16_t bpid)
  71 {
  72         struct rte_crypto_sym_op *sym_op = op->sym;
  73         struct ctxt_priv *priv = sess->ctxt;
  74         struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
  75         struct sec_flow_context *flc;
  76         struct rte_mbuf *mbuf;
  77         uint32_t in_len = 0, out_len = 0;
  78
  79         if (sym_op->m_dst)
  80                 mbuf = sym_op->m_dst;
  81         else
  82                 mbuf = sym_op->m_src;
  83
  84         /* first FLE entry used to store mbuf and session ctxt */
  85         fle = (struct qbman_fle *)rte_malloc(NULL,
  86                         FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
  87                         RTE_CACHE_LINE_SIZE);
  88         if (unlikely(!fle)) {
  89                 DPAA2_SEC_DP_ERR("Proto:SG: Memory alloc failed for SGE");
  90                 return -ENOMEM;
  91         }
  92         memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
  93         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
  94         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
  95
  96         /* Save the shared descriptor */
  97         flc = &priv->flc_desc[0].flc;
  98
  99         op_fle = fle + 1;
 100         ip_fle = fle + 2;
 101         sge = fle + 3;
 102
 103         if (likely(bpid < MAX_BPID)) {
 104                 DPAA2_SET_FD_BPID(fd, bpid);
 105                 DPAA2_SET_FLE_BPID(op_fle, bpid);
 106                 DPAA2_SET_FLE_BPID(ip_fle, bpid);
 107         } else {
 108                 DPAA2_SET_FD_IVP(fd);
 109                 DPAA2_SET_FLE_IVP(op_fle);
 110                 DPAA2_SET_FLE_IVP(ip_fle);
 111         }
 112
 113         /* Configure FD as a FRAME LIST */
 114         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
 115         DPAA2_SET_FD_COMPOUND_FMT(fd);
 116         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 117
 118         /* Configure Output FLE with Scatter/Gather Entry */
 119         DPAA2_SET_FLE_SG_EXT(op_fle);
 120         DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
 121
 122         /* Configure Output SGE for Encap/Decap */
 123         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 124         DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 125         /* o/p segs */
 126         while (mbuf->next) {
 127                 sge->length = mbuf->data_len;
 128                 out_len += sge->length;
 129                 sge++;
 130                 mbuf = mbuf->next;
 131                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 132                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 133         }
 134         /* using buf_len for last buf - so that extra data can be added */
 135         sge->length = mbuf->buf_len - mbuf->data_off;
 136         out_len += sge->length;
 137
 138         DPAA2_SET_FLE_FIN(sge);
 139         op_fle->length = out_len;
 140
 141         sge++;
 142         mbuf = sym_op->m_src;
 143
 144         /* Configure Input FLE with Scatter/Gather Entry */
 145         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
 146         DPAA2_SET_FLE_SG_EXT(ip_fle);
 147         DPAA2_SET_FLE_FIN(ip_fle);
 148
 149         /* Configure input SGE for Encap/Decap */
 150         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 151         DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 152         sge->length = mbuf->data_len;
 153         in_len += sge->length;
 154
 155         mbuf = mbuf->next;
 156         /* i/p segs */
 157         while (mbuf) {
 158                 sge++;
 159                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 160                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 161                 sge->length = mbuf->data_len;
 162                 in_len += sge->length;
 163                 mbuf = mbuf->next;
 164         }
 165         ip_fle->length = in_len;
 166         DPAA2_SET_FLE_FIN(sge);
 167
 168         /* In case of PDCP, per packet HFN is stored in
 169          * mbuf priv after sym_op.
 170          */
 171         if (sess->ctxt_type == DPAA2_SEC_PDCP && sess->pdcp.hfn_ovd) {
 172                 uint32_t hfn_ovd = *(uint32_t *)((uint8_t *)op +
 173                                         sess->pdcp.hfn_ovd_offset);
 174                 /*enable HFN override override */
 175                 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, hfn_ovd);
 176                 DPAA2_SET_FLE_INTERNAL_JD(op_fle, hfn_ovd);
 177                 DPAA2_SET_FD_INTERNAL_JD(fd, hfn_ovd);
 178         }
 179         DPAA2_SET_FD_LEN(fd, ip_fle->length);
 180
 181         return 0;
 182 }
 183
 184 static inline int
 185 build_proto_compound_fd(dpaa2_sec_session *sess,
 186                struct rte_crypto_op *op,
 187                struct qbman_fd *fd, uint16_t bpid)
 188 {
 189         struct rte_crypto_sym_op *sym_op = op->sym;
 190         struct ctxt_priv *priv = sess->ctxt;
 191         struct qbman_fle *fle, *ip_fle, *op_fle;
 192         struct sec_flow_context *flc;
 193         struct rte_mbuf *src_mbuf = sym_op->m_src;
 194         struct rte_mbuf *dst_mbuf = sym_op->m_dst;
 195         int retval;
 196
 197         if (!dst_mbuf)
 198                 dst_mbuf = src_mbuf;
 199
 200         /* Save the shared descriptor */
 201         flc = &priv->flc_desc[0].flc;
 202
 203         /* we are using the first FLE entry to store Mbuf */
 204         retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
 205         if (retval) {
 206                 DPAA2_SEC_DP_ERR("Memory alloc failed");
 207                 return -ENOMEM;
 208         }
 209         memset(fle, 0, FLE_POOL_BUF_SIZE);
 210         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 211         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
 212
 213         op_fle = fle + 1;
 214         ip_fle = fle + 2;
 215
 216         if (likely(bpid < MAX_BPID)) {
 217                 DPAA2_SET_FD_BPID(fd, bpid);
 218                 DPAA2_SET_FLE_BPID(op_fle, bpid);
 219                 DPAA2_SET_FLE_BPID(ip_fle, bpid);
 220         } else {
 221                 DPAA2_SET_FD_IVP(fd);
 222                 DPAA2_SET_FLE_IVP(op_fle);
 223                 DPAA2_SET_FLE_IVP(ip_fle);
 224         }
 225
 226         /* Configure FD as a FRAME LIST */
 227         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
 228         DPAA2_SET_FD_COMPOUND_FMT(fd);
 229         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 230
 231         /* Configure Output FLE with dst mbuf data  */
 232         DPAA2_SET_FLE_ADDR(op_fle, DPAA2_MBUF_VADDR_TO_IOVA(dst_mbuf));
 233         DPAA2_SET_FLE_OFFSET(op_fle, dst_mbuf->data_off);
 234         DPAA2_SET_FLE_LEN(op_fle, dst_mbuf->buf_len);
 235
 236         /* Configure Input FLE with src mbuf data */
 237         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_MBUF_VADDR_TO_IOVA(src_mbuf));
 238         DPAA2_SET_FLE_OFFSET(ip_fle, src_mbuf->data_off);
 239         DPAA2_SET_FLE_LEN(ip_fle, src_mbuf->pkt_len);
 240
 241         DPAA2_SET_FD_LEN(fd, ip_fle->length);
 242         DPAA2_SET_FLE_FIN(ip_fle);
 243
 244         /* In case of PDCP, per packet HFN is stored in
 245          * mbuf priv after sym_op.
 246          */
 247         if (sess->ctxt_type == DPAA2_SEC_PDCP && sess->pdcp.hfn_ovd) {
 248                 uint32_t hfn_ovd = *(uint32_t *)((uint8_t *)op +
 249                                         sess->pdcp.hfn_ovd_offset);
 250                 /*enable HFN override override */
 251                 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, hfn_ovd);
 252                 DPAA2_SET_FLE_INTERNAL_JD(op_fle, hfn_ovd);
 253                 DPAA2_SET_FD_INTERNAL_JD(fd, hfn_ovd);
 254         }
 255
 256         return 0;
 257
 258 }
 259
 260 static inline int
 261 build_proto_fd(dpaa2_sec_session *sess,
 262                struct rte_crypto_op *op,
 263                struct qbman_fd *fd, uint16_t bpid)
 264 {
 265         struct rte_crypto_sym_op *sym_op = op->sym;
 266         if (sym_op->m_dst)
 267                 return build_proto_compound_fd(sess, op, fd, bpid);
 268
 269         struct ctxt_priv *priv = sess->ctxt;
 270         struct sec_flow_context *flc;
 271         struct rte_mbuf *mbuf = sym_op->m_src;
 272
 273         if (likely(bpid < MAX_BPID))
 274                 DPAA2_SET_FD_BPID(fd, bpid);
 275         else
 276                 DPAA2_SET_FD_IVP(fd);
 277
 278         /* Save the shared descriptor */
 279         flc = &priv->flc_desc[0].flc;
 280
 281         DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
 282         DPAA2_SET_FD_OFFSET(fd, sym_op->m_src->data_off);
 283         DPAA2_SET_FD_LEN(fd, sym_op->m_src->pkt_len);
 284         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 285
 286         /* save physical address of mbuf */
 287         op->sym->aead.digest.phys_addr = mbuf->buf_iova;
 288         mbuf->buf_iova = (size_t)op;
 289
 290         return 0;
 291 }
 292 #endif
 293
 294 static inline int
 295 build_authenc_gcm_sg_fd(dpaa2_sec_session *sess,
 296                  struct rte_crypto_op *op,
 297                  struct qbman_fd *fd, __rte_unused uint16_t bpid)
 298 {
 299         struct rte_crypto_sym_op *sym_op = op->sym;
 300         struct ctxt_priv *priv = sess->ctxt;
 301         struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
 302         struct sec_flow_context *flc;
 303         uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len;
 304         int icv_len = sess->digest_length;
 305         uint8_t *old_icv;
 306         struct rte_mbuf *mbuf;
 307         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 308                         sess->iv.offset);
 309
 310         if (sym_op->m_dst)
 311                 mbuf = sym_op->m_dst;
 312         else
 313                 mbuf = sym_op->m_src;
 314
 315         /* first FLE entry used to store mbuf and session ctxt */
 316         fle = (struct qbman_fle *)rte_malloc(NULL,
 317                         FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
 318                         RTE_CACHE_LINE_SIZE);
 319         if (unlikely(!fle)) {
 320                 DPAA2_SEC_ERR("GCM SG: Memory alloc failed for SGE");
 321                 return -ENOMEM;
 322         }
 323         memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
 324         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 325         DPAA2_FLE_SAVE_CTXT(fle, (size_t)priv);
 326
 327         op_fle = fle + 1;
 328         ip_fle = fle + 2;
 329         sge = fle + 3;
 330
 331         /* Save the shared descriptor */
 332         flc = &priv->flc_desc[0].flc;
 333
 334         /* Configure FD as a FRAME LIST */
 335         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
 336         DPAA2_SET_FD_COMPOUND_FMT(fd);
 337         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 338
 339         DPAA2_SEC_DP_DEBUG("GCM SG: auth_off: 0x%x/length %d, digest-len=%d\n"
 340                    "iv-len=%d data_off: 0x%x\n",
 341                    sym_op->aead.data.offset,
 342                    sym_op->aead.data.length,
 343                    sess->digest_length,
 344                    sess->iv.length,
 345                    sym_op->m_src->data_off);
 346
 347         /* Configure Output FLE with Scatter/Gather Entry */
 348         DPAA2_SET_FLE_SG_EXT(op_fle);
 349         DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
 350
 351         if (auth_only_len)
 352                 DPAA2_SET_FLE_INTERNAL_JD(op_fle, auth_only_len);
 353
 354         op_fle->length = (sess->dir == DIR_ENC) ?
 355                         (sym_op->aead.data.length + icv_len) :
 356                         sym_op->aead.data.length;
 357
 358         /* Configure Output SGE for Encap/Decap */
 359         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 360         DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off + sym_op->aead.data.offset);
 361         sge->length = mbuf->data_len - sym_op->aead.data.offset;
 362
 363         mbuf = mbuf->next;
 364         /* o/p segs */
 365         while (mbuf) {
 366                 sge++;
 367                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 368                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 369                 sge->length = mbuf->data_len;
 370                 mbuf = mbuf->next;
 371         }
 372         sge->length -= icv_len;
 373
 374         if (sess->dir == DIR_ENC) {
 375                 sge++;
 376                 DPAA2_SET_FLE_ADDR(sge,
 377                                 DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data));
 378                 sge->length = icv_len;
 379         }
 380         DPAA2_SET_FLE_FIN(sge);
 381
 382         sge++;
 383         mbuf = sym_op->m_src;
 384
 385         /* Configure Input FLE with Scatter/Gather Entry */
 386         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
 387         DPAA2_SET_FLE_SG_EXT(ip_fle);
 388         DPAA2_SET_FLE_FIN(ip_fle);
 389         ip_fle->length = (sess->dir == DIR_ENC) ?
 390                 (sym_op->aead.data.length + sess->iv.length + auth_only_len) :
 391                 (sym_op->aead.data.length + sess->iv.length + auth_only_len +
 392                  icv_len);
 393
 394         /* Configure Input SGE for Encap/Decap */
 395         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr));
 396         sge->length = sess->iv.length;
 397
 398         sge++;
 399         if (auth_only_len) {
 400                 DPAA2_SET_FLE_ADDR(sge,
 401                                 DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data));
 402                 sge->length = auth_only_len;
 403                 sge++;
 404         }
 405
 406         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 407         DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset +
 408                                 mbuf->data_off);
 409         sge->length = mbuf->data_len - sym_op->aead.data.offset;
 410
 411         mbuf = mbuf->next;
 412         /* i/p segs */
 413         while (mbuf) {
 414                 sge++;
 415                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 416                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 417                 sge->length = mbuf->data_len;
 418                 mbuf = mbuf->next;
 419         }
 420
 421         if (sess->dir == DIR_DEC) {
 422                 sge++;
 423                 old_icv = (uint8_t *)(sge + 1);
 424                 memcpy(old_icv, sym_op->aead.digest.data, icv_len);
 425                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
 426                 sge->length = icv_len;
 427         }
 428
 429         DPAA2_SET_FLE_FIN(sge);
 430         if (auth_only_len) {
 431                 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, auth_only_len);
 432                 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
 433         }
 434         DPAA2_SET_FD_LEN(fd, ip_fle->length);
 435
 436         return 0;
 437 }
 438
 439 static inline int
 440 build_authenc_gcm_fd(dpaa2_sec_session *sess,
 441                      struct rte_crypto_op *op,
 442                      struct qbman_fd *fd, uint16_t bpid)
 443 {
 444         struct rte_crypto_sym_op *sym_op = op->sym;
 445         struct ctxt_priv *priv = sess->ctxt;
 446         struct qbman_fle *fle, *sge;
 447         struct sec_flow_context *flc;
 448         uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len;
 449         int icv_len = sess->digest_length, retval;
 450         uint8_t *old_icv;
 451         struct rte_mbuf *dst;
 452         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 453                         sess->iv.offset);
 454
 455         if (sym_op->m_dst)
 456                 dst = sym_op->m_dst;
 457         else
 458                 dst = sym_op->m_src;
 459
 460         /* TODO we are using the first FLE entry to store Mbuf and session ctxt.
 461          * Currently we donot know which FLE has the mbuf stored.
 462          * So while retreiving we can go back 1 FLE from the FD -ADDR
 463          * to get the MBUF Addr from the previous FLE.
 464          * We can have a better approach to use the inline Mbuf
 465          */
 466         retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
 467         if (retval) {
 468                 DPAA2_SEC_ERR("GCM: Memory alloc failed for SGE");
 469                 return -ENOMEM;
 470         }
 471         memset(fle, 0, FLE_POOL_BUF_SIZE);
 472         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 473         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
 474         fle = fle + 1;
 475         sge = fle + 2;
 476         if (likely(bpid < MAX_BPID)) {
 477                 DPAA2_SET_FD_BPID(fd, bpid);
 478                 DPAA2_SET_FLE_BPID(fle, bpid);
 479                 DPAA2_SET_FLE_BPID(fle + 1, bpid);
 480                 DPAA2_SET_FLE_BPID(sge, bpid);
 481                 DPAA2_SET_FLE_BPID(sge + 1, bpid);
 482                 DPAA2_SET_FLE_BPID(sge + 2, bpid);
 483                 DPAA2_SET_FLE_BPID(sge + 3, bpid);
 484         } else {
 485                 DPAA2_SET_FD_IVP(fd);
 486                 DPAA2_SET_FLE_IVP(fle);
 487                 DPAA2_SET_FLE_IVP((fle + 1));
 488                 DPAA2_SET_FLE_IVP(sge);
 489                 DPAA2_SET_FLE_IVP((sge + 1));
 490                 DPAA2_SET_FLE_IVP((sge + 2));
 491                 DPAA2_SET_FLE_IVP((sge + 3));
 492         }
 493
 494         /* Save the shared descriptor */
 495         flc = &priv->flc_desc[0].flc;
 496         /* Configure FD as a FRAME LIST */
 497         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
 498         DPAA2_SET_FD_COMPOUND_FMT(fd);
 499         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 500
 501         DPAA2_SEC_DP_DEBUG("GCM: auth_off: 0x%x/length %d, digest-len=%d\n"
 502                    "iv-len=%d data_off: 0x%x\n",
 503                    sym_op->aead.data.offset,
 504                    sym_op->aead.data.length,
 505                    sess->digest_length,
 506                    sess->iv.length,
 507                    sym_op->m_src->data_off);
 508
 509         /* Configure Output FLE with Scatter/Gather Entry */
 510         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
 511         if (auth_only_len)
 512                 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
 513         fle->length = (sess->dir == DIR_ENC) ?
 514                         (sym_op->aead.data.length + icv_len) :
 515                         sym_op->aead.data.length;
 516
 517         DPAA2_SET_FLE_SG_EXT(fle);
 518
 519         /* Configure Output SGE for Encap/Decap */
 520         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst));
 521         DPAA2_SET_FLE_OFFSET(sge, dst->data_off + sym_op->aead.data.offset);
 522         sge->length = sym_op->aead.data.length;
 523
 524         if (sess->dir == DIR_ENC) {
 525                 sge++;
 526                 DPAA2_SET_FLE_ADDR(sge,
 527                                 DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data));
 528                 sge->length = sess->digest_length;
 529         }
 530         DPAA2_SET_FLE_FIN(sge);
 531
 532         sge++;
 533         fle++;
 534
 535         /* Configure Input FLE with Scatter/Gather Entry */
 536         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
 537         DPAA2_SET_FLE_SG_EXT(fle);
 538         DPAA2_SET_FLE_FIN(fle);
 539         fle->length = (sess->dir == DIR_ENC) ?
 540                 (sym_op->aead.data.length + sess->iv.length + auth_only_len) :
 541                 (sym_op->aead.data.length + sess->iv.length + auth_only_len +
 542                  sess->digest_length);
 543
 544         /* Configure Input SGE for Encap/Decap */
 545         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr));
 546         sge->length = sess->iv.length;
 547         sge++;
 548         if (auth_only_len) {
 549                 DPAA2_SET_FLE_ADDR(sge,
 550                                 DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data));
 551                 sge->length = auth_only_len;
 552                 DPAA2_SET_FLE_BPID(sge, bpid);
 553                 sge++;
 554         }
 555
 556         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
 557         DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset +
 558                                 sym_op->m_src->data_off);
 559         sge->length = sym_op->aead.data.length;
 560         if (sess->dir == DIR_DEC) {
 561                 sge++;
 562                 old_icv = (uint8_t *)(sge + 1);
 563                 memcpy(old_icv, sym_op->aead.digest.data,
 564                        sess->digest_length);
 565                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
 566                 sge->length = sess->digest_length;
 567         }
 568         DPAA2_SET_FLE_FIN(sge);
 569
 570         if (auth_only_len) {
 571                 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
 572                 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
 573         }
 574
 575         DPAA2_SET_FD_LEN(fd, fle->length);
 576         return 0;
 577 }
 578
 579 static inline int
 580 build_authenc_sg_fd(dpaa2_sec_session *sess,
 581                  struct rte_crypto_op *op,
 582                  struct qbman_fd *fd, __rte_unused uint16_t bpid)
 583 {
 584         struct rte_crypto_sym_op *sym_op = op->sym;
 585         struct ctxt_priv *priv = sess->ctxt;
 586         struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
 587         struct sec_flow_context *flc;
 588         uint16_t auth_hdr_len = sym_op->cipher.data.offset -
 589                                 sym_op->auth.data.offset;
 590         uint16_t auth_tail_len = sym_op->auth.data.length -
 591                                 sym_op->cipher.data.length - auth_hdr_len;
 592         uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
 593         int icv_len = sess->digest_length;
 594         uint8_t *old_icv;
 595         struct rte_mbuf *mbuf;
 596         uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 597                         sess->iv.offset);
 598
 599         if (sym_op->m_dst)
 600                 mbuf = sym_op->m_dst;
 601         else
 602                 mbuf = sym_op->m_src;
 603
 604         /* first FLE entry used to store mbuf and session ctxt */
 605         fle = (struct qbman_fle *)rte_malloc(NULL,
 606                         FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
 607                         RTE_CACHE_LINE_SIZE);
 608         if (unlikely(!fle)) {
 609                 DPAA2_SEC_ERR("AUTHENC SG: Memory alloc failed for SGE");
 610                 return -ENOMEM;
 611         }
 612         memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
 613         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 614         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
 615
 616         op_fle = fle + 1;
 617         ip_fle = fle + 2;
 618         sge = fle + 3;
 619
 620         /* Save the shared descriptor */
 621         flc = &priv->flc_desc[0].flc;
 622
 623         /* Configure FD as a FRAME LIST */
 624         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
 625         DPAA2_SET_FD_COMPOUND_FMT(fd);
 626         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 627
 628         DPAA2_SEC_DP_DEBUG(
 629                 "AUTHENC SG: auth_off: 0x%x/length %d, digest-len=%d\n"
 630                 "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n",
 631                 sym_op->auth.data.offset,
 632                 sym_op->auth.data.length,
 633                 sess->digest_length,
 634                 sym_op->cipher.data.offset,
 635                 sym_op->cipher.data.length,
 636                 sess->iv.length,
 637                 sym_op->m_src->data_off);
 638
 639         /* Configure Output FLE with Scatter/Gather Entry */
 640         DPAA2_SET_FLE_SG_EXT(op_fle);
 641         DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
 642
 643         if (auth_only_len)
 644                 DPAA2_SET_FLE_INTERNAL_JD(op_fle, auth_only_len);
 645
 646         op_fle->length = (sess->dir == DIR_ENC) ?
 647                         (sym_op->cipher.data.length + icv_len) :
 648                         sym_op->cipher.data.length;
 649
 650         /* Configure Output SGE for Encap/Decap */
 651         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 652         DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off + sym_op->auth.data.offset);
 653         sge->length = mbuf->data_len - sym_op->auth.data.offset;
 654
 655         mbuf = mbuf->next;
 656         /* o/p segs */
 657         while (mbuf) {
 658                 sge++;
 659                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 660                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 661                 sge->length = mbuf->data_len;
 662                 mbuf = mbuf->next;
 663         }
 664         sge->length -= icv_len;
 665
 666         if (sess->dir == DIR_ENC) {
 667                 sge++;
 668                 DPAA2_SET_FLE_ADDR(sge,
 669                                 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
 670                 sge->length = icv_len;
 671         }
 672         DPAA2_SET_FLE_FIN(sge);
 673
 674         sge++;
 675         mbuf = sym_op->m_src;
 676
 677         /* Configure Input FLE with Scatter/Gather Entry */
 678         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
 679         DPAA2_SET_FLE_SG_EXT(ip_fle);
 680         DPAA2_SET_FLE_FIN(ip_fle);
 681         ip_fle->length = (sess->dir == DIR_ENC) ?
 682                         (sym_op->auth.data.length + sess->iv.length) :
 683                         (sym_op->auth.data.length + sess->iv.length +
 684                          icv_len);
 685
 686         /* Configure Input SGE for Encap/Decap */
 687         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
 688         sge->length = sess->iv.length;
 689
 690         sge++;
 691         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 692         DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset +
 693                                 mbuf->data_off);
 694         sge->length = mbuf->data_len - sym_op->auth.data.offset;
 695
 696         mbuf = mbuf->next;
 697         /* i/p segs */
 698         while (mbuf) {
 699                 sge++;
 700                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 701                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 702                 sge->length = mbuf->data_len;
 703                 mbuf = mbuf->next;
 704         }
 705         sge->length -= icv_len;
 706
 707         if (sess->dir == DIR_DEC) {
 708                 sge++;
 709                 old_icv = (uint8_t *)(sge + 1);
 710                 memcpy(old_icv, sym_op->auth.digest.data,
 711                        icv_len);
 712                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
 713                 sge->length = icv_len;
 714         }
 715
 716         DPAA2_SET_FLE_FIN(sge);
 717         if (auth_only_len) {
 718                 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, auth_only_len);
 719                 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
 720         }
 721         DPAA2_SET_FD_LEN(fd, ip_fle->length);
 722
 723         return 0;
 724 }
 725
 726 static inline int
 727 build_authenc_fd(dpaa2_sec_session *sess,
 728                  struct rte_crypto_op *op,
 729                  struct qbman_fd *fd, uint16_t bpid)
 730 {
 731         struct rte_crypto_sym_op *sym_op = op->sym;
 732         struct ctxt_priv *priv = sess->ctxt;
 733         struct qbman_fle *fle, *sge;
 734         struct sec_flow_context *flc;
 735         uint16_t auth_hdr_len = sym_op->cipher.data.offset -
 736                                 sym_op->auth.data.offset;
 737         uint16_t auth_tail_len = sym_op->auth.data.length -
 738                                 sym_op->cipher.data.length - auth_hdr_len;
 739         uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
 740
 741         int icv_len = sess->digest_length, retval;
 742         uint8_t *old_icv;
 743         uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 744                         sess->iv.offset);
 745         struct rte_mbuf *dst;
 746
 747         if (sym_op->m_dst)
 748                 dst = sym_op->m_dst;
 749         else
 750                 dst = sym_op->m_src;
 751
 752         /* we are using the first FLE entry to store Mbuf.
 753          * Currently we donot know which FLE has the mbuf stored.
 754          * So while retreiving we can go back 1 FLE from the FD -ADDR
 755          * to get the MBUF Addr from the previous FLE.
 756          * We can have a better approach to use the inline Mbuf
 757          */
 758         retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
 759         if (retval) {
 760                 DPAA2_SEC_ERR("Memory alloc failed for SGE");
 761                 return -ENOMEM;
 762         }
 763         memset(fle, 0, FLE_POOL_BUF_SIZE);
 764         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 765         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
 766         fle = fle + 1;
 767         sge = fle + 2;
 768         if (likely(bpid < MAX_BPID)) {
 769                 DPAA2_SET_FD_BPID(fd, bpid);
 770                 DPAA2_SET_FLE_BPID(fle, bpid);
 771                 DPAA2_SET_FLE_BPID(fle + 1, bpid);
 772                 DPAA2_SET_FLE_BPID(sge, bpid);
 773                 DPAA2_SET_FLE_BPID(sge + 1, bpid);
 774                 DPAA2_SET_FLE_BPID(sge + 2, bpid);
 775                 DPAA2_SET_FLE_BPID(sge + 3, bpid);
 776         } else {
 777                 DPAA2_SET_FD_IVP(fd);
 778                 DPAA2_SET_FLE_IVP(fle);
 779                 DPAA2_SET_FLE_IVP((fle + 1));
 780                 DPAA2_SET_FLE_IVP(sge);
 781                 DPAA2_SET_FLE_IVP((sge + 1));
 782                 DPAA2_SET_FLE_IVP((sge + 2));
 783                 DPAA2_SET_FLE_IVP((sge + 3));
 784         }
 785
 786         /* Save the shared descriptor */
 787         flc = &priv->flc_desc[0].flc;
 788         /* Configure FD as a FRAME LIST */
 789         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
 790         DPAA2_SET_FD_COMPOUND_FMT(fd);
 791         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 792
 793         DPAA2_SEC_DP_DEBUG(
 794                 "AUTHENC: auth_off: 0x%x/length %d, digest-len=%d\n"
 795                 "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n",
 796                 sym_op->auth.data.offset,
 797                 sym_op->auth.data.length,
 798                 sess->digest_length,
 799                 sym_op->cipher.data.offset,
 800                 sym_op->cipher.data.length,
 801                 sess->iv.length,
 802                 sym_op->m_src->data_off);
 803
 804         /* Configure Output FLE with Scatter/Gather Entry */
 805         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
 806         if (auth_only_len)
 807                 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
 808         fle->length = (sess->dir == DIR_ENC) ?
 809                         (sym_op->cipher.data.length + icv_len) :
 810                         sym_op->cipher.data.length;
 811
 812         DPAA2_SET_FLE_SG_EXT(fle);
 813
 814         /* Configure Output SGE for Encap/Decap */
 815         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst));
 816         DPAA2_SET_FLE_OFFSET(sge, sym_op->cipher.data.offset +
 817                                 dst->data_off);
 818         sge->length = sym_op->cipher.data.length;
 819
 820         if (sess->dir == DIR_ENC) {
 821                 sge++;
 822                 DPAA2_SET_FLE_ADDR(sge,
 823                                 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
 824                 sge->length = sess->digest_length;
 825                 DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length +
 826                                         sess->iv.length));
 827         }
 828         DPAA2_SET_FLE_FIN(sge);
 829
 830         sge++;
 831         fle++;
 832
 833         /* Configure Input FLE with Scatter/Gather Entry */
 834         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
 835         DPAA2_SET_FLE_SG_EXT(fle);
 836         DPAA2_SET_FLE_FIN(fle);
 837         fle->length = (sess->dir == DIR_ENC) ?
 838                         (sym_op->auth.data.length + sess->iv.length) :
 839                         (sym_op->auth.data.length + sess->iv.length +
 840                          sess->digest_length);
 841
 842         /* Configure Input SGE for Encap/Decap */
 843         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
 844         sge->length = sess->iv.length;
 845         sge++;
 846
 847         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
 848         DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset +
 849                                 sym_op->m_src->data_off);
 850         sge->length = sym_op->auth.data.length;
 851         if (sess->dir == DIR_DEC) {
 852                 sge++;
 853                 old_icv = (uint8_t *)(sge + 1);
 854                 memcpy(old_icv, sym_op->auth.digest.data,
 855                        sess->digest_length);
 856                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
 857                 sge->length = sess->digest_length;
 858                 DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length +
 859                                  sess->digest_length +
 860                                  sess->iv.length));
 861         }
 862         DPAA2_SET_FLE_FIN(sge);
 863         if (auth_only_len) {
 864                 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
 865                 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
 866         }
 867         return 0;
 868 }
 869
 870 static inline int build_auth_sg_fd(
 871                 dpaa2_sec_session *sess,
 872                 struct rte_crypto_op *op,
 873                 struct qbman_fd *fd,
 874                 __rte_unused uint16_t bpid)
 875 {
 876         struct rte_crypto_sym_op *sym_op = op->sym;
 877         struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
 878         struct sec_flow_context *flc;
 879         struct ctxt_priv *priv = sess->ctxt;
 880         int data_len, data_offset;
 881         uint8_t *old_digest;
 882         struct rte_mbuf *mbuf;
 883
 884         data_len = sym_op->auth.data.length;
 885         data_offset = sym_op->auth.data.offset;
 886
 887         if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
 888             sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
 889                 if ((data_len & 7) || (data_offset & 7)) {
 890                         DPAA2_SEC_ERR("AUTH: len/offset must be full bytes");
 891                         return -ENOTSUP;
 892                 }
 893
 894                 data_len = data_len >> 3;
 895                 data_offset = data_offset >> 3;
 896         }
 897
 898         mbuf = sym_op->m_src;
 899         fle = (struct qbman_fle *)rte_malloc(NULL,
 900                         FLE_SG_MEM_SIZE(mbuf->nb_segs),
 901                         RTE_CACHE_LINE_SIZE);
 902         if (unlikely(!fle)) {
 903                 DPAA2_SEC_ERR("AUTH SG: Memory alloc failed for SGE");
 904                 return -ENOMEM;
 905         }
 906         memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs));
 907         /* first FLE entry used to store mbuf and session ctxt */
 908         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 909         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
 910         op_fle = fle + 1;
 911         ip_fle = fle + 2;
 912         sge = fle + 3;
 913
 914         flc = &priv->flc_desc[DESC_INITFINAL].flc;
 915         /* sg FD */
 916         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 917         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
 918         DPAA2_SET_FD_COMPOUND_FMT(fd);
 919
 920         /* o/p fle */
 921         DPAA2_SET_FLE_ADDR(op_fle,
 922                                 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
 923         op_fle->length = sess->digest_length;
 924
 925         /* i/p fle */
 926         DPAA2_SET_FLE_SG_EXT(ip_fle);
 927         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
 928         ip_fle->length = data_len;
 929
 930         if (sess->iv.length) {
 931                 uint8_t *iv_ptr;
 932
 933                 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 934                                                    sess->iv.offset);
 935
 936                 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
 937                         iv_ptr = conv_to_snow_f9_iv(iv_ptr);
 938                         sge->length = 12;
 939                 } else if (sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
 940                         iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
 941                         sge->length = 8;
 942                 } else {
 943                         sge->length = sess->iv.length;
 944                 }
 945                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
 946                 ip_fle->length += sge->length;
 947                 sge++;
 948         }
 949         /* i/p 1st seg */
 950         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 951         DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
 952
 953         if (data_len <= (mbuf->data_len - data_offset)) {
 954                 sge->length = data_len;
 955                 data_len = 0;
 956         } else {
 957                 sge->length = mbuf->data_len - data_offset;
 958
 959                 /* remaining i/p segs */
 960                 while ((data_len = data_len - sge->length) &&
 961                        (mbuf = mbuf->next)) {
 962                         sge++;
 963                         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 964                         DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 965                         if (data_len > mbuf->data_len)
 966                                 sge->length = mbuf->data_len;
 967                         else
 968                                 sge->length = data_len;
 969                 }
 970         }
 971
 972         if (sess->dir == DIR_DEC) {
 973                 /* Digest verification case */
 974                 sge++;
 975                 old_digest = (uint8_t *)(sge + 1);
 976                 rte_memcpy(old_digest, sym_op->auth.digest.data,
 977                            sess->digest_length);
 978                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest));
 979                 sge->length = sess->digest_length;
 980                 ip_fle->length += sess->digest_length;
 981         }
 982         DPAA2_SET_FLE_FIN(sge);
 983         DPAA2_SET_FLE_FIN(ip_fle);
 984         DPAA2_SET_FD_LEN(fd, ip_fle->length);
 985
 986         return 0;
 987 }
 988
 989 static inline int
 990 build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
 991               struct qbman_fd *fd, uint16_t bpid)
 992 {
 993         struct rte_crypto_sym_op *sym_op = op->sym;
 994         struct qbman_fle *fle, *sge;
 995         struct sec_flow_context *flc;
 996         struct ctxt_priv *priv = sess->ctxt;
 997         int data_len, data_offset;
 998         uint8_t *old_digest;
 999         int retval;
1000
1001         data_len = sym_op->auth.data.length;
1002         data_offset = sym_op->auth.data.offset;
1003
1004         if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
1005             sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
1006                 if ((data_len & 7) || (data_offset & 7)) {
1007                         DPAA2_SEC_ERR("AUTH: len/offset must be full bytes");
1008                         return -ENOTSUP;
1009                 }
1010
1011                 data_len = data_len >> 3;
1012                 data_offset = data_offset >> 3;
1013         }
1014
1015         retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
1016         if (retval) {
1017                 DPAA2_SEC_ERR("AUTH Memory alloc failed for SGE");
1018                 return -ENOMEM;
1019         }
1020         memset(fle, 0, FLE_POOL_BUF_SIZE);
1021         /* TODO we are using the first FLE entry to store Mbuf.
1022          * Currently we donot know which FLE has the mbuf stored.
1023          * So while retreiving we can go back 1 FLE from the FD -ADDR
1024          * to get the MBUF Addr from the previous FLE.
1025          * We can have a better approach to use the inline Mbuf
1026          */
1027         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1028         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1029         fle = fle + 1;
1030         sge = fle + 2;
1031
1032         if (likely(bpid < MAX_BPID)) {
1033                 DPAA2_SET_FD_BPID(fd, bpid);
1034                 DPAA2_SET_FLE_BPID(fle, bpid);
1035                 DPAA2_SET_FLE_BPID(fle + 1, bpid);
1036                 DPAA2_SET_FLE_BPID(sge, bpid);
1037                 DPAA2_SET_FLE_BPID(sge + 1, bpid);
1038         } else {
1039                 DPAA2_SET_FD_IVP(fd);
1040                 DPAA2_SET_FLE_IVP(fle);
1041                 DPAA2_SET_FLE_IVP((fle + 1));
1042                 DPAA2_SET_FLE_IVP(sge);
1043                 DPAA2_SET_FLE_IVP((sge + 1));
1044         }
1045
1046         flc = &priv->flc_desc[DESC_INITFINAL].flc;
1047         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1048         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
1049         DPAA2_SET_FD_COMPOUND_FMT(fd);
1050
1051         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
1052         fle->length = sess->digest_length;
1053         fle++;
1054
1055         /* Setting input FLE */
1056         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
1057         DPAA2_SET_FLE_SG_EXT(fle);
1058         fle->length = data_len;
1059
1060         if (sess->iv.length) {
1061                 uint8_t *iv_ptr;
1062
1063                 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1064                                                    sess->iv.offset);
1065
1066                 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
1067                         iv_ptr = conv_to_snow_f9_iv(iv_ptr);
1068                         sge->length = 12;
1069                 } else if (sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
1070                         iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
1071                         sge->length = 8;
1072                 } else {
1073                         sge->length = sess->iv.length;
1074                 }
1075
1076                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1077                 fle->length = fle->length + sge->length;
1078                 sge++;
1079         }
1080
1081         /* Setting data to authenticate */
1082         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
1083         DPAA2_SET_FLE_OFFSET(sge, data_offset + sym_op->m_src->data_off);
1084         sge->length = data_len;
1085
1086         if (sess->dir == DIR_DEC) {
1087                 sge++;
1088                 old_digest = (uint8_t *)(sge + 1);
1089                 rte_memcpy(old_digest, sym_op->auth.digest.data,
1090                            sess->digest_length);
1091                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest));
1092                 sge->length = sess->digest_length;
1093                 fle->length = fle->length + sess->digest_length;
1094         }
1095
1096         DPAA2_SET_FLE_FIN(sge);
1097         DPAA2_SET_FLE_FIN(fle);
1098         DPAA2_SET_FD_LEN(fd, fle->length);
1099
1100         return 0;
1101 }
1102
1103 static int
1104 build_cipher_sg_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
1105                 struct qbman_fd *fd, __rte_unused uint16_t bpid)
1106 {
1107         struct rte_crypto_sym_op *sym_op = op->sym;
1108         struct qbman_fle *ip_fle, *op_fle, *sge, *fle;
1109         int data_len, data_offset;
1110         struct sec_flow_context *flc;
1111         struct ctxt_priv *priv = sess->ctxt;
1112         struct rte_mbuf *mbuf;
1113         uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1114                         sess->iv.offset);
1115
1116         data_len = sym_op->cipher.data.length;
1117         data_offset = sym_op->cipher.data.offset;
1118
1119         if (sess->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
1120                 sess->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
1121                 if ((data_len & 7) || (data_offset & 7)) {
1122                         DPAA2_SEC_ERR("CIPHER: len/offset must be full bytes");
1123                         return -ENOTSUP;
1124                 }
1125
1126                 data_len = data_len >> 3;
1127                 data_offset = data_offset >> 3;
1128         }
1129
1130         if (sym_op->m_dst)
1131                 mbuf = sym_op->m_dst;
1132         else
1133                 mbuf = sym_op->m_src;
1134
1135         /* first FLE entry used to store mbuf and session ctxt */
1136         fle = (struct qbman_fle *)rte_malloc(NULL,
1137                         FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
1138                         RTE_CACHE_LINE_SIZE);
1139         if (!fle) {
1140                 DPAA2_SEC_ERR("CIPHER SG: Memory alloc failed for SGE");
1141                 return -ENOMEM;
1142         }
1143         memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
1144         /* first FLE entry used to store mbuf and session ctxt */
1145         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1146         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1147
1148         op_fle = fle + 1;
1149         ip_fle = fle + 2;
1150         sge = fle + 3;
1151
1152         flc = &priv->flc_desc[0].flc;
1153
1154         DPAA2_SEC_DP_DEBUG(
1155                 "CIPHER SG: cipher_off: 0x%x/length %d, ivlen=%d"
1156                 " data_off: 0x%x\n",
1157                 data_offset,
1158                 data_len,
1159                 sess->iv.length,
1160                 sym_op->m_src->data_off);
1161
1162         /* o/p fle */
1163         DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
1164         op_fle->length = data_len;
1165         DPAA2_SET_FLE_SG_EXT(op_fle);
1166
1167         /* o/p 1st seg */
1168         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1169         DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
1170         sge->length = mbuf->data_len - data_offset;
1171
1172         mbuf = mbuf->next;
1173         /* o/p segs */
1174         while (mbuf) {
1175                 sge++;
1176                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1177                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
1178                 sge->length = mbuf->data_len;
1179                 mbuf = mbuf->next;
1180         }
1181         DPAA2_SET_FLE_FIN(sge);
1182
1183         DPAA2_SEC_DP_DEBUG(
1184                 "CIPHER SG: 1 - flc = %p, fle = %p FLEaddr = %x-%x, len %d\n",
1185                 flc, fle, fle->addr_hi, fle->addr_lo,
1186                 fle->length);
1187
1188         /* i/p fle */
1189         mbuf = sym_op->m_src;
1190         sge++;
1191         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
1192         ip_fle->length = sess->iv.length + data_len;
1193         DPAA2_SET_FLE_SG_EXT(ip_fle);
1194
1195         /* i/p IV */
1196         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1197         DPAA2_SET_FLE_OFFSET(sge, 0);
1198         sge->length = sess->iv.length;
1199
1200         sge++;
1201
1202         /* i/p 1st seg */
1203         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1204         DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
1205         sge->length = mbuf->data_len - data_offset;
1206
1207         mbuf = mbuf->next;
1208         /* i/p segs */
1209         while (mbuf) {
1210                 sge++;
1211                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1212                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
1213                 sge->length = mbuf->data_len;
1214                 mbuf = mbuf->next;
1215         }
1216         DPAA2_SET_FLE_FIN(sge);
1217         DPAA2_SET_FLE_FIN(ip_fle);
1218
1219         /* sg fd */
1220         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
1221         DPAA2_SET_FD_LEN(fd, ip_fle->length);
1222         DPAA2_SET_FD_COMPOUND_FMT(fd);
1223         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1224
1225         DPAA2_SEC_DP_DEBUG(
1226                 "CIPHER SG: fdaddr =%" PRIx64 " bpid =%d meta =%d"
1227                 " off =%d, len =%d\n",
1228                 DPAA2_GET_FD_ADDR(fd),
1229                 DPAA2_GET_FD_BPID(fd),
1230                 rte_dpaa2_bpid_info[bpid].meta_data_size,
1231                 DPAA2_GET_FD_OFFSET(fd),
1232                 DPAA2_GET_FD_LEN(fd));
1233         return 0;
1234 }
1235
1236 static int
1237 build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
1238                 struct qbman_fd *fd, uint16_t bpid)
1239 {
1240         struct rte_crypto_sym_op *sym_op = op->sym;
1241         struct qbman_fle *fle, *sge;
1242         int retval, data_len, data_offset;
1243         struct sec_flow_context *flc;
1244         struct ctxt_priv *priv = sess->ctxt;
1245         uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1246                         sess->iv.offset);
1247         struct rte_mbuf *dst;
1248
1249         data_len = sym_op->cipher.data.length;
1250         data_offset = sym_op->cipher.data.offset;
1251
1252         if (sess->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
1253                 sess->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
1254                 if ((data_len & 7) || (data_offset & 7)) {
1255                         DPAA2_SEC_ERR("CIPHER: len/offset must be full bytes");
1256                         return -ENOTSUP;
1257                 }
1258
1259                 data_len = data_len >> 3;
1260                 data_offset = data_offset >> 3;
1261         }
1262
1263         if (sym_op->m_dst)
1264                 dst = sym_op->m_dst;
1265         else
1266                 dst = sym_op->m_src;
1267
1268         retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
1269         if (retval) {
1270                 DPAA2_SEC_ERR("CIPHER: Memory alloc failed for SGE");
1271                 return -ENOMEM;
1272         }
1273         memset(fle, 0, FLE_POOL_BUF_SIZE);
1274         /* TODO we are using the first FLE entry to store Mbuf.
1275          * Currently we donot know which FLE has the mbuf stored.
1276          * So while retreiving we can go back 1 FLE from the FD -ADDR
1277          * to get the MBUF Addr from the previous FLE.
1278          * We can have a better approach to use the inline Mbuf
1279          */
1280         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1281         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1282         fle = fle + 1;
1283         sge = fle + 2;
1284
1285         if (likely(bpid < MAX_BPID)) {
1286                 DPAA2_SET_FD_BPID(fd, bpid);
1287                 DPAA2_SET_FLE_BPID(fle, bpid);
1288                 DPAA2_SET_FLE_BPID(fle + 1, bpid);
1289                 DPAA2_SET_FLE_BPID(sge, bpid);
1290                 DPAA2_SET_FLE_BPID(sge + 1, bpid);
1291         } else {
1292                 DPAA2_SET_FD_IVP(fd);
1293                 DPAA2_SET_FLE_IVP(fle);
1294                 DPAA2_SET_FLE_IVP((fle + 1));
1295                 DPAA2_SET_FLE_IVP(sge);
1296                 DPAA2_SET_FLE_IVP((sge + 1));
1297         }
1298
1299         flc = &priv->flc_desc[0].flc;
1300         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
1301         DPAA2_SET_FD_LEN(fd, data_len + sess->iv.length);
1302         DPAA2_SET_FD_COMPOUND_FMT(fd);
1303         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1304
1305         DPAA2_SEC_DP_DEBUG(
1306                 "CIPHER: cipher_off: 0x%x/length %d, ivlen=%d,"
1307                 " data_off: 0x%x\n",
1308                 data_offset,
1309                 data_len,
1310                 sess->iv.length,
1311                 sym_op->m_src->data_off);
1312
1313         DPAA2_SET_FLE_ADDR(fle, DPAA2_MBUF_VADDR_TO_IOVA(dst));
1314         DPAA2_SET_FLE_OFFSET(fle, data_offset + dst->data_off);
1315
1316         fle->length = data_len + sess->iv.length;
1317
1318         DPAA2_SEC_DP_DEBUG(
1319                 "CIPHER: 1 - flc = %p, fle = %p FLEaddr = %x-%x, length %d\n",
1320                 flc, fle, fle->addr_hi, fle->addr_lo,
1321                 fle->length);
1322
1323         fle++;
1324
1325         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
1326         fle->length = data_len + sess->iv.length;
1327
1328         DPAA2_SET_FLE_SG_EXT(fle);
1329
1330         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1331         sge->length = sess->iv.length;
1332
1333         sge++;
1334         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
1335         DPAA2_SET_FLE_OFFSET(sge, data_offset + sym_op->m_src->data_off);
1336
1337         sge->length = data_len;
1338         DPAA2_SET_FLE_FIN(sge);
1339         DPAA2_SET_FLE_FIN(fle);
1340
1341         DPAA2_SEC_DP_DEBUG(
1342                 "CIPHER: fdaddr =%" PRIx64 " bpid =%d meta =%d"
1343                 " off =%d, len =%d\n",
1344                 DPAA2_GET_FD_ADDR(fd),
1345                 DPAA2_GET_FD_BPID(fd),
1346                 rte_dpaa2_bpid_info[bpid].meta_data_size,
1347                 DPAA2_GET_FD_OFFSET(fd),
1348                 DPAA2_GET_FD_LEN(fd));
1349
1350         return 0;
1351 }
1352
1353 static inline int
1354 build_sec_fd(struct rte_crypto_op *op,
1355              struct qbman_fd *fd, uint16_t bpid)
1356 {
1357         int ret = -1;
1358         dpaa2_sec_session *sess;
1359
1360         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)
1361                 sess = (dpaa2_sec_session *)get_sym_session_private_data(
1362                                 op->sym->session, cryptodev_driver_id);
1363 #ifdef RTE_LIB_SECURITY
1364         else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
1365                 sess = (dpaa2_sec_session *)get_sec_session_private_data(
1366                                 op->sym->sec_session);
1367 #endif
1368         else
1369                 return -ENOTSUP;
1370
1371         if (!sess)
1372                 return -EINVAL;
1373
1374         /* Any of the buffer is segmented*/
1375         if (!rte_pktmbuf_is_contiguous(op->sym->m_src) ||
1376                   ((op->sym->m_dst != NULL) &&
1377                    !rte_pktmbuf_is_contiguous(op->sym->m_dst))) {
1378                 switch (sess->ctxt_type) {
1379                 case DPAA2_SEC_CIPHER:
1380                         ret = build_cipher_sg_fd(sess, op, fd, bpid);
1381                         break;
1382                 case DPAA2_SEC_AUTH:
1383                         ret = build_auth_sg_fd(sess, op, fd, bpid);
1384                         break;
1385                 case DPAA2_SEC_AEAD:
1386                         ret = build_authenc_gcm_sg_fd(sess, op, fd, bpid);
1387                         break;
1388                 case DPAA2_SEC_CIPHER_HASH:
1389                         ret = build_authenc_sg_fd(sess, op, fd, bpid);
1390                         break;
1391 #ifdef RTE_LIB_SECURITY
1392                 case DPAA2_SEC_IPSEC:
1393                 case DPAA2_SEC_PDCP:
1394                         ret = build_proto_compound_sg_fd(sess, op, fd, bpid);
1395                         break;
1396 #endif
1397                 case DPAA2_SEC_HASH_CIPHER:
1398                 default:
1399                         DPAA2_SEC_ERR("error: Unsupported session");
1400                 }
1401         } else {
1402                 switch (sess->ctxt_type) {
1403                 case DPAA2_SEC_CIPHER:
1404                         ret = build_cipher_fd(sess, op, fd, bpid);
1405                         break;
1406                 case DPAA2_SEC_AUTH:
1407                         ret = build_auth_fd(sess, op, fd, bpid);
1408                         break;
1409                 case DPAA2_SEC_AEAD:
1410                         ret = build_authenc_gcm_fd(sess, op, fd, bpid);
1411                         break;
1412                 case DPAA2_SEC_CIPHER_HASH:
1413                         ret = build_authenc_fd(sess, op, fd, bpid);
1414                         break;
1415 #ifdef RTE_LIB_SECURITY
1416                 case DPAA2_SEC_IPSEC:
1417                         ret = build_proto_fd(sess, op, fd, bpid);
1418                         break;
1419                 case DPAA2_SEC_PDCP:
1420                         ret = build_proto_compound_fd(sess, op, fd, bpid);
1421                         break;
1422 #endif
1423                 case DPAA2_SEC_HASH_CIPHER:
1424                 default:
1425                         DPAA2_SEC_ERR("error: Unsupported session");
1426                         ret = -ENOTSUP;
1427                 }
1428         }
1429         return ret;
1430 }
1431
1432 static uint16_t
1433 dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1434                         uint16_t nb_ops)
1435 {
1436         /* Function to transmit the frames to given device and VQ*/
1437         uint32_t loop;
1438         int32_t ret;
1439         struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
1440         uint32_t frames_to_send, retry_count;
1441         struct qbman_eq_desc eqdesc;
1442         struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp;
1443         struct qbman_swp *swp;
1444         uint16_t num_tx = 0;
1445         uint32_t flags[MAX_TX_RING_SLOTS] = {0};
1446         /*todo - need to support multiple buffer pools */
1447         uint16_t bpid;
1448         struct rte_mempool *mb_pool;
1449
1450         if (unlikely(nb_ops == 0))
1451                 return 0;
1452
1453         if (ops[0]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1454                 DPAA2_SEC_ERR("sessionless crypto op not supported");
1455                 return 0;
1456         }
1457         /*Prepare enqueue descriptor*/
1458         qbman_eq_desc_clear(&eqdesc);
1459         qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ);
1460         qbman_eq_desc_set_response(&eqdesc, 0, 0);
1461         qbman_eq_desc_set_fq(&eqdesc, dpaa2_qp->tx_vq.fqid);
1462
1463         if (!DPAA2_PER_LCORE_DPIO) {
1464                 ret = dpaa2_affine_qbman_swp();
1465                 if (ret) {
1466                         DPAA2_SEC_ERR(
1467                                 "Failed to allocate IO portal, tid: %d\n",
1468                                 rte_gettid());
1469                         return 0;
1470                 }
1471         }
1472         swp = DPAA2_PER_LCORE_PORTAL;
1473
1474         while (nb_ops) {
1475                 frames_to_send = (nb_ops > dpaa2_eqcr_size) ?
1476                         dpaa2_eqcr_size : nb_ops;
1477
1478                 for (loop = 0; loop < frames_to_send; loop++) {
1479                         if (*dpaa2_seqn((*ops)->sym->m_src)) {
1480                                 uint8_t dqrr_index =
1481                                         *dpaa2_seqn((*ops)->sym->m_src) - 1;
1482
1483                                 flags[loop] = QBMAN_ENQUEUE_FLAG_DCA | dqrr_index;
1484                                 DPAA2_PER_LCORE_DQRR_SIZE--;
1485                                 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index);
1486                                 *dpaa2_seqn((*ops)->sym->m_src) =
1487                                         DPAA2_INVALID_MBUF_SEQN;
1488                         }
1489
1490                         /*Clear the unused FD fields before sending*/
1491                         memset(&fd_arr[loop], 0, sizeof(struct qbman_fd));
1492                         mb_pool = (*ops)->sym->m_src->pool;
1493                         bpid = mempool_to_bpid(mb_pool);
1494                         ret = build_sec_fd(*ops, &fd_arr[loop], bpid);
1495                         if (ret) {
1496                                 DPAA2_SEC_ERR("error: Improper packet contents"
1497                                               " for crypto operation");
1498                                 goto skip_tx;
1499                         }
1500                         ops++;
1501                 }
1502
1503                 loop = 0;
1504                 retry_count = 0;
1505                 while (loop < frames_to_send) {
1506                         ret = qbman_swp_enqueue_multiple(swp, &eqdesc,
1507                                                          &fd_arr[loop],
1508                                                          &flags[loop],
1509                                                          frames_to_send - loop);
1510                         if (unlikely(ret < 0)) {
1511                                 retry_count++;
1512                                 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) {
1513                                         num_tx += loop;
1514                                         nb_ops -= loop;
1515                                         goto skip_tx;
1516                                 }
1517                         } else {
1518                                 loop += ret;
1519                                 retry_count = 0;
1520                         }
1521                 }
1522
1523                 num_tx += loop;
1524                 nb_ops -= loop;
1525         }
1526 skip_tx:
1527         dpaa2_qp->tx_vq.tx_pkts += num_tx;
1528         dpaa2_qp->tx_vq.err_pkts += nb_ops;
1529         return num_tx;
1530 }
1531
1532 #ifdef RTE_LIB_SECURITY
1533 static inline struct rte_crypto_op *
1534 sec_simple_fd_to_mbuf(const struct qbman_fd *fd)
1535 {
1536         struct rte_crypto_op *op;
1537         uint16_t len = DPAA2_GET_FD_LEN(fd);
1538         int16_t diff = 0;
1539         dpaa2_sec_session *sess_priv __rte_unused;
1540
1541         struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF(
1542                 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)),
1543                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
1544
1545         diff = len - mbuf->pkt_len;
1546         mbuf->pkt_len += diff;
1547         mbuf->data_len += diff;
1548         op = (struct rte_crypto_op *)(size_t)mbuf->buf_iova;
1549         mbuf->buf_iova = op->sym->aead.digest.phys_addr;
1550         op->sym->aead.digest.phys_addr = 0L;
1551
1552         sess_priv = (dpaa2_sec_session *)get_sec_session_private_data(
1553                                 op->sym->sec_session);
1554         if (sess_priv->dir == DIR_ENC)
1555                 mbuf->data_off += SEC_FLC_DHR_OUTBOUND;
1556         else
1557                 mbuf->data_off += SEC_FLC_DHR_INBOUND;
1558
1559         return op;
1560 }
1561 #endif
1562
1563 static inline struct rte_crypto_op *
1564 sec_fd_to_mbuf(const struct qbman_fd *fd)
1565 {
1566         struct qbman_fle *fle;
1567         struct rte_crypto_op *op;
1568         struct ctxt_priv *priv;
1569         struct rte_mbuf *dst, *src;
1570
1571 #ifdef RTE_LIB_SECURITY
1572         if (DPAA2_FD_GET_FORMAT(fd) == qbman_fd_single)
1573                 return sec_simple_fd_to_mbuf(fd);
1574 #endif
1575         fle = (struct qbman_fle *)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
1576
1577         DPAA2_SEC_DP_DEBUG("FLE addr = %x - %x, offset = %x\n",
1578                            fle->addr_hi, fle->addr_lo, fle->fin_bpid_offset);
1579
1580         /* we are using the first FLE entry to store Mbuf.
1581          * Currently we donot know which FLE has the mbuf stored.
1582          * So while retreiving we can go back 1 FLE from the FD -ADDR
1583          * to get the MBUF Addr from the previous FLE.
1584          * We can have a better approach to use the inline Mbuf
1585          */
1586
1587         if (unlikely(DPAA2_GET_FD_IVP(fd))) {
1588                 /* TODO complete it. */
1589                 DPAA2_SEC_ERR("error: non inline buffer");
1590                 return NULL;
1591         }
1592         op = (struct rte_crypto_op *)DPAA2_GET_FLE_ADDR((fle - 1));
1593
1594         /* Prefeth op */
1595         src = op->sym->m_src;
1596         rte_prefetch0(src);
1597
1598         if (op->sym->m_dst) {
1599                 dst = op->sym->m_dst;
1600                 rte_prefetch0(dst);
1601         } else
1602                 dst = src;
1603
1604 #ifdef RTE_LIB_SECURITY
1605         if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
1606                 uint16_t len = DPAA2_GET_FD_LEN(fd);
1607                 dst->pkt_len = len;
1608                 while (dst->next != NULL) {
1609                         len -= dst->data_len;
1610                         dst = dst->next;
1611                 }
1612                 dst->data_len = len;
1613         }
1614 #endif
1615         DPAA2_SEC_DP_DEBUG("mbuf %p BMAN buf addr %p,"
1616                 " fdaddr =%" PRIx64 " bpid =%d meta =%d off =%d, len =%d\n",
1617                 (void *)dst,
1618                 dst->buf_addr,
1619                 DPAA2_GET_FD_ADDR(fd),
1620                 DPAA2_GET_FD_BPID(fd),
1621                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
1622                 DPAA2_GET_FD_OFFSET(fd),
1623                 DPAA2_GET_FD_LEN(fd));
1624
1625         /* free the fle memory */
1626         if (likely(rte_pktmbuf_is_contiguous(src))) {
1627                 priv = (struct ctxt_priv *)(size_t)DPAA2_GET_FLE_CTXT(fle - 1);
1628                 rte_mempool_put(priv->fle_pool, (void *)(fle-1));
1629         } else
1630                 rte_free((void *)(fle-1));
1631
1632         return op;
1633 }
1634
1635 static void
1636 dpaa2_sec_dump(struct rte_crypto_op *op)
1637 {
1638         int i;
1639         dpaa2_sec_session *sess = NULL;
1640         struct ctxt_priv *priv;
1641         uint8_t bufsize;
1642         struct rte_crypto_sym_op *sym_op;
1643
1644         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)
1645                 sess = (dpaa2_sec_session *)get_sym_session_private_data(
1646                         op->sym->session, cryptodev_driver_id);
1647 #ifdef RTE_LIBRTE_SECURITY
1648         else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
1649                 sess = (dpaa2_sec_session *)get_sec_session_private_data(
1650                         op->sym->sec_session);
1651 #endif
1652
1653         if (sess == NULL)
1654                 goto mbuf_dump;
1655
1656         priv = (struct ctxt_priv *)sess->ctxt;
1657         printf("\n****************************************\n"
1658                 "session params:\n\tContext type:\t%d\n\tDirection:\t%s\n"
1659                 "\tCipher alg:\t%d\n\tAuth alg:\t%d\n\tAead alg:\t%d\n"
1660                 "\tCipher key len:\t%zd\n", sess->ctxt_type,
1661                 (sess->dir == DIR_ENC) ? "DIR_ENC" : "DIR_DEC",
1662                 sess->cipher_alg, sess->auth_alg, sess->aead_alg,
1663                 sess->cipher_key.length);
1664                 rte_hexdump(stdout, "cipher key", sess->cipher_key.data,
1665                                 sess->cipher_key.length);
1666                 rte_hexdump(stdout, "auth key", sess->auth_key.data,
1667                                 sess->auth_key.length);
1668         printf("\tAuth key len:\t%zd\n\tIV len:\t\t%d\n\tIV offset:\t%d\n"
1669                 "\tdigest length:\t%d\n\tstatus:\t\t%d\n\taead auth only"
1670                 " len:\t%d\n\taead cipher text:\t%d\n",
1671                 sess->auth_key.length, sess->iv.length, sess->iv.offset,
1672                 sess->digest_length, sess->status,
1673                 sess->ext_params.aead_ctxt.auth_only_len,
1674                 sess->ext_params.aead_ctxt.auth_cipher_text);
1675 #ifdef RTE_LIBRTE_SECURITY
1676         printf("PDCP session params:\n"
1677                 "\tDomain:\t\t%d\n\tBearer:\t\t%d\n\tpkt_dir:\t%d\n\thfn_ovd:"
1678                 "\t%d\n\tsn_size:\t%d\n\thfn_ovd_offset:\t%d\n\thfn:\t\t%d\n"
1679                 "\thfn_threshold:\t0x%x\n", sess->pdcp.domain,
1680                 sess->pdcp.bearer, sess->pdcp.pkt_dir, sess->pdcp.hfn_ovd,
1681                 sess->pdcp.sn_size, sess->pdcp.hfn_ovd_offset, sess->pdcp.hfn,
1682                 sess->pdcp.hfn_threshold);
1683
1684 #endif
1685         bufsize = (uint8_t)priv->flc_desc[0].flc.word1_sdl;
1686         printf("Descriptor Dump:\n");
1687         for (i = 0; i < bufsize; i++)
1688                 printf("\tDESC[%d]:0x%x\n", i, priv->flc_desc[0].desc[i]);
1689
1690         printf("\n");
1691 mbuf_dump:
1692         sym_op = op->sym;
1693         if (sym_op->m_src) {
1694                 printf("Source mbuf:\n");
1695                 rte_pktmbuf_dump(stdout, sym_op->m_src, sym_op->m_src->data_len);
1696         }
1697         if (sym_op->m_dst) {
1698                 printf("Destination mbuf:\n");
1699                 rte_pktmbuf_dump(stdout, sym_op->m_dst, sym_op->m_dst->data_len);
1700         }
1701
1702         printf("Session address = %p\ncipher offset: %d, length: %d\n"
1703                 "auth offset: %d, length:  %d\n aead offset: %d, length: %d\n"
1704                 , sym_op->session,
1705                 sym_op->cipher.data.offset, sym_op->cipher.data.length,
1706                 sym_op->auth.data.offset, sym_op->auth.data.length,
1707                 sym_op->aead.data.offset, sym_op->aead.data.length);
1708         printf("\n");
1709
1710 }
1711
1712 static uint16_t
1713 dpaa2_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops,
1714                         uint16_t nb_ops)
1715 {
1716         /* Function is responsible to receive frames for a given device and VQ*/
1717         struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp;
1718         struct qbman_result *dq_storage;
1719         uint32_t fqid = dpaa2_qp->rx_vq.fqid;
1720         int ret, num_rx = 0;
1721         uint8_t is_last = 0, status;
1722         struct qbman_swp *swp;
1723         const struct qbman_fd *fd;
1724         struct qbman_pull_desc pulldesc;
1725
1726         if (!DPAA2_PER_LCORE_DPIO) {
1727                 ret = dpaa2_affine_qbman_swp();
1728                 if (ret) {
1729                         DPAA2_SEC_ERR(
1730                                 "Failed to allocate IO portal, tid: %d\n",
1731                                 rte_gettid());
1732                         return 0;
1733                 }
1734         }
1735         swp = DPAA2_PER_LCORE_PORTAL;
1736         dq_storage = dpaa2_qp->rx_vq.q_storage->dq_storage[0];
1737
1738         qbman_pull_desc_clear(&pulldesc);
1739         qbman_pull_desc_set_numframes(&pulldesc,
1740                                       (nb_ops > dpaa2_dqrr_size) ?
1741                                       dpaa2_dqrr_size : nb_ops);
1742         qbman_pull_desc_set_fq(&pulldesc, fqid);
1743         qbman_pull_desc_set_storage(&pulldesc, dq_storage,
1744                                     (dma_addr_t)DPAA2_VADDR_TO_IOVA(dq_storage),
1745                                     1);
1746
1747         /*Issue a volatile dequeue command. */
1748         while (1) {
1749                 if (qbman_swp_pull(swp, &pulldesc)) {
1750                         DPAA2_SEC_WARN(
1751                                 "SEC VDQ command is not issued : QBMAN busy");
1752                         /* Portal was busy, try again */
1753                         continue;
1754                 }
1755                 break;
1756         };
1757
1758         /* Receive the packets till Last Dequeue entry is found with
1759          * respect to the above issues PULL command.
1760          */
1761         while (!is_last) {
1762                 /* Check if the previous issued command is completed.
1763                  * Also seems like the SWP is shared between the Ethernet Driver
1764                  * and the SEC driver.
1765                  */
1766                 while (!qbman_check_command_complete(dq_storage))
1767                         ;
1768
1769                 /* Loop until the dq_storage is updated with
1770                  * new token by QBMAN
1771                  */
1772                 while (!qbman_check_new_result(dq_storage))
1773                         ;
1774                 /* Check whether Last Pull command is Expired and
1775                  * setting Condition for Loop termination
1776                  */
1777                 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
1778                         is_last = 1;
1779                         /* Check for valid frame. */
1780                         status = (uint8_t)qbman_result_DQ_flags(dq_storage);
1781                         if (unlikely(
1782                                 (status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) {
1783                                 DPAA2_SEC_DP_DEBUG("No frame is delivered\n");
1784                                 continue;
1785                         }
1786                 }
1787
1788                 fd = qbman_result_DQ_fd(dq_storage);
1789                 ops[num_rx] = sec_fd_to_mbuf(fd);
1790
1791                 if (unlikely(fd->simple.frc)) {
1792                         /* TODO Parse SEC errors */
1793                         if (dpaa2_sec_dp_dump > DPAA2_SEC_DP_NO_DUMP) {
1794                                 DPAA2_SEC_DP_ERR("SEC returned Error - %x\n",
1795                                                  fd->simple.frc);
1796                                 if (dpaa2_sec_dp_dump > DPAA2_SEC_DP_ERR_DUMP)
1797                                         dpaa2_sec_dump(ops[num_rx]);
1798                         }
1799
1800                         dpaa2_qp->rx_vq.err_pkts += 1;
1801                         ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_ERROR;
1802                 } else {
1803                         ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1804                 }
1805
1806                 num_rx++;
1807                 dq_storage++;
1808         } /* End of Packet Rx loop */
1809
1810         dpaa2_qp->rx_vq.rx_pkts += num_rx;
1811
1812         DPAA2_SEC_DP_DEBUG("SEC RX pkts %d err pkts %" PRIu64 "\n", num_rx,
1813                                 dpaa2_qp->rx_vq.err_pkts);
1814         /*Return the total number of packets received to DPAA2 app*/
1815         return num_rx;
1816 }
1817
1818 /** Release queue pair */
1819 static int
1820 dpaa2_sec_queue_pair_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
1821 {
1822         struct dpaa2_sec_qp *qp =
1823                 (struct dpaa2_sec_qp *)dev->data->queue_pairs[queue_pair_id];
1824
1825         PMD_INIT_FUNC_TRACE();
1826
1827         if (qp->rx_vq.q_storage) {
1828                 dpaa2_free_dq_storage(qp->rx_vq.q_storage);
1829                 rte_free(qp->rx_vq.q_storage);
1830         }
1831         rte_free(qp);
1832
1833         dev->data->queue_pairs[queue_pair_id] = NULL;
1834
1835         return 0;
1836 }
1837
1838 /** Setup a queue pair */
1839 static int
1840 dpaa2_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
1841                 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
1842                 __rte_unused int socket_id)
1843 {
1844         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
1845         struct dpaa2_sec_qp *qp;
1846         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
1847         struct dpseci_rx_queue_cfg cfg;
1848         int32_t retcode;
1849
1850         PMD_INIT_FUNC_TRACE();
1851
1852         /* If qp is already in use free ring memory and qp metadata. */
1853         if (dev->data->queue_pairs[qp_id] != NULL) {
1854                 DPAA2_SEC_INFO("QP already setup");
1855                 return 0;
1856         }
1857
1858         DPAA2_SEC_DEBUG("dev =%p, queue =%d, conf =%p",
1859                     dev, qp_id, qp_conf);
1860
1861         memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
1862
1863         qp = rte_malloc(NULL, sizeof(struct dpaa2_sec_qp),
1864                         RTE_CACHE_LINE_SIZE);
1865         if (!qp) {
1866                 DPAA2_SEC_ERR("malloc failed for rx/tx queues");
1867                 return -ENOMEM;
1868         }
1869
1870         qp->rx_vq.crypto_data = dev->data;
1871         qp->tx_vq.crypto_data = dev->data;
1872         qp->rx_vq.q_storage = rte_malloc("sec dq storage",
1873                 sizeof(struct queue_storage_info_t),
1874                 RTE_CACHE_LINE_SIZE);
1875         if (!qp->rx_vq.q_storage) {
1876                 DPAA2_SEC_ERR("malloc failed for q_storage");
1877                 return -ENOMEM;
1878         }
1879         memset(qp->rx_vq.q_storage, 0, sizeof(struct queue_storage_info_t));
1880
1881         if (dpaa2_alloc_dq_storage(qp->rx_vq.q_storage)) {
1882                 DPAA2_SEC_ERR("Unable to allocate dequeue storage");
1883                 return -ENOMEM;
1884         }
1885
1886         dev->data->queue_pairs[qp_id] = qp;
1887
1888         cfg.options = cfg.options | DPSECI_QUEUE_OPT_USER_CTX;
1889         cfg.user_ctx = (size_t)(&qp->rx_vq);
1890         retcode = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
1891                                       qp_id, &cfg);
1892         return retcode;
1893 }
1894
1895 /** Returns the size of the aesni gcm session structure */
1896 static unsigned int
1897 dpaa2_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
1898 {
1899         PMD_INIT_FUNC_TRACE();
1900
1901         return sizeof(dpaa2_sec_session);
1902 }
1903
1904 static int
1905 dpaa2_sec_cipher_init(struct rte_cryptodev *dev,
1906                       struct rte_crypto_sym_xform *xform,
1907                       dpaa2_sec_session *session)
1908 {
1909         struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
1910         struct alginfo cipherdata;
1911         int bufsize, ret = 0;
1912         struct ctxt_priv *priv;
1913         struct sec_flow_context *flc;
1914
1915         PMD_INIT_FUNC_TRACE();
1916
1917         /* For SEC CIPHER only one descriptor is required. */
1918         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
1919                         sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
1920                         RTE_CACHE_LINE_SIZE);
1921         if (priv == NULL) {
1922                 DPAA2_SEC_ERR("No Memory for priv CTXT");
1923                 return -ENOMEM;
1924         }
1925
1926         priv->fle_pool = dev_priv->fle_pool;
1927
1928         flc = &priv->flc_desc[0].flc;
1929
1930         session->ctxt_type = DPAA2_SEC_CIPHER;
1931         session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
1932                         RTE_CACHE_LINE_SIZE);
1933         if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
1934                 DPAA2_SEC_ERR("No Memory for cipher key");
1935                 rte_free(priv);
1936                 return -ENOMEM;
1937         }
1938         session->cipher_key.length = xform->cipher.key.length;
1939
1940         memcpy(session->cipher_key.data, xform->cipher.key.data,
1941                xform->cipher.key.length);
1942         cipherdata.key = (size_t)session->cipher_key.data;
1943         cipherdata.keylen = session->cipher_key.length;
1944         cipherdata.key_enc_flags = 0;
1945         cipherdata.key_type = RTA_DATA_IMM;
1946
1947         /* Set IV parameters */
1948         session->iv.offset = xform->cipher.iv.offset;
1949         session->iv.length = xform->cipher.iv.length;
1950         session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
1951                                 DIR_ENC : DIR_DEC;
1952
1953         switch (xform->cipher.algo) {
1954         case RTE_CRYPTO_CIPHER_AES_CBC:
1955                 cipherdata.algtype = OP_ALG_ALGSEL_AES;
1956                 cipherdata.algmode = OP_ALG_AAI_CBC;
1957                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
1958                 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1959                                                 SHR_NEVER, &cipherdata,
1960                                                 session->iv.length,
1961                                                 session->dir);
1962                 break;
1963         case RTE_CRYPTO_CIPHER_3DES_CBC:
1964                 cipherdata.algtype = OP_ALG_ALGSEL_3DES;
1965                 cipherdata.algmode = OP_ALG_AAI_CBC;
1966                 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
1967                 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1968                                                 SHR_NEVER, &cipherdata,
1969                                                 session->iv.length,
1970                                                 session->dir);
1971                 break;
1972         case RTE_CRYPTO_CIPHER_DES_CBC:
1973                 cipherdata.algtype = OP_ALG_ALGSEL_DES;
1974                 cipherdata.algmode = OP_ALG_AAI_CBC;
1975                 session->cipher_alg = RTE_CRYPTO_CIPHER_DES_CBC;
1976                 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1977                                                 SHR_NEVER, &cipherdata,
1978                                                 session->iv.length,
1979                                                 session->dir);
1980                 break;
1981         case RTE_CRYPTO_CIPHER_AES_CTR:
1982                 cipherdata.algtype = OP_ALG_ALGSEL_AES;
1983                 cipherdata.algmode = OP_ALG_AAI_CTR;
1984                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
1985                 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1986                                                 SHR_NEVER, &cipherdata,
1987                                                 session->iv.length,
1988                                                 session->dir);
1989                 break;
1990         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
1991                 cipherdata.algtype = OP_ALG_ALGSEL_SNOW_F8;
1992                 session->cipher_alg = RTE_CRYPTO_CIPHER_SNOW3G_UEA2;
1993                 bufsize = cnstr_shdsc_snow_f8(priv->flc_desc[0].desc, 1, 0,
1994                                               &cipherdata,
1995                                               session->dir);
1996                 break;
1997         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
1998                 cipherdata.algtype = OP_ALG_ALGSEL_ZUCE;
1999                 session->cipher_alg = RTE_CRYPTO_CIPHER_ZUC_EEA3;
2000                 bufsize = cnstr_shdsc_zuce(priv->flc_desc[0].desc, 1, 0,
2001                                               &cipherdata,
2002                                               session->dir);
2003                 break;
2004         case RTE_CRYPTO_CIPHER_KASUMI_F8:
2005         case RTE_CRYPTO_CIPHER_AES_F8:
2006         case RTE_CRYPTO_CIPHER_AES_ECB:
2007         case RTE_CRYPTO_CIPHER_3DES_ECB:
2008         case RTE_CRYPTO_CIPHER_3DES_CTR:
2009         case RTE_CRYPTO_CIPHER_AES_XTS:
2010         case RTE_CRYPTO_CIPHER_ARC4:
2011         case RTE_CRYPTO_CIPHER_NULL:
2012                 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2013                         xform->cipher.algo);
2014                 ret = -ENOTSUP;
2015                 goto error_out;
2016         default:
2017                 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
2018                         xform->cipher.algo);
2019                 ret = -ENOTSUP;
2020                 goto error_out;
2021         }
2022
2023         if (bufsize < 0) {
2024                 DPAA2_SEC_ERR("Crypto: Descriptor build failed");
2025                 ret = -EINVAL;
2026                 goto error_out;
2027         }
2028
2029         flc->word1_sdl = (uint8_t)bufsize;
2030         session->ctxt = priv;
2031
2032 #ifdef CAAM_DESC_DEBUG
2033         int i;
2034         for (i = 0; i < bufsize; i++)
2035                 DPAA2_SEC_DEBUG("DESC[%d]:0x%x", i, priv->flc_desc[0].desc[i]);
2036 #endif
2037         return ret;
2038
2039 error_out:
2040         rte_free(session->cipher_key.data);
2041         rte_free(priv);
2042         return ret;
2043 }
2044
2045 static int
2046 dpaa2_sec_auth_init(struct rte_cryptodev *dev,
2047                     struct rte_crypto_sym_xform *xform,
2048                     dpaa2_sec_session *session)
2049 {
2050         struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
2051         struct alginfo authdata;
2052         int bufsize, ret = 0;
2053         struct ctxt_priv *priv;
2054         struct sec_flow_context *flc;
2055
2056         PMD_INIT_FUNC_TRACE();
2057
2058         /* For SEC AUTH three descriptors are required for various stages */
2059         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2060                         sizeof(struct ctxt_priv) + 3 *
2061                         sizeof(struct sec_flc_desc),
2062                         RTE_CACHE_LINE_SIZE);
2063         if (priv == NULL) {
2064                 DPAA2_SEC_ERR("No Memory for priv CTXT");
2065                 return -ENOMEM;
2066         }
2067
2068         priv->fle_pool = dev_priv->fle_pool;
2069         flc = &priv->flc_desc[DESC_INITFINAL].flc;
2070
2071         session->ctxt_type = DPAA2_SEC_AUTH;
2072         session->auth_key.length = xform->auth.key.length;
2073         if (xform->auth.key.length) {
2074                 session->auth_key.data = rte_zmalloc(NULL,
2075                         xform->auth.key.length,
2076                         RTE_CACHE_LINE_SIZE);
2077                 if (session->auth_key.data == NULL) {
2078                         DPAA2_SEC_ERR("Unable to allocate memory for auth key");
2079                         rte_free(priv);
2080                         return -ENOMEM;
2081                 }
2082                 memcpy(session->auth_key.data, xform->auth.key.data,
2083                        xform->auth.key.length);
2084                 authdata.key = (size_t)session->auth_key.data;
2085                 authdata.key_enc_flags = 0;
2086                 authdata.key_type = RTA_DATA_IMM;
2087         }
2088         authdata.keylen = session->auth_key.length;
2089
2090         session->digest_length = xform->auth.digest_length;
2091         session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
2092                                 DIR_ENC : DIR_DEC;
2093
2094         switch (xform->auth.algo) {
2095         case RTE_CRYPTO_AUTH_SHA1_HMAC:
2096                 authdata.algtype = OP_ALG_ALGSEL_SHA1;
2097                 authdata.algmode = OP_ALG_AAI_HMAC;
2098                 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
2099                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2100                                            1, 0, SHR_NEVER, &authdata,
2101                                            !session->dir,
2102                                            session->digest_length);
2103                 break;
2104         case RTE_CRYPTO_AUTH_MD5_HMAC:
2105                 authdata.algtype = OP_ALG_ALGSEL_MD5;
2106                 authdata.algmode = OP_ALG_AAI_HMAC;
2107                 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
2108                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2109                                            1, 0, SHR_NEVER, &authdata,
2110                                            !session->dir,
2111                                            session->digest_length);
2112                 break;
2113         case RTE_CRYPTO_AUTH_SHA256_HMAC:
2114                 authdata.algtype = OP_ALG_ALGSEL_SHA256;
2115                 authdata.algmode = OP_ALG_AAI_HMAC;
2116                 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
2117                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2118                                            1, 0, SHR_NEVER, &authdata,
2119                                            !session->dir,
2120                                            session->digest_length);
2121                 break;
2122         case RTE_CRYPTO_AUTH_SHA384_HMAC:
2123                 authdata.algtype = OP_ALG_ALGSEL_SHA384;
2124                 authdata.algmode = OP_ALG_AAI_HMAC;
2125                 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
2126                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2127                                            1, 0, SHR_NEVER, &authdata,
2128                                            !session->dir,
2129                                            session->digest_length);
2130                 break;
2131         case RTE_CRYPTO_AUTH_SHA512_HMAC:
2132                 authdata.algtype = OP_ALG_ALGSEL_SHA512;
2133                 authdata.algmode = OP_ALG_AAI_HMAC;
2134                 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
2135                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2136                                            1, 0, SHR_NEVER, &authdata,
2137                                            !session->dir,
2138                                            session->digest_length);
2139                 break;
2140         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2141                 authdata.algtype = OP_ALG_ALGSEL_SHA224;
2142                 authdata.algmode = OP_ALG_AAI_HMAC;
2143                 session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC;
2144                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2145                                            1, 0, SHR_NEVER, &authdata,
2146                                            !session->dir,
2147                                            session->digest_length);
2148                 break;
2149         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2150                 authdata.algtype = OP_ALG_ALGSEL_SNOW_F9;
2151                 authdata.algmode = OP_ALG_AAI_F9;
2152                 session->auth_alg = RTE_CRYPTO_AUTH_SNOW3G_UIA2;
2153                 session->iv.offset = xform->auth.iv.offset;
2154                 session->iv.length = xform->auth.iv.length;
2155                 bufsize = cnstr_shdsc_snow_f9(priv->flc_desc[DESC_INITFINAL].desc,
2156                                               1, 0, &authdata,
2157                                               !session->dir,
2158                                               session->digest_length);
2159                 break;
2160         case RTE_CRYPTO_AUTH_ZUC_EIA3:
2161                 authdata.algtype = OP_ALG_ALGSEL_ZUCA;
2162                 authdata.algmode = OP_ALG_AAI_F9;
2163                 session->auth_alg = RTE_CRYPTO_AUTH_ZUC_EIA3;
2164                 session->iv.offset = xform->auth.iv.offset;
2165                 session->iv.length = xform->auth.iv.length;
2166                 bufsize = cnstr_shdsc_zuca(priv->flc_desc[DESC_INITFINAL].desc,
2167                                            1, 0, &authdata,
2168                                            !session->dir,
2169                                            session->digest_length);
2170                 break;
2171         case RTE_CRYPTO_AUTH_SHA1:
2172                 authdata.algtype = OP_ALG_ALGSEL_SHA1;
2173                 authdata.algmode = OP_ALG_AAI_HASH;
2174                 session->auth_alg = RTE_CRYPTO_AUTH_SHA1;
2175                 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2176                                            1, 0, SHR_NEVER, &authdata,
2177                                            !session->dir,
2178                                            session->digest_length);
2179                 break;
2180         case RTE_CRYPTO_AUTH_MD5:
2181                 authdata.algtype = OP_ALG_ALGSEL_MD5;
2182                 authdata.algmode = OP_ALG_AAI_HASH;
2183                 session->auth_alg = RTE_CRYPTO_AUTH_MD5;
2184                 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2185                                            1, 0, SHR_NEVER, &authdata,
2186                                            !session->dir,
2187                                            session->digest_length);
2188                 break;
2189         case RTE_CRYPTO_AUTH_SHA256:
2190                 authdata.algtype = OP_ALG_ALGSEL_SHA256;
2191                 authdata.algmode = OP_ALG_AAI_HASH;
2192                 session->auth_alg = RTE_CRYPTO_AUTH_SHA256;
2193                 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2194                                            1, 0, SHR_NEVER, &authdata,
2195                                            !session->dir,
2196                                            session->digest_length);
2197                 break;
2198         case RTE_CRYPTO_AUTH_SHA384:
2199                 authdata.algtype = OP_ALG_ALGSEL_SHA384;
2200                 authdata.algmode = OP_ALG_AAI_HASH;
2201                 session->auth_alg = RTE_CRYPTO_AUTH_SHA384;
2202                 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2203                                            1, 0, SHR_NEVER, &authdata,
2204                                            !session->dir,
2205                                            session->digest_length);
2206                 break;
2207         case RTE_CRYPTO_AUTH_SHA512:
2208                 authdata.algtype = OP_ALG_ALGSEL_SHA512;
2209                 authdata.algmode = OP_ALG_AAI_HASH;
2210                 session->auth_alg = RTE_CRYPTO_AUTH_SHA512;
2211                 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2212                                            1, 0, SHR_NEVER, &authdata,
2213                                            !session->dir,
2214                                            session->digest_length);
2215                 break;
2216         case RTE_CRYPTO_AUTH_SHA224:
2217                 authdata.algtype = OP_ALG_ALGSEL_SHA224;
2218                 authdata.algmode = OP_ALG_AAI_HASH;
2219                 session->auth_alg = RTE_CRYPTO_AUTH_SHA224;
2220                 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2221                                            1, 0, SHR_NEVER, &authdata,
2222                                            !session->dir,
2223                                            session->digest_length);
2224                 break;
2225         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2226                 authdata.algtype = OP_ALG_ALGSEL_AES;
2227                 authdata.algmode = OP_ALG_AAI_XCBC_MAC;
2228                 session->auth_alg = RTE_CRYPTO_AUTH_AES_XCBC_MAC;
2229                 bufsize = cnstr_shdsc_aes_mac(
2230                                         priv->flc_desc[DESC_INITFINAL].desc,
2231                                         1, 0, SHR_NEVER, &authdata,
2232                                         !session->dir,
2233                                         session->digest_length);
2234                 break;
2235         case RTE_CRYPTO_AUTH_AES_CMAC:
2236                 authdata.algtype = OP_ALG_ALGSEL_AES;
2237                 authdata.algmode = OP_ALG_AAI_CMAC;
2238                 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
2239                 bufsize = cnstr_shdsc_aes_mac(
2240                                            priv->flc_desc[DESC_INITFINAL].desc,
2241                                            1, 0, SHR_NEVER, &authdata,
2242                                            !session->dir,
2243                                            session->digest_length);
2244                 break;
2245         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2246         case RTE_CRYPTO_AUTH_AES_GMAC:
2247         case RTE_CRYPTO_AUTH_KASUMI_F9:
2248         case RTE_CRYPTO_AUTH_NULL:
2249                 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %un",
2250                               xform->auth.algo);
2251                 ret = -ENOTSUP;
2252                 goto error_out;
2253         default:
2254                 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
2255                               xform->auth.algo);
2256                 ret = -ENOTSUP;
2257                 goto error_out;
2258         }
2259
2260         if (bufsize < 0) {
2261                 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2262                 ret = -EINVAL;
2263                 goto error_out;
2264         }
2265
2266         flc->word1_sdl = (uint8_t)bufsize;
2267         session->ctxt = priv;
2268 #ifdef CAAM_DESC_DEBUG
2269         int i;
2270         for (i = 0; i < bufsize; i++)
2271                 DPAA2_SEC_DEBUG("DESC[%d]:0x%x",
2272                                 i, priv->flc_desc[DESC_INITFINAL].desc[i]);
2273 #endif
2274
2275         return ret;
2276
2277 error_out:
2278         rte_free(session->auth_key.data);
2279         rte_free(priv);
2280         return ret;
2281 }
2282
2283 static int
2284 dpaa2_sec_aead_init(struct rte_cryptodev *dev,
2285                     struct rte_crypto_sym_xform *xform,
2286                     dpaa2_sec_session *session)
2287 {
2288         struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt;
2289         struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
2290         struct alginfo aeaddata;
2291         int bufsize;
2292         struct ctxt_priv *priv;
2293         struct sec_flow_context *flc;
2294         struct rte_crypto_aead_xform *aead_xform = &xform->aead;
2295         int err, ret = 0;
2296
2297         PMD_INIT_FUNC_TRACE();
2298
2299         /* Set IV parameters */
2300         session->iv.offset = aead_xform->iv.offset;
2301         session->iv.length = aead_xform->iv.length;
2302         session->ctxt_type = DPAA2_SEC_AEAD;
2303
2304         /* For SEC AEAD only one descriptor is required */
2305         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2306                         sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
2307                         RTE_CACHE_LINE_SIZE);
2308         if (priv == NULL) {
2309                 DPAA2_SEC_ERR("No Memory for priv CTXT");
2310                 return -ENOMEM;
2311         }
2312
2313         priv->fle_pool = dev_priv->fle_pool;
2314         flc = &priv->flc_desc[0].flc;
2315
2316         session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length,
2317                                                RTE_CACHE_LINE_SIZE);
2318         if (session->aead_key.data == NULL && aead_xform->key.length > 0) {
2319                 DPAA2_SEC_ERR("No Memory for aead key");
2320                 rte_free(priv);
2321                 return -ENOMEM;
2322         }
2323         memcpy(session->aead_key.data, aead_xform->key.data,
2324                aead_xform->key.length);
2325
2326         session->digest_length = aead_xform->digest_length;
2327         session->aead_key.length = aead_xform->key.length;
2328         ctxt->auth_only_len = aead_xform->aad_length;
2329
2330         aeaddata.key = (size_t)session->aead_key.data;
2331         aeaddata.keylen = session->aead_key.length;
2332         aeaddata.key_enc_flags = 0;
2333         aeaddata.key_type = RTA_DATA_IMM;
2334
2335         switch (aead_xform->algo) {
2336         case RTE_CRYPTO_AEAD_AES_GCM:
2337                 aeaddata.algtype = OP_ALG_ALGSEL_AES;
2338                 aeaddata.algmode = OP_ALG_AAI_GCM;
2339                 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM;
2340                 break;
2341         case RTE_CRYPTO_AEAD_AES_CCM:
2342                 DPAA2_SEC_ERR("Crypto: Unsupported AEAD alg %u",
2343                               aead_xform->algo);
2344                 ret = -ENOTSUP;
2345                 goto error_out;
2346         default:
2347                 DPAA2_SEC_ERR("Crypto: Undefined AEAD specified %u",
2348                               aead_xform->algo);
2349                 ret = -ENOTSUP;
2350                 goto error_out;
2351         }
2352         session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
2353                                 DIR_ENC : DIR_DEC;
2354
2355         priv->flc_desc[0].desc[0] = aeaddata.keylen;
2356         err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
2357                                DESC_JOB_IO_LEN,
2358                                (unsigned int *)priv->flc_desc[0].desc,
2359                                &priv->flc_desc[0].desc[1], 1);
2360
2361         if (err < 0) {
2362                 DPAA2_SEC_ERR("Crypto: Incorrect key lengths");
2363                 ret = -EINVAL;
2364                 goto error_out;
2365         }
2366         if (priv->flc_desc[0].desc[1] & 1) {
2367                 aeaddata.key_type = RTA_DATA_IMM;
2368         } else {
2369                 aeaddata.key = DPAA2_VADDR_TO_IOVA(aeaddata.key);
2370                 aeaddata.key_type = RTA_DATA_PTR;
2371         }
2372         priv->flc_desc[0].desc[0] = 0;
2373         priv->flc_desc[0].desc[1] = 0;
2374
2375         if (session->dir == DIR_ENC)
2376                 bufsize = cnstr_shdsc_gcm_encap(
2377                                 priv->flc_desc[0].desc, 1, 0, SHR_NEVER,
2378                                 &aeaddata, session->iv.length,
2379                                 session->digest_length);
2380         else
2381                 bufsize = cnstr_shdsc_gcm_decap(
2382                                 priv->flc_desc[0].desc, 1, 0, SHR_NEVER,
2383                                 &aeaddata, session->iv.length,
2384                                 session->digest_length);
2385         if (bufsize < 0) {
2386                 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2387                 ret = -EINVAL;
2388                 goto error_out;
2389         }
2390
2391         flc->word1_sdl = (uint8_t)bufsize;
2392         session->ctxt = priv;
2393 #ifdef CAAM_DESC_DEBUG
2394         int i;
2395         for (i = 0; i < bufsize; i++)
2396                 DPAA2_SEC_DEBUG("DESC[%d]:0x%x\n",
2397                             i, priv->flc_desc[0].desc[i]);
2398 #endif
2399         return ret;
2400
2401 error_out:
2402         rte_free(session->aead_key.data);
2403         rte_free(priv);
2404         return ret;
2405 }
2406
2407
2408 static int
2409 dpaa2_sec_aead_chain_init(struct rte_cryptodev *dev,
2410                     struct rte_crypto_sym_xform *xform,
2411                     dpaa2_sec_session *session)
2412 {
2413         struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
2414         struct alginfo authdata, cipherdata;
2415         int bufsize;
2416         struct ctxt_priv *priv;
2417         struct sec_flow_context *flc;
2418         struct rte_crypto_cipher_xform *cipher_xform;
2419         struct rte_crypto_auth_xform *auth_xform;
2420         int err, ret = 0;
2421
2422         PMD_INIT_FUNC_TRACE();
2423
2424         if (session->ext_params.aead_ctxt.auth_cipher_text) {
2425                 cipher_xform = &xform->cipher;
2426                 auth_xform = &xform->next->auth;
2427                 session->ctxt_type =
2428                         (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2429                         DPAA2_SEC_CIPHER_HASH : DPAA2_SEC_HASH_CIPHER;
2430         } else {
2431                 cipher_xform = &xform->next->cipher;
2432                 auth_xform = &xform->auth;
2433                 session->ctxt_type =
2434                         (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2435                         DPAA2_SEC_HASH_CIPHER : DPAA2_SEC_CIPHER_HASH;
2436         }
2437
2438         /* Set IV parameters */
2439         session->iv.offset = cipher_xform->iv.offset;
2440         session->iv.length = cipher_xform->iv.length;
2441
2442         /* For SEC AEAD only one descriptor is required */
2443         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2444                         sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
2445                         RTE_CACHE_LINE_SIZE);
2446         if (priv == NULL) {
2447                 DPAA2_SEC_ERR("No Memory for priv CTXT");
2448                 return -ENOMEM;
2449         }
2450
2451         priv->fle_pool = dev_priv->fle_pool;
2452         flc = &priv->flc_desc[0].flc;
2453
2454         session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length,
2455                                                RTE_CACHE_LINE_SIZE);
2456         if (session->cipher_key.data == NULL && cipher_xform->key.length > 0) {
2457                 DPAA2_SEC_ERR("No Memory for cipher key");
2458                 rte_free(priv);
2459                 return -ENOMEM;
2460         }
2461         session->cipher_key.length = cipher_xform->key.length;
2462         session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length,
2463                                              RTE_CACHE_LINE_SIZE);
2464         if (session->auth_key.data == NULL && auth_xform->key.length > 0) {
2465                 DPAA2_SEC_ERR("No Memory for auth key");
2466                 rte_free(session->cipher_key.data);
2467                 rte_free(priv);
2468                 return -ENOMEM;
2469         }
2470         session->auth_key.length = auth_xform->key.length;
2471         memcpy(session->cipher_key.data, cipher_xform->key.data,
2472                cipher_xform->key.length);
2473         memcpy(session->auth_key.data, auth_xform->key.data,
2474                auth_xform->key.length);
2475
2476         authdata.key = (size_t)session->auth_key.data;
2477         authdata.keylen = session->auth_key.length;
2478         authdata.key_enc_flags = 0;
2479         authdata.key_type = RTA_DATA_IMM;
2480
2481         session->digest_length = auth_xform->digest_length;
2482
2483         switch (auth_xform->algo) {
2484         case RTE_CRYPTO_AUTH_SHA1_HMAC:
2485                 authdata.algtype = OP_ALG_ALGSEL_SHA1;
2486                 authdata.algmode = OP_ALG_AAI_HMAC;
2487                 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
2488                 break;
2489         case RTE_CRYPTO_AUTH_MD5_HMAC:
2490                 authdata.algtype = OP_ALG_ALGSEL_MD5;
2491                 authdata.algmode = OP_ALG_AAI_HMAC;
2492                 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
2493                 break;
2494         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2495                 authdata.algtype = OP_ALG_ALGSEL_SHA224;
2496                 authdata.algmode = OP_ALG_AAI_HMAC;
2497                 session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC;
2498                 break;
2499         case RTE_CRYPTO_AUTH_SHA256_HMAC:
2500                 authdata.algtype = OP_ALG_ALGSEL_SHA256;
2501                 authdata.algmode = OP_ALG_AAI_HMAC;
2502                 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
2503                 break;
2504         case RTE_CRYPTO_AUTH_SHA384_HMAC:
2505                 authdata.algtype = OP_ALG_ALGSEL_SHA384;
2506                 authdata.algmode = OP_ALG_AAI_HMAC;
2507                 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
2508                 break;
2509         case RTE_CRYPTO_AUTH_SHA512_HMAC:
2510                 authdata.algtype = OP_ALG_ALGSEL_SHA512;
2511                 authdata.algmode = OP_ALG_AAI_HMAC;
2512                 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
2513                 break;
2514         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2515                 authdata.algtype = OP_ALG_ALGSEL_AES;
2516                 authdata.algmode = OP_ALG_AAI_XCBC_MAC;
2517                 session->auth_alg = RTE_CRYPTO_AUTH_AES_XCBC_MAC;
2518                 break;
2519         case RTE_CRYPTO_AUTH_AES_CMAC:
2520                 authdata.algtype = OP_ALG_ALGSEL_AES;
2521                 authdata.algmode = OP_ALG_AAI_CMAC;
2522                 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
2523                 break;
2524         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2525         case RTE_CRYPTO_AUTH_AES_GMAC:
2526         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2527         case RTE_CRYPTO_AUTH_NULL:
2528         case RTE_CRYPTO_AUTH_SHA1:
2529         case RTE_CRYPTO_AUTH_SHA256:
2530         case RTE_CRYPTO_AUTH_SHA512:
2531         case RTE_CRYPTO_AUTH_SHA224:
2532         case RTE_CRYPTO_AUTH_SHA384:
2533         case RTE_CRYPTO_AUTH_MD5:
2534         case RTE_CRYPTO_AUTH_KASUMI_F9:
2535         case RTE_CRYPTO_AUTH_ZUC_EIA3:
2536                 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
2537                               auth_xform->algo);
2538                 ret = -ENOTSUP;
2539                 goto error_out;
2540         default:
2541                 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
2542                               auth_xform->algo);
2543                 ret = -ENOTSUP;
2544                 goto error_out;
2545         }
2546         cipherdata.key = (size_t)session->cipher_key.data;
2547         cipherdata.keylen = session->cipher_key.length;
2548         cipherdata.key_enc_flags = 0;
2549         cipherdata.key_type = RTA_DATA_IMM;
2550
2551         switch (cipher_xform->algo) {
2552         case RTE_CRYPTO_CIPHER_AES_CBC:
2553                 cipherdata.algtype = OP_ALG_ALGSEL_AES;
2554                 cipherdata.algmode = OP_ALG_AAI_CBC;
2555                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
2556                 break;
2557         case RTE_CRYPTO_CIPHER_3DES_CBC:
2558                 cipherdata.algtype = OP_ALG_ALGSEL_3DES;
2559                 cipherdata.algmode = OP_ALG_AAI_CBC;
2560                 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
2561                 break;
2562         case RTE_CRYPTO_CIPHER_DES_CBC:
2563                 cipherdata.algtype = OP_ALG_ALGSEL_DES;
2564                 cipherdata.algmode = OP_ALG_AAI_CBC;
2565                 session->cipher_alg = RTE_CRYPTO_CIPHER_DES_CBC;
2566                 break;
2567         case RTE_CRYPTO_CIPHER_AES_CTR:
2568                 cipherdata.algtype = OP_ALG_ALGSEL_AES;
2569                 cipherdata.algmode = OP_ALG_AAI_CTR;
2570                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
2571                 break;
2572         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2573         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2574         case RTE_CRYPTO_CIPHER_NULL:
2575         case RTE_CRYPTO_CIPHER_3DES_ECB:
2576         case RTE_CRYPTO_CIPHER_3DES_CTR:
2577         case RTE_CRYPTO_CIPHER_AES_ECB:
2578         case RTE_CRYPTO_CIPHER_KASUMI_F8:
2579                 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2580                               cipher_xform->algo);
2581                 ret = -ENOTSUP;
2582                 goto error_out;
2583         default:
2584                 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
2585                               cipher_xform->algo);
2586                 ret = -ENOTSUP;
2587                 goto error_out;
2588         }
2589         session->dir = (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2590                                 DIR_ENC : DIR_DEC;
2591
2592         priv->flc_desc[0].desc[0] = cipherdata.keylen;
2593         priv->flc_desc[0].desc[1] = authdata.keylen;
2594         err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
2595                                DESC_JOB_IO_LEN,
2596                                (unsigned int *)priv->flc_desc[0].desc,
2597                                &priv->flc_desc[0].desc[2], 2);
2598
2599         if (err < 0) {
2600                 DPAA2_SEC_ERR("Crypto: Incorrect key lengths");
2601                 ret = -EINVAL;
2602                 goto error_out;
2603         }
2604         if (priv->flc_desc[0].desc[2] & 1) {
2605                 cipherdata.key_type = RTA_DATA_IMM;
2606         } else {
2607                 cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key);
2608                 cipherdata.key_type = RTA_DATA_PTR;
2609         }
2610         if (priv->flc_desc[0].desc[2] & (1 << 1)) {
2611                 authdata.key_type = RTA_DATA_IMM;
2612         } else {
2613                 authdata.key = DPAA2_VADDR_TO_IOVA(authdata.key);
2614                 authdata.key_type = RTA_DATA_PTR;
2615         }
2616         priv->flc_desc[0].desc[0] = 0;
2617         priv->flc_desc[0].desc[1] = 0;
2618         priv->flc_desc[0].desc[2] = 0;
2619
2620         if (session->ctxt_type == DPAA2_SEC_CIPHER_HASH) {
2621                 bufsize = cnstr_shdsc_authenc(priv->flc_desc[0].desc, 1,
2622                                               0, SHR_SERIAL,
2623                                               &cipherdata, &authdata,
2624                                               session->iv.length,
2625                                               session->digest_length,
2626                                               session->dir);
2627                 if (bufsize < 0) {
2628                         DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2629                         ret = -EINVAL;
2630                         goto error_out;
2631                 }
2632         } else {
2633                 DPAA2_SEC_ERR("Hash before cipher not supported");
2634                 ret = -ENOTSUP;
2635                 goto error_out;
2636         }
2637
2638         flc->word1_sdl = (uint8_t)bufsize;
2639         session->ctxt = priv;
2640 #ifdef CAAM_DESC_DEBUG
2641         int i;
2642         for (i = 0; i < bufsize; i++)
2643                 DPAA2_SEC_DEBUG("DESC[%d]:0x%x",
2644                             i, priv->flc_desc[0].desc[i]);
2645 #endif
2646
2647         return ret;
2648
2649 error_out:
2650         rte_free(session->cipher_key.data);
2651         rte_free(session->auth_key.data);
2652         rte_free(priv);
2653         return ret;
2654 }
2655
2656 static int
2657 dpaa2_sec_set_session_parameters(struct rte_cryptodev *dev,
2658                             struct rte_crypto_sym_xform *xform, void *sess)
2659 {
2660         dpaa2_sec_session *session = sess;
2661         int ret;
2662
2663         PMD_INIT_FUNC_TRACE();
2664
2665         if (unlikely(sess == NULL)) {
2666                 DPAA2_SEC_ERR("Invalid session struct");
2667                 return -EINVAL;
2668         }
2669
2670         memset(session, 0, sizeof(dpaa2_sec_session));
2671         /* Default IV length = 0 */
2672         session->iv.length = 0;
2673
2674         /* Cipher Only */
2675         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
2676                 ret = dpaa2_sec_cipher_init(dev, xform, session);
2677
2678         /* Authentication Only */
2679         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
2680                    xform->next == NULL) {
2681                 ret = dpaa2_sec_auth_init(dev, xform, session);
2682
2683         /* Cipher then Authenticate */
2684         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
2685                    xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
2686                 session->ext_params.aead_ctxt.auth_cipher_text = true;
2687                 if (xform->cipher.algo == RTE_CRYPTO_CIPHER_NULL)
2688                         ret = dpaa2_sec_auth_init(dev, xform, session);
2689                 else if (xform->next->auth.algo == RTE_CRYPTO_AUTH_NULL)
2690                         ret = dpaa2_sec_cipher_init(dev, xform, session);
2691                 else
2692                         ret = dpaa2_sec_aead_chain_init(dev, xform, session);
2693         /* Authenticate then Cipher */
2694         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
2695                    xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
2696                 session->ext_params.aead_ctxt.auth_cipher_text = false;
2697                 if (xform->auth.algo == RTE_CRYPTO_AUTH_NULL)
2698                         ret = dpaa2_sec_cipher_init(dev, xform, session);
2699                 else if (xform->next->cipher.algo == RTE_CRYPTO_CIPHER_NULL)
2700                         ret = dpaa2_sec_auth_init(dev, xform, session);
2701                 else
2702                         ret = dpaa2_sec_aead_chain_init(dev, xform, session);
2703         /* AEAD operation for AES-GCM kind of Algorithms */
2704         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
2705                    xform->next == NULL) {
2706                 ret = dpaa2_sec_aead_init(dev, xform, session);
2707
2708         } else {
2709                 DPAA2_SEC_ERR("Invalid crypto type");
2710                 return -EINVAL;
2711         }
2712
2713         return ret;
2714 }
2715
2716 #ifdef RTE_LIB_SECURITY
2717 static int
2718 dpaa2_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform,
2719                         dpaa2_sec_session *session,
2720                         struct alginfo *aeaddata)
2721 {
2722         PMD_INIT_FUNC_TRACE();
2723
2724         session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length,
2725                                                RTE_CACHE_LINE_SIZE);
2726         if (session->aead_key.data == NULL && aead_xform->key.length > 0) {
2727                 DPAA2_SEC_ERR("No Memory for aead key");
2728                 return -ENOMEM;
2729         }
2730         memcpy(session->aead_key.data, aead_xform->key.data,
2731                aead_xform->key.length);
2732
2733         session->digest_length = aead_xform->digest_length;
2734         session->aead_key.length = aead_xform->key.length;
2735
2736         aeaddata->key = (size_t)session->aead_key.data;
2737         aeaddata->keylen = session->aead_key.length;
2738         aeaddata->key_enc_flags = 0;
2739         aeaddata->key_type = RTA_DATA_IMM;
2740
2741         switch (aead_xform->algo) {
2742         case RTE_CRYPTO_AEAD_AES_GCM:
2743                 switch (session->digest_length) {
2744                 case 8:
2745                         aeaddata->algtype = OP_PCL_IPSEC_AES_GCM8;
2746                         break;
2747                 case 12:
2748                         aeaddata->algtype = OP_PCL_IPSEC_AES_GCM12;
2749                         break;
2750                 case 16:
2751                         aeaddata->algtype = OP_PCL_IPSEC_AES_GCM16;
2752                         break;
2753                 default:
2754                         DPAA2_SEC_ERR("Crypto: Undefined GCM digest %d",
2755                                       session->digest_length);
2756                         return -EINVAL;
2757                 }
2758                 aeaddata->algmode = OP_ALG_AAI_GCM;
2759                 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM;
2760                 break;
2761         case RTE_CRYPTO_AEAD_AES_CCM:
2762                 switch (session->digest_length) {
2763                 case 8:
2764                         aeaddata->algtype = OP_PCL_IPSEC_AES_CCM8;
2765                         break;
2766                 case 12:
2767                         aeaddata->algtype = OP_PCL_IPSEC_AES_CCM12;
2768                         break;
2769                 case 16:
2770                         aeaddata->algtype = OP_PCL_IPSEC_AES_CCM16;
2771                         break;
2772                 default:
2773                         DPAA2_SEC_ERR("Crypto: Undefined CCM digest %d",
2774                                       session->digest_length);
2775                         return -EINVAL;
2776                 }
2777                 aeaddata->algmode = OP_ALG_AAI_CCM;
2778                 session->aead_alg = RTE_CRYPTO_AEAD_AES_CCM;
2779                 break;
2780         default:
2781                 DPAA2_SEC_ERR("Crypto: Undefined AEAD specified %u",
2782                               aead_xform->algo);
2783                 return -ENOTSUP;
2784         }
2785         session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
2786                                 DIR_ENC : DIR_DEC;
2787
2788         return 0;
2789 }
2790
2791 static int
2792 dpaa2_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform,
2793         struct rte_crypto_auth_xform *auth_xform,
2794         dpaa2_sec_session *session,
2795         struct alginfo *cipherdata,
2796         struct alginfo *authdata)
2797 {
2798         if (cipher_xform) {
2799                 session->cipher_key.data = rte_zmalloc(NULL,
2800                                                        cipher_xform->key.length,
2801                                                        RTE_CACHE_LINE_SIZE);
2802                 if (session->cipher_key.data == NULL &&
2803                                 cipher_xform->key.length > 0) {
2804                         DPAA2_SEC_ERR("No Memory for cipher key");
2805                         return -ENOMEM;
2806                 }
2807
2808                 session->cipher_key.length = cipher_xform->key.length;
2809                 memcpy(session->cipher_key.data, cipher_xform->key.data,
2810                                 cipher_xform->key.length);
2811                 session->cipher_alg = cipher_xform->algo;
2812         } else {
2813                 session->cipher_key.data = NULL;
2814                 session->cipher_key.length = 0;
2815                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
2816         }
2817
2818         if (auth_xform) {
2819                 session->auth_key.data = rte_zmalloc(NULL,
2820                                                 auth_xform->key.length,
2821                                                 RTE_CACHE_LINE_SIZE);
2822                 if (session->auth_key.data == NULL &&
2823                                 auth_xform->key.length > 0) {
2824                         DPAA2_SEC_ERR("No Memory for auth key");
2825                         return -ENOMEM;
2826                 }
2827                 session->auth_key.length = auth_xform->key.length;
2828                 memcpy(session->auth_key.data, auth_xform->key.data,
2829                                 auth_xform->key.length);
2830                 session->auth_alg = auth_xform->algo;
2831                 session->digest_length = auth_xform->digest_length;
2832         } else {
2833                 session->auth_key.data = NULL;
2834                 session->auth_key.length = 0;
2835                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
2836         }
2837
2838         authdata->key = (size_t)session->auth_key.data;
2839         authdata->keylen = session->auth_key.length;
2840         authdata->key_enc_flags = 0;
2841         authdata->key_type = RTA_DATA_IMM;
2842         switch (session->auth_alg) {
2843         case RTE_CRYPTO_AUTH_SHA1_HMAC:
2844                 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA1_96;
2845                 authdata->algmode = OP_ALG_AAI_HMAC;
2846                 break;
2847         case RTE_CRYPTO_AUTH_MD5_HMAC:
2848                 authdata->algtype = OP_PCL_IPSEC_HMAC_MD5_96;
2849                 authdata->algmode = OP_ALG_AAI_HMAC;
2850                 break;
2851         case RTE_CRYPTO_AUTH_SHA256_HMAC:
2852                 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_256_128;
2853                 authdata->algmode = OP_ALG_AAI_HMAC;
2854                 if (session->digest_length != 16)
2855                         DPAA2_SEC_WARN(
2856                         "+++Using sha256-hmac truncated len is non-standard,"
2857                         "it will not work with lookaside proto");
2858                 break;
2859         case RTE_CRYPTO_AUTH_SHA384_HMAC:
2860                 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_384_192;
2861                 authdata->algmode = OP_ALG_AAI_HMAC;
2862                 break;
2863         case RTE_CRYPTO_AUTH_SHA512_HMAC:
2864                 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_512_256;
2865                 authdata->algmode = OP_ALG_AAI_HMAC;
2866                 break;
2867         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2868                 authdata->algtype = OP_PCL_IPSEC_AES_XCBC_MAC_96;
2869                 authdata->algmode = OP_ALG_AAI_XCBC_MAC;
2870                 break;
2871         case RTE_CRYPTO_AUTH_AES_CMAC:
2872                 authdata->algtype = OP_PCL_IPSEC_AES_CMAC_96;
2873                 authdata->algmode = OP_ALG_AAI_CMAC;
2874                 break;
2875         case RTE_CRYPTO_AUTH_NULL:
2876                 authdata->algtype = OP_PCL_IPSEC_HMAC_NULL;
2877                 break;
2878         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2879         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2880         case RTE_CRYPTO_AUTH_SHA1:
2881         case RTE_CRYPTO_AUTH_SHA256:
2882         case RTE_CRYPTO_AUTH_SHA512:
2883         case RTE_CRYPTO_AUTH_SHA224:
2884         case RTE_CRYPTO_AUTH_SHA384:
2885         case RTE_CRYPTO_AUTH_MD5:
2886         case RTE_CRYPTO_AUTH_AES_GMAC:
2887         case RTE_CRYPTO_AUTH_KASUMI_F9:
2888         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2889         case RTE_CRYPTO_AUTH_ZUC_EIA3:
2890                 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
2891                               session->auth_alg);
2892                 return -ENOTSUP;
2893         default:
2894                 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
2895                               session->auth_alg);
2896                 return -ENOTSUP;
2897         }
2898         cipherdata->key = (size_t)session->cipher_key.data;
2899         cipherdata->keylen = session->cipher_key.length;
2900         cipherdata->key_enc_flags = 0;
2901         cipherdata->key_type = RTA_DATA_IMM;
2902
2903         switch (session->cipher_alg) {
2904         case RTE_CRYPTO_CIPHER_AES_CBC:
2905                 cipherdata->algtype = OP_PCL_IPSEC_AES_CBC;
2906                 cipherdata->algmode = OP_ALG_AAI_CBC;
2907                 break;
2908         case RTE_CRYPTO_CIPHER_3DES_CBC:
2909                 cipherdata->algtype = OP_PCL_IPSEC_3DES;
2910                 cipherdata->algmode = OP_ALG_AAI_CBC;
2911                 break;
2912         case RTE_CRYPTO_CIPHER_DES_CBC:
2913                 cipherdata->algtype = OP_PCL_IPSEC_DES;
2914                 cipherdata->algmode = OP_ALG_AAI_CBC;
2915                 break;
2916         case RTE_CRYPTO_CIPHER_AES_CTR:
2917                 cipherdata->algtype = OP_PCL_IPSEC_AES_CTR;
2918                 cipherdata->algmode = OP_ALG_AAI_CTR;
2919                 break;
2920         case RTE_CRYPTO_CIPHER_NULL:
2921                 cipherdata->algtype = OP_PCL_IPSEC_NULL;
2922                 break;
2923         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2924         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2925         case RTE_CRYPTO_CIPHER_3DES_ECB:
2926         case RTE_CRYPTO_CIPHER_3DES_CTR:
2927         case RTE_CRYPTO_CIPHER_AES_ECB:
2928         case RTE_CRYPTO_CIPHER_KASUMI_F8:
2929                 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2930                               session->cipher_alg);
2931                 return -ENOTSUP;
2932         default:
2933                 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
2934                               session->cipher_alg);
2935                 return -ENOTSUP;
2936         }
2937
2938         return 0;
2939 }
2940
2941 static int
2942 dpaa2_sec_set_ipsec_session(struct rte_cryptodev *dev,
2943                             struct rte_security_session_conf *conf,
2944                             void *sess)
2945 {
2946         struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
2947         struct rte_crypto_cipher_xform *cipher_xform = NULL;
2948         struct rte_crypto_auth_xform *auth_xform = NULL;
2949         struct rte_crypto_aead_xform *aead_xform = NULL;
2950         dpaa2_sec_session *session = (dpaa2_sec_session *)sess;
2951         struct ctxt_priv *priv;
2952         struct alginfo authdata, cipherdata;
2953         int bufsize;
2954         struct sec_flow_context *flc;
2955         struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
2956         int ret = -1;
2957
2958         PMD_INIT_FUNC_TRACE();
2959
2960         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2961                                 sizeof(struct ctxt_priv) +
2962                                 sizeof(struct sec_flc_desc),
2963                                 RTE_CACHE_LINE_SIZE);
2964
2965         if (priv == NULL) {
2966                 DPAA2_SEC_ERR("No memory for priv CTXT");
2967                 return -ENOMEM;
2968         }
2969
2970         priv->fle_pool = dev_priv->fle_pool;
2971         flc = &priv->flc_desc[0].flc;
2972
2973         if (ipsec_xform->life.bytes_hard_limit != 0 ||
2974             ipsec_xform->life.bytes_soft_limit != 0 ||
2975             ipsec_xform->life.packets_hard_limit != 0 ||
2976             ipsec_xform->life.packets_soft_limit != 0)
2977                 return -ENOTSUP;
2978
2979         memset(session, 0, sizeof(dpaa2_sec_session));
2980
2981         if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
2982                 cipher_xform = &conf->crypto_xform->cipher;
2983                 if (conf->crypto_xform->next)
2984                         auth_xform = &conf->crypto_xform->next->auth;
2985                 ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform,
2986                                         session, &cipherdata, &authdata);
2987         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
2988                 auth_xform = &conf->crypto_xform->auth;
2989                 if (conf->crypto_xform->next)
2990                         cipher_xform = &conf->crypto_xform->next->cipher;
2991                 ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform,
2992                                         session, &cipherdata, &authdata);
2993         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
2994                 aead_xform = &conf->crypto_xform->aead;
2995                 ret = dpaa2_sec_ipsec_aead_init(aead_xform,
2996                                         session, &cipherdata);
2997                 authdata.keylen = 0;
2998                 authdata.algtype = 0;
2999         } else {
3000                 DPAA2_SEC_ERR("XFORM not specified");
3001                 ret = -EINVAL;
3002                 goto out;
3003         }
3004         if (ret) {
3005                 DPAA2_SEC_ERR("Failed to process xform");
3006                 goto out;
3007         }
3008
3009         session->ctxt_type = DPAA2_SEC_IPSEC;
3010         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
3011                 uint8_t *hdr = NULL;
3012                 struct ip ip4_hdr;
3013                 struct rte_ipv6_hdr ip6_hdr;
3014                 struct ipsec_encap_pdb encap_pdb;
3015
3016                 flc->dhr = SEC_FLC_DHR_OUTBOUND;
3017                 /* For Sec Proto only one descriptor is required. */
3018                 memset(&encap_pdb, 0, sizeof(struct ipsec_encap_pdb));
3019
3020                 /* copy algo specific data to PDB */
3021                 switch (cipherdata.algtype) {
3022                 case OP_PCL_IPSEC_AES_CTR:
3023                         encap_pdb.ctr.ctr_initial = 0x00000001;
3024                         encap_pdb.ctr.ctr_nonce = ipsec_xform->salt;
3025                         break;
3026                 case OP_PCL_IPSEC_AES_GCM8:
3027                 case OP_PCL_IPSEC_AES_GCM12:
3028                 case OP_PCL_IPSEC_AES_GCM16:
3029                         memcpy(encap_pdb.gcm.salt,
3030                                 (uint8_t *)&(ipsec_xform->salt), 4);
3031                         break;
3032                 }
3033
3034                 encap_pdb.options = (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
3035                         PDBOPTS_ESP_OIHI_PDB_INL |
3036                         PDBOPTS_ESP_IVSRC |
3037                         PDBHMO_ESP_SNR;
3038                 if (ipsec_xform->options.dec_ttl)
3039                         encap_pdb.options |= PDBHMO_ESP_ENCAP_DTTL;
3040                 if (ipsec_xform->options.esn)
3041                         encap_pdb.options |= PDBOPTS_ESP_ESN;
3042                 encap_pdb.spi = ipsec_xform->spi;
3043                 session->dir = DIR_ENC;
3044                 if (ipsec_xform->tunnel.type ==
3045                                 RTE_SECURITY_IPSEC_TUNNEL_IPV4) {
3046                         encap_pdb.ip_hdr_len = sizeof(struct ip);
3047                         ip4_hdr.ip_v = IPVERSION;
3048                         ip4_hdr.ip_hl = 5;
3049                         ip4_hdr.ip_len = rte_cpu_to_be_16(sizeof(ip4_hdr));
3050                         ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
3051                         ip4_hdr.ip_id = 0;
3052                         ip4_hdr.ip_off = 0;
3053                         ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
3054                         ip4_hdr.ip_p = IPPROTO_ESP;
3055                         ip4_hdr.ip_sum = 0;
3056                         ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
3057                         ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
3058                         ip4_hdr.ip_sum = calc_chksum((uint16_t *)(void *)
3059                                         &ip4_hdr, sizeof(struct ip));
3060                         hdr = (uint8_t *)&ip4_hdr;
3061                 } else if (ipsec_xform->tunnel.type ==
3062                                 RTE_SECURITY_IPSEC_TUNNEL_IPV6) {
3063                         ip6_hdr.vtc_flow = rte_cpu_to_be_32(
3064                                 DPAA2_IPv6_DEFAULT_VTC_FLOW |
3065                                 ((ipsec_xform->tunnel.ipv6.dscp <<
3066                                         RTE_IPV6_HDR_TC_SHIFT) &
3067                                         RTE_IPV6_HDR_TC_MASK) |
3068                                 ((ipsec_xform->tunnel.ipv6.flabel <<
3069                                         RTE_IPV6_HDR_FL_SHIFT) &
3070                                         RTE_IPV6_HDR_FL_MASK));
3071                         /* Payload length will be updated by HW */
3072                         ip6_hdr.payload_len = 0;
3073                         ip6_hdr.hop_limits =
3074                                         ipsec_xform->tunnel.ipv6.hlimit;
3075                         ip6_hdr.proto = (ipsec_xform->proto ==
3076                                         RTE_SECURITY_IPSEC_SA_PROTO_ESP) ?
3077                                         IPPROTO_ESP : IPPROTO_AH;
3078                         memcpy(&ip6_hdr.src_addr,
3079                                 &ipsec_xform->tunnel.ipv6.src_addr, 16);
3080                         memcpy(&ip6_hdr.dst_addr,
3081                                 &ipsec_xform->tunnel.ipv6.dst_addr, 16);
3082                         encap_pdb.ip_hdr_len = sizeof(struct rte_ipv6_hdr);
3083                         hdr = (uint8_t *)&ip6_hdr;
3084                 }
3085
3086                 bufsize = cnstr_shdsc_ipsec_new_encap(priv->flc_desc[0].desc,
3087                                 1, 0, (rta_sec_era >= RTA_SEC_ERA_10) ?
3088                                 SHR_WAIT : SHR_SERIAL, &encap_pdb,
3089                                 hdr, &cipherdata, &authdata);
3090         } else if (ipsec_xform->direction ==
3091                         RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
3092                 struct ipsec_decap_pdb decap_pdb;
3093
3094                 flc->dhr = SEC_FLC_DHR_INBOUND;
3095                 memset(&decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
3096                 /* copy algo specific data to PDB */
3097                 switch (cipherdata.algtype) {
3098                 case OP_PCL_IPSEC_AES_CTR:
3099                         decap_pdb.ctr.ctr_initial = 0x00000001;
3100                         decap_pdb.ctr.ctr_nonce = ipsec_xform->salt;
3101                         break;
3102                 case OP_PCL_IPSEC_AES_GCM8:
3103                 case OP_PCL_IPSEC_AES_GCM12:
3104                 case OP_PCL_IPSEC_AES_GCM16:
3105                         memcpy(decap_pdb.gcm.salt,
3106                                 (uint8_t *)&(ipsec_xform->salt), 4);
3107                         break;
3108                 }
3109
3110                 decap_pdb.options = (ipsec_xform->tunnel.type ==
3111                                 RTE_SECURITY_IPSEC_TUNNEL_IPV4) ?
3112                                 sizeof(struct ip) << 16 :
3113                                 sizeof(struct rte_ipv6_hdr) << 16;
3114                 if (ipsec_xform->options.esn)
3115                         decap_pdb.options |= PDBOPTS_ESP_ESN;
3116
3117                 if (ipsec_xform->replay_win_sz) {
3118                         uint32_t win_sz;
3119                         win_sz = rte_align32pow2(ipsec_xform->replay_win_sz);
3120
3121                         if (rta_sec_era < RTA_SEC_ERA_10 && win_sz > 128) {
3122                                 DPAA2_SEC_INFO("Max Anti replay Win sz = 128");
3123                                 win_sz = 128;
3124                         }
3125                         switch (win_sz) {
3126                         case 1:
3127                         case 2:
3128                         case 4:
3129                         case 8:
3130                         case 16:
3131                         case 32:
3132                                 decap_pdb.options |= PDBOPTS_ESP_ARS32;
3133                                 break;
3134                         case 64:
3135                                 decap_pdb.options |= PDBOPTS_ESP_ARS64;
3136                                 break;
3137                         case 256:
3138                                 decap_pdb.options |= PDBOPTS_ESP_ARS256;
3139                                 break;
3140                         case 512:
3141                                 decap_pdb.options |= PDBOPTS_ESP_ARS512;
3142                                 break;
3143                         case 1024:
3144                                 decap_pdb.options |= PDBOPTS_ESP_ARS1024;
3145                                 break;
3146                         case 128:
3147                         default:
3148                                 decap_pdb.options |= PDBOPTS_ESP_ARS128;
3149                         }
3150                 }
3151                 session->dir = DIR_DEC;
3152                 bufsize = cnstr_shdsc_ipsec_new_decap(priv->flc_desc[0].desc,
3153                                 1, 0, (rta_sec_era >= RTA_SEC_ERA_10) ?
3154                                 SHR_WAIT : SHR_SERIAL,
3155                                 &decap_pdb, &cipherdata, &authdata);
3156         } else
3157                 goto out;
3158
3159         if (bufsize < 0) {
3160                 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
3161                 goto out;
3162         }
3163
3164         flc->word1_sdl = (uint8_t)bufsize;
3165
3166         /* Enable the stashing control bit */
3167         DPAA2_SET_FLC_RSC(flc);
3168         flc->word2_rflc_31_0 = lower_32_bits(
3169                         (size_t)&(((struct dpaa2_sec_qp *)
3170                         dev->data->queue_pairs[0])->rx_vq) | 0x14);
3171         flc->word3_rflc_63_32 = upper_32_bits(
3172                         (size_t)&(((struct dpaa2_sec_qp *)
3173                         dev->data->queue_pairs[0])->rx_vq));
3174
3175         /* Set EWS bit i.e. enable write-safe */
3176         DPAA2_SET_FLC_EWS(flc);
3177         /* Set BS = 1 i.e reuse input buffers as output buffers */
3178         DPAA2_SET_FLC_REUSE_BS(flc);
3179         /* Set FF = 10; reuse input buffers if they provide sufficient space */
3180         DPAA2_SET_FLC_REUSE_FF(flc);
3181
3182         session->ctxt = priv;
3183
3184         return 0;
3185 out:
3186         rte_free(session->auth_key.data);
3187         rte_free(session->cipher_key.data);
3188         rte_free(priv);
3189         return ret;
3190 }
3191
3192 static int
3193 dpaa2_sec_set_pdcp_session(struct rte_cryptodev *dev,
3194                            struct rte_security_session_conf *conf,
3195                            void *sess)
3196 {
3197         struct rte_security_pdcp_xform *pdcp_xform = &conf->pdcp;
3198         struct rte_crypto_sym_xform *xform = conf->crypto_xform;
3199         struct rte_crypto_auth_xform *auth_xform = NULL;
3200         struct rte_crypto_cipher_xform *cipher_xform = NULL;
3201         dpaa2_sec_session *session = (dpaa2_sec_session *)sess;
3202         struct ctxt_priv *priv;
3203         struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
3204         struct alginfo authdata, cipherdata;
3205         struct alginfo *p_authdata = NULL;
3206         int bufsize = -1;
3207         struct sec_flow_context *flc;
3208 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
3209         int swap = true;
3210 #else
3211         int swap = false;
3212 #endif
3213
3214         PMD_INIT_FUNC_TRACE();
3215
3216         memset(session, 0, sizeof(dpaa2_sec_session));
3217
3218         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
3219                                 sizeof(struct ctxt_priv) +
3220                                 sizeof(struct sec_flc_desc),
3221                                 RTE_CACHE_LINE_SIZE);
3222
3223         if (priv == NULL) {
3224                 DPAA2_SEC_ERR("No memory for priv CTXT");
3225                 return -ENOMEM;
3226         }
3227
3228         priv->fle_pool = dev_priv->fle_pool;
3229         flc = &priv->flc_desc[0].flc;
3230
3231         /* find xfrm types */
3232         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
3233                 cipher_xform = &xform->cipher;
3234                 if (xform->next != NULL) {
3235                         session->ext_params.aead_ctxt.auth_cipher_text = true;
3236                         auth_xform = &xform->next->auth;
3237                 }
3238         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
3239                 auth_xform = &xform->auth;
3240                 if (xform->next != NULL) {
3241                         session->ext_params.aead_ctxt.auth_cipher_text = false;
3242                         cipher_xform = &xform->next->cipher;
3243                 }
3244         } else {
3245                 DPAA2_SEC_ERR("Invalid crypto type");
3246                 return -EINVAL;
3247         }
3248
3249         session->ctxt_type = DPAA2_SEC_PDCP;
3250         if (cipher_xform) {
3251                 session->cipher_key.data = rte_zmalloc(NULL,
3252                                                cipher_xform->key.length,
3253                                                RTE_CACHE_LINE_SIZE);
3254                 if (session->cipher_key.data == NULL &&
3255                                 cipher_xform->key.length > 0) {
3256                         DPAA2_SEC_ERR("No Memory for cipher key");
3257                         rte_free(priv);
3258                         return -ENOMEM;
3259                 }
3260                 session->cipher_key.length = cipher_xform->key.length;
3261                 memcpy(session->cipher_key.data, cipher_xform->key.data,
3262                         cipher_xform->key.length);
3263                 session->dir =
3264                         (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
3265                                         DIR_ENC : DIR_DEC;
3266                 session->cipher_alg = cipher_xform->algo;
3267         } else {
3268                 session->cipher_key.data = NULL;
3269                 session->cipher_key.length = 0;
3270                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
3271                 session->dir = DIR_ENC;
3272         }
3273
3274         session->pdcp.domain = pdcp_xform->domain;
3275         session->pdcp.bearer = pdcp_xform->bearer;
3276         session->pdcp.pkt_dir = pdcp_xform->pkt_dir;
3277         session->pdcp.sn_size = pdcp_xform->sn_size;
3278         session->pdcp.hfn = pdcp_xform->hfn;
3279         session->pdcp.hfn_threshold = pdcp_xform->hfn_threshold;
3280         session->pdcp.hfn_ovd = pdcp_xform->hfn_ovrd;
3281         /* hfv ovd offset location is stored in iv.offset value*/
3282         if (cipher_xform)
3283                 session->pdcp.hfn_ovd_offset = cipher_xform->iv.offset;
3284
3285         cipherdata.key = (size_t)session->cipher_key.data;
3286         cipherdata.keylen = session->cipher_key.length;
3287         cipherdata.key_enc_flags = 0;
3288         cipherdata.key_type = RTA_DATA_IMM;
3289
3290         switch (session->cipher_alg) {
3291         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
3292                 cipherdata.algtype = PDCP_CIPHER_TYPE_SNOW;
3293                 break;
3294         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
3295                 cipherdata.algtype = PDCP_CIPHER_TYPE_ZUC;
3296                 break;
3297         case RTE_CRYPTO_CIPHER_AES_CTR:
3298                 cipherdata.algtype = PDCP_CIPHER_TYPE_AES;
3299                 break;
3300         case RTE_CRYPTO_CIPHER_NULL:
3301                 cipherdata.algtype = PDCP_CIPHER_TYPE_NULL;
3302                 break;
3303         default:
3304                 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
3305                               session->cipher_alg);
3306                 goto out;
3307         }
3308
3309         if (auth_xform) {
3310                 session->auth_key.data = rte_zmalloc(NULL,
3311                                                      auth_xform->key.length,
3312                                                      RTE_CACHE_LINE_SIZE);
3313                 if (!session->auth_key.data &&
3314                     auth_xform->key.length > 0) {
3315                         DPAA2_SEC_ERR("No Memory for auth key");
3316                         rte_free(session->cipher_key.data);
3317                         rte_free(priv);
3318                         return -ENOMEM;
3319                 }
3320                 session->auth_key.length = auth_xform->key.length;
3321                 memcpy(session->auth_key.data, auth_xform->key.data,
3322                        auth_xform->key.length);
3323                 session->auth_alg = auth_xform->algo;
3324         } else {
3325                 session->auth_key.data = NULL;
3326                 session->auth_key.length = 0;
3327                 session->auth_alg = 0;
3328         }
3329         authdata.key = (size_t)session->auth_key.data;
3330         authdata.keylen = session->auth_key.length;
3331         authdata.key_enc_flags = 0;
3332         authdata.key_type = RTA_DATA_IMM;
3333
3334         if (session->auth_alg) {
3335                 switch (session->auth_alg) {
3336                 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
3337                         authdata.algtype = PDCP_AUTH_TYPE_SNOW;
3338                         break;
3339                 case RTE_CRYPTO_AUTH_ZUC_EIA3:
3340                         authdata.algtype = PDCP_AUTH_TYPE_ZUC;
3341                         break;
3342                 case RTE_CRYPTO_AUTH_AES_CMAC:
3343                         authdata.algtype = PDCP_AUTH_TYPE_AES;
3344                         break;
3345                 case RTE_CRYPTO_AUTH_NULL:
3346                         authdata.algtype = PDCP_AUTH_TYPE_NULL;
3347                         break;
3348                 default:
3349                         DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
3350                                       session->auth_alg);
3351                         goto out;
3352                 }
3353
3354                 p_authdata = &authdata;
3355         } else if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
3356                 DPAA2_SEC_ERR("Crypto: Integrity must for c-plane");
3357                 goto out;
3358         }
3359
3360         if (pdcp_xform->sdap_enabled) {
3361                 int nb_keys_to_inline =
3362                         rta_inline_pdcp_sdap_query(authdata.algtype,
3363                                         cipherdata.algtype,
3364                                         session->pdcp.sn_size,
3365                                         session->pdcp.hfn_ovd);
3366                 if (nb_keys_to_inline >= 1) {
3367                         cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key);
3368                         cipherdata.key_type = RTA_DATA_PTR;
3369                 }
3370                 if (nb_keys_to_inline >= 2) {
3371                         authdata.key = DPAA2_VADDR_TO_IOVA(authdata.key);
3372                         authdata.key_type = RTA_DATA_PTR;
3373                 }
3374         } else {
3375                 if (rta_inline_pdcp_query(authdata.algtype,
3376                                         cipherdata.algtype,
3377                                         session->pdcp.sn_size,
3378                                         session->pdcp.hfn_ovd)) {
3379                         cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key);
3380                         cipherdata.key_type = RTA_DATA_PTR;
3381                 }
3382         }
3383
3384         if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
3385                 if (session->dir == DIR_ENC)
3386                         bufsize = cnstr_shdsc_pdcp_c_plane_encap(
3387                                         priv->flc_desc[0].desc, 1, swap,
3388                                         pdcp_xform->hfn,
3389                                         session->pdcp.sn_size,
3390                                         pdcp_xform->bearer,
3391                                         pdcp_xform->pkt_dir,
3392                                         pdcp_xform->hfn_threshold,
3393                                         &cipherdata, &authdata);
3394                 else if (session->dir == DIR_DEC)
3395                         bufsize = cnstr_shdsc_pdcp_c_plane_decap(
3396                                         priv->flc_desc[0].desc, 1, swap,
3397                                         pdcp_xform->hfn,
3398                                         session->pdcp.sn_size,
3399                                         pdcp_xform->bearer,
3400                                         pdcp_xform->pkt_dir,
3401                                         pdcp_xform->hfn_threshold,
3402                                         &cipherdata, &authdata);
3403
3404         } else if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_SHORT_MAC) {
3405                 bufsize = cnstr_shdsc_pdcp_short_mac(priv->flc_desc[0].desc,
3406                                                      1, swap, &authdata);
3407         } else {
3408                 if (session->dir == DIR_ENC) {
3409                         if (pdcp_xform->sdap_enabled)
3410                                 bufsize = cnstr_shdsc_pdcp_sdap_u_plane_encap(
3411                                         priv->flc_desc[0].desc, 1, swap,
3412                                         session->pdcp.sn_size,
3413                                         pdcp_xform->hfn,
3414                                         pdcp_xform->bearer,
3415                                         pdcp_xform->pkt_dir,
3416                                         pdcp_xform->hfn_threshold,
3417                                         &cipherdata, p_authdata);
3418                         else
3419                                 bufsize = cnstr_shdsc_pdcp_u_plane_encap(
3420                                         priv->flc_desc[0].desc, 1, swap,
3421                                         session->pdcp.sn_size,
3422                                         pdcp_xform->hfn,
3423                                         pdcp_xform->bearer,
3424                                         pdcp_xform->pkt_dir,
3425                                         pdcp_xform->hfn_threshold,
3426                                         &cipherdata, p_authdata);
3427                 } else if (session->dir == DIR_DEC) {
3428                         if (pdcp_xform->sdap_enabled)
3429                                 bufsize = cnstr_shdsc_pdcp_sdap_u_plane_decap(
3430                                         priv->flc_desc[0].desc, 1, swap,
3431                                         session->pdcp.sn_size,
3432                                         pdcp_xform->hfn,
3433                                         pdcp_xform->bearer,
3434                                         pdcp_xform->pkt_dir,
3435                                         pdcp_xform->hfn_threshold,
3436                                         &cipherdata, p_authdata);
3437                         else
3438                                 bufsize = cnstr_shdsc_pdcp_u_plane_decap(
3439                                         priv->flc_desc[0].desc, 1, swap,
3440                                         session->pdcp.sn_size,
3441                                         pdcp_xform->hfn,
3442                                         pdcp_xform->bearer,
3443                                         pdcp_xform->pkt_dir,
3444                                         pdcp_xform->hfn_threshold,
3445                                         &cipherdata, p_authdata);
3446                 }
3447         }
3448
3449         if (bufsize < 0) {
3450                 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
3451                 goto out;
3452         }
3453
3454         /* Enable the stashing control bit */
3455         DPAA2_SET_FLC_RSC(flc);
3456         flc->word2_rflc_31_0 = lower_32_bits(
3457                         (size_t)&(((struct dpaa2_sec_qp *)
3458                         dev->data->queue_pairs[0])->rx_vq) | 0x14);
3459         flc->word3_rflc_63_32 = upper_32_bits(
3460                         (size_t)&(((struct dpaa2_sec_qp *)
3461                         dev->data->queue_pairs[0])->rx_vq));
3462
3463         flc->word1_sdl = (uint8_t)bufsize;
3464
3465         /* TODO - check the perf impact or
3466          * align as per descriptor type
3467          * Set EWS bit i.e. enable write-safe
3468          * DPAA2_SET_FLC_EWS(flc);
3469          */
3470
3471         /* Set BS = 1 i.e reuse input buffers as output buffers */
3472         DPAA2_SET_FLC_REUSE_BS(flc);
3473         /* Set FF = 10; reuse input buffers if they provide sufficient space */
3474         DPAA2_SET_FLC_REUSE_FF(flc);
3475
3476         session->ctxt = priv;
3477
3478         return 0;
3479 out:
3480         rte_free(session->auth_key.data);
3481         rte_free(session->cipher_key.data);
3482         rte_free(priv);
3483         return -EINVAL;
3484 }
3485
3486 static int
3487 dpaa2_sec_security_session_create(void *dev,
3488                                   struct rte_security_session_conf *conf,
3489                                   struct rte_security_session *sess,
3490                                   struct rte_mempool *mempool)
3491 {
3492         void *sess_private_data;
3493         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
3494         int ret;
3495
3496         if (rte_mempool_get(mempool, &sess_private_data)) {
3497                 DPAA2_SEC_ERR("Couldn't get object from session mempool");
3498                 return -ENOMEM;
3499         }
3500
3501         switch (conf->protocol) {
3502         case RTE_SECURITY_PROTOCOL_IPSEC:
3503                 ret = dpaa2_sec_set_ipsec_session(cdev, conf,
3504                                 sess_private_data);
3505                 break;
3506         case RTE_SECURITY_PROTOCOL_MACSEC:
3507                 return -ENOTSUP;
3508         case RTE_SECURITY_PROTOCOL_PDCP:
3509                 ret = dpaa2_sec_set_pdcp_session(cdev, conf,
3510                                 sess_private_data);
3511                 break;
3512         default:
3513                 return -EINVAL;
3514         }
3515         if (ret != 0) {
3516                 DPAA2_SEC_ERR("Failed to configure session parameters");
3517                 /* Return session to mempool */
3518                 rte_mempool_put(mempool, sess_private_data);
3519                 return ret;
3520         }
3521
3522         set_sec_session_private_data(sess, sess_private_data);
3523
3524         return ret;
3525 }
3526
3527 /** Clear the memory of session so it doesn't leave key material behind */
3528 static int
3529 dpaa2_sec_security_session_destroy(void *dev __rte_unused,
3530                 struct rte_security_session *sess)
3531 {
3532         PMD_INIT_FUNC_TRACE();
3533         void *sess_priv = get_sec_session_private_data(sess);
3534
3535         dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv;
3536
3537         if (sess_priv) {
3538                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
3539
3540                 rte_free(s->ctxt);
3541                 rte_free(s->cipher_key.data);
3542                 rte_free(s->auth_key.data);
3543                 memset(s, 0, sizeof(dpaa2_sec_session));
3544                 set_sec_session_private_data(sess, NULL);
3545                 rte_mempool_put(sess_mp, sess_priv);
3546         }
3547         return 0;
3548 }
3549 #endif
3550 static int
3551 dpaa2_sec_sym_session_configure(struct rte_cryptodev *dev,
3552                 struct rte_crypto_sym_xform *xform,
3553                 struct rte_cryptodev_sym_session *sess,
3554                 struct rte_mempool *mempool)
3555 {
3556         void *sess_private_data;
3557         int ret;
3558
3559         if (rte_mempool_get(mempool, &sess_private_data)) {
3560                 DPAA2_SEC_ERR("Couldn't get object from session mempool");
3561                 return -ENOMEM;
3562         }
3563
3564         ret = dpaa2_sec_set_session_parameters(dev, xform, sess_private_data);
3565         if (ret != 0) {
3566                 DPAA2_SEC_ERR("Failed to configure session parameters");
3567                 /* Return session to mempool */
3568                 rte_mempool_put(mempool, sess_private_data);
3569                 return ret;
3570         }
3571
3572         set_sym_session_private_data(sess, dev->driver_id,
3573                 sess_private_data);
3574
3575         return 0;
3576 }
3577
3578 /** Clear the memory of session so it doesn't leave key material behind */
3579 static void
3580 dpaa2_sec_sym_session_clear(struct rte_cryptodev *dev,
3581                 struct rte_cryptodev_sym_session *sess)
3582 {
3583         PMD_INIT_FUNC_TRACE();
3584         uint8_t index = dev->driver_id;
3585         void *sess_priv = get_sym_session_private_data(sess, index);
3586         dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv;
3587
3588         if (sess_priv) {
3589                 rte_free(s->ctxt);
3590                 rte_free(s->cipher_key.data);
3591                 rte_free(s->auth_key.data);
3592                 memset(s, 0, sizeof(dpaa2_sec_session));
3593                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
3594                 set_sym_session_private_data(sess, index, NULL);
3595                 rte_mempool_put(sess_mp, sess_priv);
3596         }
3597 }
3598
3599 static int
3600 dpaa2_sec_dev_configure(struct rte_cryptodev *dev __rte_unused,
3601                         struct rte_cryptodev_config *config __rte_unused)
3602 {
3603         PMD_INIT_FUNC_TRACE();
3604
3605         return 0;
3606 }
3607
3608 static int
3609 dpaa2_sec_dev_start(struct rte_cryptodev *dev)
3610 {
3611         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3612         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3613         struct dpseci_attr attr;
3614         struct dpaa2_queue *dpaa2_q;
3615         struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3616                                         dev->data->queue_pairs;
3617         struct dpseci_rx_queue_attr rx_attr;
3618         struct dpseci_tx_queue_attr tx_attr;
3619         int ret, i;
3620
3621         PMD_INIT_FUNC_TRACE();
3622
3623         memset(&attr, 0, sizeof(struct dpseci_attr));
3624
3625         ret = dpseci_enable(dpseci, CMD_PRI_LOW, priv->token);
3626         if (ret) {
3627                 DPAA2_SEC_ERR("DPSECI with HW_ID = %d ENABLE FAILED",
3628                               priv->hw_id);
3629                 goto get_attr_failure;
3630         }
3631         ret = dpseci_get_attributes(dpseci, CMD_PRI_LOW, priv->token, &attr);
3632         if (ret) {
3633                 DPAA2_SEC_ERR("DPSEC ATTRIBUTE READ FAILED, disabling DPSEC");
3634                 goto get_attr_failure;
3635         }
3636         for (i = 0; i < attr.num_rx_queues && qp[i]; i++) {
3637                 dpaa2_q = &qp[i]->rx_vq;
3638                 dpseci_get_rx_queue(dpseci, CMD_PRI_LOW, priv->token, i,
3639                                     &rx_attr);
3640                 dpaa2_q->fqid = rx_attr.fqid;
3641                 DPAA2_SEC_DEBUG("rx_fqid: %d", dpaa2_q->fqid);
3642         }
3643         for (i = 0; i < attr.num_tx_queues && qp[i]; i++) {
3644                 dpaa2_q = &qp[i]->tx_vq;
3645                 dpseci_get_tx_queue(dpseci, CMD_PRI_LOW, priv->token, i,
3646                                     &tx_attr);
3647                 dpaa2_q->fqid = tx_attr.fqid;
3648                 DPAA2_SEC_DEBUG("tx_fqid: %d", dpaa2_q->fqid);
3649         }
3650
3651         return 0;
3652 get_attr_failure:
3653         dpseci_disable(dpseci, CMD_PRI_LOW, priv->token);
3654         return -1;
3655 }
3656
3657 static void
3658 dpaa2_sec_dev_stop(struct rte_cryptodev *dev)
3659 {
3660         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3661         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3662         int ret;
3663
3664         PMD_INIT_FUNC_TRACE();
3665
3666         ret = dpseci_disable(dpseci, CMD_PRI_LOW, priv->token);
3667         if (ret) {
3668                 DPAA2_SEC_ERR("Failure in disabling dpseci %d device",
3669                              priv->hw_id);
3670                 return;
3671         }
3672
3673         ret = dpseci_reset(dpseci, CMD_PRI_LOW, priv->token);
3674         if (ret < 0) {
3675                 DPAA2_SEC_ERR("SEC Device cannot be reset:Error = %0x", ret);
3676                 return;
3677         }
3678 }
3679
3680 static int
3681 dpaa2_sec_dev_close(struct rte_cryptodev *dev __rte_unused)
3682 {
3683         PMD_INIT_FUNC_TRACE();
3684
3685         return 0;
3686 }
3687
3688 static void
3689 dpaa2_sec_dev_infos_get(struct rte_cryptodev *dev,
3690                         struct rte_cryptodev_info *info)
3691 {
3692         struct dpaa2_sec_dev_private *internals = dev->data->dev_private;
3693
3694         PMD_INIT_FUNC_TRACE();
3695         if (info != NULL) {
3696                 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
3697                 info->feature_flags = dev->feature_flags;
3698                 info->capabilities = dpaa2_sec_capabilities;
3699                 /* No limit of number of sessions */
3700                 info->sym.max_nb_sessions = 0;
3701                 info->driver_id = cryptodev_driver_id;
3702         }
3703 }
3704
3705 static
3706 void dpaa2_sec_stats_get(struct rte_cryptodev *dev,
3707                          struct rte_cryptodev_stats *stats)
3708 {
3709         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3710         struct fsl_mc_io dpseci;
3711         struct dpseci_sec_counters counters = {0};
3712         struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3713                                         dev->data->queue_pairs;
3714         int ret, i;
3715
3716         PMD_INIT_FUNC_TRACE();
3717         if (stats == NULL) {
3718                 DPAA2_SEC_ERR("Invalid stats ptr NULL");
3719                 return;
3720         }
3721         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
3722                 if (qp == NULL || qp[i] == NULL) {
3723                         DPAA2_SEC_DEBUG("Uninitialised queue pair");
3724                         continue;
3725                 }
3726
3727                 stats->enqueued_count += qp[i]->tx_vq.tx_pkts;
3728                 stats->dequeued_count += qp[i]->rx_vq.rx_pkts;
3729                 stats->enqueue_err_count += qp[i]->tx_vq.err_pkts;
3730                 stats->dequeue_err_count += qp[i]->rx_vq.err_pkts;
3731         }
3732
3733         /* In case as secondary process access stats, MCP portal in priv-hw
3734          * may have primary process address. Need the secondary process
3735          * based MCP portal address for this object.
3736          */
3737         dpseci.regs = dpaa2_get_mcp_ptr(MC_PORTAL_INDEX);
3738         ret = dpseci_get_sec_counters(&dpseci, CMD_PRI_LOW, priv->token,
3739                                       &counters);
3740         if (ret) {
3741                 DPAA2_SEC_ERR("SEC counters failed");
3742         } else {
3743                 DPAA2_SEC_INFO("dpseci hardware stats:"
3744                             "\n\tNum of Requests Dequeued = %" PRIu64
3745                             "\n\tNum of Outbound Encrypt Requests = %" PRIu64
3746                             "\n\tNum of Inbound Decrypt Requests = %" PRIu64
3747                             "\n\tNum of Outbound Bytes Encrypted = %" PRIu64
3748                             "\n\tNum of Outbound Bytes Protected = %" PRIu64
3749                             "\n\tNum of Inbound Bytes Decrypted = %" PRIu64
3750                             "\n\tNum of Inbound Bytes Validated = %" PRIu64,
3751                             counters.dequeued_requests,
3752                             counters.ob_enc_requests,
3753                             counters.ib_dec_requests,
3754                             counters.ob_enc_bytes,
3755                             counters.ob_prot_bytes,
3756                             counters.ib_dec_bytes,
3757                             counters.ib_valid_bytes);
3758         }
3759 }
3760
3761 static
3762 void dpaa2_sec_stats_reset(struct rte_cryptodev *dev)
3763 {
3764         int i;
3765         struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3766                                    (dev->data->queue_pairs);
3767
3768         PMD_INIT_FUNC_TRACE();
3769
3770         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
3771                 if (qp[i] == NULL) {
3772                         DPAA2_SEC_DEBUG("Uninitialised queue pair");
3773                         continue;
3774                 }
3775                 qp[i]->tx_vq.rx_pkts = 0;
3776                 qp[i]->tx_vq.tx_pkts = 0;
3777                 qp[i]->tx_vq.err_pkts = 0;
3778                 qp[i]->rx_vq.rx_pkts = 0;
3779                 qp[i]->rx_vq.tx_pkts = 0;
3780                 qp[i]->rx_vq.err_pkts = 0;
3781         }
3782 }
3783
3784 static void __rte_hot
3785 dpaa2_sec_process_parallel_event(struct qbman_swp *swp,
3786                                  const struct qbman_fd *fd,
3787                                  const struct qbman_result *dq,
3788                                  struct dpaa2_queue *rxq,
3789                                  struct rte_event *ev)
3790 {
3791         /* Prefetching mbuf */
3792         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-
3793                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size));
3794
3795         /* Prefetching ipsec crypto_op stored in priv data of mbuf */
3796         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64));
3797
3798         ev->flow_id = rxq->ev.flow_id;
3799         ev->sub_event_type = rxq->ev.sub_event_type;
3800         ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
3801         ev->op = RTE_EVENT_OP_NEW;
3802         ev->sched_type = rxq->ev.sched_type;
3803         ev->queue_id = rxq->ev.queue_id;
3804         ev->priority = rxq->ev.priority;
3805         ev->event_ptr = sec_fd_to_mbuf(fd);
3806
3807         qbman_swp_dqrr_consume(swp, dq);
3808 }
3809 static void
3810 dpaa2_sec_process_atomic_event(struct qbman_swp *swp __rte_unused,
3811                                  const struct qbman_fd *fd,
3812                                  const struct qbman_result *dq,
3813                                  struct dpaa2_queue *rxq,
3814                                  struct rte_event *ev)
3815 {
3816         uint8_t dqrr_index;
3817         struct rte_crypto_op *crypto_op = (struct rte_crypto_op *)ev->event_ptr;
3818         /* Prefetching mbuf */
3819         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-
3820                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size));
3821
3822         /* Prefetching ipsec crypto_op stored in priv data of mbuf */
3823         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64));
3824
3825         ev->flow_id = rxq->ev.flow_id;
3826         ev->sub_event_type = rxq->ev.sub_event_type;
3827         ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
3828         ev->op = RTE_EVENT_OP_NEW;
3829         ev->sched_type = rxq->ev.sched_type;
3830         ev->queue_id = rxq->ev.queue_id;
3831         ev->priority = rxq->ev.priority;
3832
3833         ev->event_ptr = sec_fd_to_mbuf(fd);
3834         dqrr_index = qbman_get_dqrr_idx(dq);
3835         *dpaa2_seqn(crypto_op->sym->m_src) = dqrr_index + 1;
3836         DPAA2_PER_LCORE_DQRR_SIZE++;
3837         DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index;
3838         DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = crypto_op->sym->m_src;
3839 }
3840
3841 int
3842 dpaa2_sec_eventq_attach(const struct rte_cryptodev *dev,
3843                 int qp_id,
3844                 struct dpaa2_dpcon_dev *dpcon,
3845                 const struct rte_event *event)
3846 {
3847         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3848         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3849         struct dpaa2_sec_qp *qp = dev->data->queue_pairs[qp_id];
3850         struct dpseci_rx_queue_cfg cfg;
3851         uint8_t priority;
3852         int ret;
3853
3854         if (event->sched_type == RTE_SCHED_TYPE_PARALLEL)
3855                 qp->rx_vq.cb = dpaa2_sec_process_parallel_event;
3856         else if (event->sched_type == RTE_SCHED_TYPE_ATOMIC)
3857                 qp->rx_vq.cb = dpaa2_sec_process_atomic_event;
3858         else
3859                 return -EINVAL;
3860
3861         priority = (RTE_EVENT_DEV_PRIORITY_LOWEST / event->priority) *
3862                    (dpcon->num_priorities - 1);
3863
3864         memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
3865         cfg.options = DPSECI_QUEUE_OPT_DEST;
3866         cfg.dest_cfg.dest_type = DPSECI_DEST_DPCON;
3867         cfg.dest_cfg.dest_id = dpcon->dpcon_id;
3868         cfg.dest_cfg.priority = priority;
3869
3870         cfg.options |= DPSECI_QUEUE_OPT_USER_CTX;
3871         cfg.user_ctx = (size_t)(qp);
3872         if (event->sched_type == RTE_SCHED_TYPE_ATOMIC) {
3873                 cfg.options |= DPSECI_QUEUE_OPT_ORDER_PRESERVATION;
3874                 cfg.order_preservation_en = 1;
3875         }
3876         ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
3877                                   qp_id, &cfg);
3878         if (ret) {
3879                 RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret);
3880                 return ret;
3881         }
3882
3883         memcpy(&qp->rx_vq.ev, event, sizeof(struct rte_event));
3884
3885         return 0;
3886 }
3887
3888 int
3889 dpaa2_sec_eventq_detach(const struct rte_cryptodev *dev,
3890                         int qp_id)
3891 {
3892         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3893         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3894         struct dpseci_rx_queue_cfg cfg;
3895         int ret;
3896
3897         memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
3898         cfg.options = DPSECI_QUEUE_OPT_DEST;
3899         cfg.dest_cfg.dest_type = DPSECI_DEST_NONE;
3900
3901         ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
3902                                   qp_id, &cfg);
3903         if (ret)
3904                 RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret);
3905
3906         return ret;
3907 }
3908
3909 static struct rte_cryptodev_ops crypto_ops = {
3910         .dev_configure        = dpaa2_sec_dev_configure,
3911         .dev_start            = dpaa2_sec_dev_start,
3912         .dev_stop             = dpaa2_sec_dev_stop,
3913         .dev_close            = dpaa2_sec_dev_close,
3914         .dev_infos_get        = dpaa2_sec_dev_infos_get,
3915         .stats_get            = dpaa2_sec_stats_get,
3916         .stats_reset          = dpaa2_sec_stats_reset,
3917         .queue_pair_setup     = dpaa2_sec_queue_pair_setup,
3918         .queue_pair_release   = dpaa2_sec_queue_pair_release,
3919         .sym_session_get_size     = dpaa2_sec_sym_session_get_size,
3920         .sym_session_configure    = dpaa2_sec_sym_session_configure,
3921         .sym_session_clear        = dpaa2_sec_sym_session_clear,
3922         /* Raw data-path API related operations */
3923         .sym_get_raw_dp_ctx_size = dpaa2_sec_get_dp_ctx_size,
3924         .sym_configure_raw_dp_ctx = dpaa2_sec_configure_raw_dp_ctx,
3925 };
3926
3927 #ifdef RTE_LIB_SECURITY
3928 static const struct rte_security_capability *
3929 dpaa2_sec_capabilities_get(void *device __rte_unused)
3930 {
3931         return dpaa2_sec_security_cap;
3932 }
3933
3934 static const struct rte_security_ops dpaa2_sec_security_ops = {
3935         .session_create = dpaa2_sec_security_session_create,
3936         .session_update = NULL,
3937         .session_stats_get = NULL,
3938         .session_destroy = dpaa2_sec_security_session_destroy,
3939         .set_pkt_metadata = NULL,
3940         .capabilities_get = dpaa2_sec_capabilities_get
3941 };
3942 #endif
3943
3944 static int
3945 dpaa2_sec_uninit(const struct rte_cryptodev *dev)
3946 {
3947         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3948         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3949         int ret;
3950
3951         PMD_INIT_FUNC_TRACE();
3952
3953         /* Function is reverse of dpaa2_sec_dev_init.
3954          * It does the following:
3955          * 1. Detach a DPSECI from attached resources i.e. buffer pools, dpbp_id
3956          * 2. Close the DPSECI device
3957          * 3. Free the allocated resources.
3958          */
3959
3960         /*Close the device at underlying layer*/
3961         ret = dpseci_close(dpseci, CMD_PRI_LOW, priv->token);
3962         if (ret) {
3963                 DPAA2_SEC_ERR("Failure closing dpseci device: err(%d)", ret);
3964                 return -1;
3965         }
3966
3967         /*Free the allocated memory for ethernet private data and dpseci*/
3968         priv->hw = NULL;
3969         rte_free(dpseci);
3970         rte_free(dev->security_ctx);
3971         rte_mempool_free(priv->fle_pool);
3972
3973         DPAA2_SEC_INFO("Closing DPAA2_SEC device %s on numa socket %u",
3974                        dev->data->name, rte_socket_id());
3975
3976         return 0;
3977 }
3978
3979 static int
3980 check_devargs_handler(__rte_unused const char *key, const char *value,
3981                       __rte_unused void *opaque)
3982 {
3983         dpaa2_sec_dp_dump = atoi(value);
3984         if (dpaa2_sec_dp_dump > DPAA2_SEC_DP_FULL_DUMP) {
3985                 DPAA2_SEC_WARN("WARN: DPAA2_SEC_DP_DUMP_LEVEL is not "
3986                               "supported, changing to FULL error prints\n");
3987                 dpaa2_sec_dp_dump = DPAA2_SEC_DP_FULL_DUMP;
3988         }
3989
3990         return 0;
3991 }
3992
3993 static void
3994 dpaa2_sec_get_devargs(struct rte_devargs *devargs, const char *key)
3995 {
3996         struct rte_kvargs *kvlist;
3997
3998         if (!devargs)
3999                 return;
4000
4001         kvlist = rte_kvargs_parse(devargs->args, NULL);
4002         if (!kvlist)
4003                 return;
4004
4005         if (!rte_kvargs_count(kvlist, key)) {
4006                 rte_kvargs_free(kvlist);
4007                 return;
4008         }
4009
4010         rte_kvargs_process(kvlist, key,
4011                         check_devargs_handler, NULL);
4012         rte_kvargs_free(kvlist);
4013 }
4014
4015 static int
4016 dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev)
4017 {
4018         struct dpaa2_sec_dev_private *internals;
4019         struct rte_device *dev = cryptodev->device;
4020         struct rte_dpaa2_device *dpaa2_dev;
4021 #ifdef RTE_LIB_SECURITY
4022         struct rte_security_ctx *security_instance;
4023 #endif
4024         struct fsl_mc_io *dpseci;
4025         uint16_t token;
4026         struct dpseci_attr attr;
4027         int retcode, hw_id;
4028         char str[30];
4029
4030         PMD_INIT_FUNC_TRACE();
4031         dpaa2_dev = container_of(dev, struct rte_dpaa2_device, device);
4032         hw_id = dpaa2_dev->object_id;
4033
4034         cryptodev->driver_id = cryptodev_driver_id;
4035         cryptodev->dev_ops = &crypto_ops;
4036
4037         cryptodev->enqueue_burst = dpaa2_sec_enqueue_burst;
4038         cryptodev->dequeue_burst = dpaa2_sec_dequeue_burst;
4039         cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
4040                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
4041                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
4042                         RTE_CRYPTODEV_FF_SECURITY |
4043                         RTE_CRYPTODEV_FF_SYM_RAW_DP |
4044                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
4045                         RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
4046                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
4047                         RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
4048                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
4049
4050         internals = cryptodev->data->dev_private;
4051
4052         /*
4053          * For secondary processes, we don't initialise any further as primary
4054          * has already done this work. Only check we don't need a different
4055          * RX function
4056          */
4057         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
4058                 DPAA2_SEC_DEBUG("Device already init by primary process");
4059                 return 0;
4060         }
4061 #ifdef RTE_LIB_SECURITY
4062         /* Initialize security_ctx only for primary process*/
4063         security_instance = rte_malloc("rte_security_instances_ops",
4064                                 sizeof(struct rte_security_ctx), 0);
4065         if (security_instance == NULL)
4066                 return -ENOMEM;
4067         security_instance->device = (void *)cryptodev;
4068         security_instance->ops = &dpaa2_sec_security_ops;
4069         security_instance->sess_cnt = 0;
4070         cryptodev->security_ctx = security_instance;
4071 #endif
4072         /*Open the rte device via MC and save the handle for further use*/
4073         dpseci = (struct fsl_mc_io *)rte_calloc(NULL, 1,
4074                                 sizeof(struct fsl_mc_io), 0);
4075         if (!dpseci) {
4076                 DPAA2_SEC_ERR(
4077                         "Error in allocating the memory for dpsec object");
4078                 return -ENOMEM;
4079         }
4080         dpseci->regs = dpaa2_get_mcp_ptr(MC_PORTAL_INDEX);
4081
4082         retcode = dpseci_open(dpseci, CMD_PRI_LOW, hw_id, &token);
4083         if (retcode != 0) {
4084                 DPAA2_SEC_ERR("Cannot open the dpsec device: Error = %x",
4085                               retcode);
4086                 goto init_error;
4087         }
4088         retcode = dpseci_get_attributes(dpseci, CMD_PRI_LOW, token, &attr);
4089         if (retcode != 0) {
4090                 DPAA2_SEC_ERR(
4091                              "Cannot get dpsec device attributed: Error = %x",
4092                              retcode);
4093                 goto init_error;
4094         }
4095         snprintf(cryptodev->data->name, sizeof(cryptodev->data->name),
4096                         "dpsec-%u", hw_id);
4097
4098         internals->max_nb_queue_pairs = attr.num_tx_queues;
4099         cryptodev->data->nb_queue_pairs = internals->max_nb_queue_pairs;
4100         internals->hw = dpseci;
4101         internals->token = token;
4102
4103         snprintf(str, sizeof(str), "sec_fle_pool_p%d_%d",
4104                         getpid(), cryptodev->data->dev_id);
4105         internals->fle_pool = rte_mempool_create((const char *)str,
4106                         FLE_POOL_NUM_BUFS,
4107                         FLE_POOL_BUF_SIZE,
4108                         FLE_POOL_CACHE_SIZE, 0,
4109                         NULL, NULL, NULL, NULL,
4110                         SOCKET_ID_ANY, 0);
4111         if (!internals->fle_pool) {
4112                 DPAA2_SEC_ERR("Mempool (%s) creation failed", str);
4113                 goto init_error;
4114         }
4115
4116         dpaa2_sec_get_devargs(cryptodev->device->devargs, DRIVER_DUMP_MODE);
4117         DPAA2_SEC_INFO("driver %s: created", cryptodev->data->name);
4118         return 0;
4119
4120 init_error:
4121         DPAA2_SEC_ERR("driver %s: create failed", cryptodev->data->name);
4122
4123         /* dpaa2_sec_uninit(crypto_dev_name); */
4124         return -EFAULT;
4125 }
4126
4127 static int
4128 cryptodev_dpaa2_sec_probe(struct rte_dpaa2_driver *dpaa2_drv __rte_unused,
4129                           struct rte_dpaa2_device *dpaa2_dev)
4130 {
4131         struct rte_cryptodev *cryptodev;
4132         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
4133
4134         int retval;
4135
4136         snprintf(cryptodev_name, sizeof(cryptodev_name), "dpsec-%d",
4137                         dpaa2_dev->object_id);
4138
4139         cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id());
4140         if (cryptodev == NULL)
4141                 return -ENOMEM;
4142
4143         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
4144                 cryptodev->data->dev_private = rte_zmalloc_socket(
4145                                         "cryptodev private structure",
4146                                         sizeof(struct dpaa2_sec_dev_private),
4147                                         RTE_CACHE_LINE_SIZE,
4148                                         rte_socket_id());
4149
4150                 if (cryptodev->data->dev_private == NULL)
4151                         rte_panic("Cannot allocate memzone for private "
4152                                   "device data");
4153         }
4154
4155         dpaa2_dev->cryptodev = cryptodev;
4156         cryptodev->device = &dpaa2_dev->device;
4157
4158         /* init user callbacks */
4159         TAILQ_INIT(&(cryptodev->link_intr_cbs));
4160
4161         if (dpaa2_svr_family == SVR_LX2160A)
4162                 rta_set_sec_era(RTA_SEC_ERA_10);
4163         else
4164                 rta_set_sec_era(RTA_SEC_ERA_8);
4165
4166         DPAA2_SEC_INFO("2-SEC ERA is %d", rta_get_sec_era());
4167
4168         /* Invoke PMD device initialization function */
4169         retval = dpaa2_sec_dev_init(cryptodev);
4170         if (retval == 0) {
4171                 rte_cryptodev_pmd_probing_finish(cryptodev);
4172                 return 0;
4173         }
4174
4175         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
4176                 rte_free(cryptodev->data->dev_private);
4177
4178         cryptodev->attached = RTE_CRYPTODEV_DETACHED;
4179
4180         return -ENXIO;
4181 }
4182
4183 static int
4184 cryptodev_dpaa2_sec_remove(struct rte_dpaa2_device *dpaa2_dev)
4185 {
4186         struct rte_cryptodev *cryptodev;
4187         int ret;
4188
4189         cryptodev = dpaa2_dev->cryptodev;
4190         if (cryptodev == NULL)
4191                 return -ENODEV;
4192
4193         ret = dpaa2_sec_uninit(cryptodev);
4194         if (ret)
4195                 return ret;
4196
4197         return rte_cryptodev_pmd_destroy(cryptodev);
4198 }
4199
4200 static struct rte_dpaa2_driver rte_dpaa2_sec_driver = {
4201         .drv_flags = RTE_DPAA2_DRV_IOVA_AS_VA,
4202         .drv_type = DPAA2_CRYPTO,
4203         .driver = {
4204                 .name = "DPAA2 SEC PMD"
4205         },
4206         .probe = cryptodev_dpaa2_sec_probe,
4207         .remove = cryptodev_dpaa2_sec_remove,
4208 };
4209
4210 static struct cryptodev_driver dpaa2_sec_crypto_drv;
4211
4212 RTE_PMD_REGISTER_DPAA2(CRYPTODEV_NAME_DPAA2_SEC_PMD, rte_dpaa2_sec_driver);
4213 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa2_sec_crypto_drv,
4214                 rte_dpaa2_sec_driver.driver, cryptodev_driver_id);
4215 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_DPAA2_SEC_PMD,
4216                 DRIVER_DUMP_MODE "=<int>");
4217 RTE_LOG_REGISTER(dpaa2_logtype_sec, pmd.crypto.dpaa2, NOTICE);